diff --git a/.gitignore b/.gitignore index 2684429fc..cece5e29c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,11 @@ # Maven build folders target/ +# ... but not code generation targets +!tool/src/org/antlr/v4/codegen/target/ + +# Node.js (npm and typings) cached dependencies +node_modules/ +typings/ # Ant build folders build/ @@ -10,14 +16,37 @@ user.build.properties # MacOSX files .DS_Store -# Python -*.pyc +## Python, selected lines from https://raw.githubusercontent.com/github/gitignore/master/Python.gitignore +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class -# CSharp -bin/ -obj/ +## CSharp and VisualStudio, selected lines from https://raw.githubusercontent.com/github/gitignore/master/VisualStudio.gitignore +# User-specific files +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) *.userprefs +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ + +# Visual Studio 2015 cache/options directory +.vs/ + # NetBeans user configuration files nbactions*.xml /nbproject/private/ @@ -57,3 +86,7 @@ bilder.pyc bild.log bild_output.txt + +# VSCode Java plugin temporary files +javac-services.0.log +javac-services.0.log.lck diff --git a/.travis.yml b/.travis.yml index 7d884cccc..9de6a9ec6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,14 +6,16 @@ jdk: - openjdk6 - oraclejdk7 - oraclejdk8 -before_install: +before_install: - sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF - sudo add-apt-repository ppa:fkrull/deadsnakes -y - sudo add-apt-repository ppa:rwky/nodejs -y - - sudo apt-get update -qq - - sudo apt-get install -qq python3.4 + - sudo apt-get update -qq + - sudo apt-get install -qq python3.5 - sudo apt-get install -qq nodejs - echo "deb http://download.mono-project.com/repo/debian wheezy/snapshots/3.12.1 main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list - sudo apt-get install -qq mono-complete - eval "$(sudo gimme 1.6.2)" - go version ; go env + - python --version + - python3 --version diff --git a/README.md b/README.md index 9605f024c..cfba83348 100644 --- a/README.md +++ b/README.md @@ -40,18 +40,18 @@ ANTLR project lead and supreme dictator for life ## Useful information * [Release notes](https://github.com/antlr/antlr4/releases) -* [Getting started with v4](https://raw.githubusercontent.com/antlr/antlr4/master/doc/getting-started.md) +* [Getting started with v4](https://github.com/antlr/antlr4/blob/master/doc/getting-started.md) * [Official site](http://www.antlr.org/) -* [Documentation](https://raw.githubusercontent.com/antlr/antlr4/master/doc/index.md) -* [FAQ](https://raw.githubusercontent.com/antlr/antlr4/master/doc/faq/index.md) +* [Documentation](https://github.com/antlr/antlr4/blob/master/doc/index.md) +* [FAQ](https://github.com/antlr/antlr4/blob/master/doc/faq/index.md) * [API](http://www.antlr.org/api/Java/index.html) * [ANTLR v3](http://www.antlr3.org/) -* [v3 to v4 Migration, differences](https://raw.githubusercontent.com/antlr/antlr4/master/doc/faq/general.md) +* [v3 to v4 Migration, differences](https://github.com/antlr/antlr4/blob/master/doc/faq/general.md) You might also find the following pages useful, particularly if you want to mess around with the various target languages. -* [How to build ANTLR itself](https://raw.githubusercontent.com/antlr/antlr4/master/doc/building-antlr.md) -* [How we create and deploy an ANTLR release](https://raw.githubusercontent.com/antlr/antlr4/master/doc/releasing-antlr.md) +* [How to build ANTLR itself](https://github.com/antlr/antlr4/blob/master/doc/building-antlr.md) +* [How we create and deploy an ANTLR release](https://github.com/antlr/antlr4/blob/master/doc/releasing-antlr.md) ## The Definitive ANTLR 4 Reference @@ -61,8 +61,12 @@ You can buy the book [The Definitive ANTLR 4 Reference](http://amzn.com/19343569 You will find the [Book source code](http://pragprog.com/titles/tpantlr2/source_code) useful. - ## Additional grammars [This repository](https://github.com/antlr/grammars-v4) is a collection of grammars without actions where the root directory name is the all-lowercase name of the language parsed by the grammar. For example, java, cpp, csharp, c, etc... + +Travis Status +--------- + + diff --git a/antlr4-maven-plugin/pom.xml b/antlr4-maven-plugin/pom.xml index 4495a2a21..b9e144d33 100644 --- a/antlr4-maven-plugin/pom.xml +++ b/antlr4-maven-plugin/pom.xml @@ -34,7 +34,7 @@ org.antlr antlr4-master - 4.5.2-SNAPSHOT + 4.5.4-SNAPSHOT antlr4-maven-plugin maven-plugin diff --git a/antlr4-maven-plugin/src/main/java/org/antlr/mojo/antlr4/Antlr4Mojo.java b/antlr4-maven-plugin/src/main/java/org/antlr/mojo/antlr4/Antlr4Mojo.java index e7eb2d87d..d8712ee6a 100644 --- a/antlr4-maven-plugin/src/main/java/org/antlr/mojo/antlr4/Antlr4Mojo.java +++ b/antlr4-maven-plugin/src/main/java/org/antlr/mojo/antlr4/Antlr4Mojo.java @@ -372,6 +372,22 @@ public class Antlr4Mojo extends AbstractMojo { scan.addSourceMapping(mapping); Set grammarFiles = scan.getIncludedSources(sourceDirectory, null); + // We don't want the plugin to run for every grammar, regardless of whether + // it's changed since the last compilation. Check the mtime of the tokens vs + // the grammar file mtime to determine whether we even need to execute. + Set grammarFilesToProcess = new HashSet(); + + for (File grammarFile : grammarFiles) { + String tokensFileName = grammarFile.getName().split("\\.")[0] + ".tokens"; + File outputFile = new File(outputDirectory, tokensFileName); + if ( (! outputFile.exists()) || + outputFile.lastModified() < grammarFile.lastModified() ) { + grammarFilesToProcess.add(grammarFile); + } + } + + grammarFiles = grammarFilesToProcess; + if (grammarFiles.isEmpty()) { getLog().info("No grammars to process"); return Collections.emptyList(); diff --git a/contributors.txt b/contributors.txt index 9855eb39b..c9d35a21b 100644 --- a/contributors.txt +++ b/contributors.txt @@ -83,6 +83,16 @@ YYYY/MM/DD, github id, Full name, email 2015/10/12, KvanTTT, Ivan Kochurkin, ivan.kochurkin@gmail.com 2015/10/21, martin-probst, Martin Probst, martin-probst@web.de 2015/10/21, hkff, Walid Benghabrit, walid.benghabrit@mines-nantes.fr +2015/11/12, cooperra, Robbie Cooper, cooperra@users.noreply.github.com 2015/11/25, abego, Udo Borkowski, ub@abego.org +2015/12/17, sebadur, Sebastian Badur, sebadur@users.noreply.github.com +2015/12/23, pboyer, Peter Boyer, peter.b.boyer@gmail.com +2015/12/24, dtymon, David Tymon, david.tymon@gmail.com +2016/02/18, reitzig, Raphael Reitzig, reitzig[at]cs.uni-kl.de +2016/03/27, beardlybread, Bradley Steinbacher, bradley.j.steinbacher@gmail.com +2016/03/29, msteiger, Martin Steiger, antlr@martin-steiger.de +2016/03/28, gagern, Martin von Gagern, gagern@ma.tum.de 2016/07/18, willfaught, Will Faught, will.faught@gmail.com 2016/08/08, wjkohnen, Wolfgang Johannes Kohnen, wjkohnen-go-antlr@ko-sys.com +2016/08/11, BurtHarris, Ralph "Burt" Harris, Burt_Harris_antlr4@azxs.33mail.com +2016/08/19, andjo403, Andreas Jonson, andjo403@hotmail.com diff --git a/doc/ace-javascript-target.md b/doc/ace-javascript-target.md index 08a512ca7..fbba9e9b9 100644 --- a/doc/ace-javascript-target.md +++ b/doc/ace-javascript-target.md @@ -176,13 +176,13 @@ What remains to be done is have our validate function actually validate the inpu To start with, let's load ANTLR and your parser, listener etc.. Easy, since you could write: -``` +```js var antlr4 = require('antlr4/index'); ``` This may work, but it's actually unreliable. The reason is that the require function used by ANTLR, which exactly mimics the NodeJS require function, uses a different syntax than the require function that comes with ACE. So we need to bring in a require function that conforms to the NodeJS syntax. I personally use one that comes from Torben Haase's Honey project, which you can find here. But hey, now we're going to have 2 'require' functions not compatible with each other! Indeed, this is why you need to take special care, as follows: -``` +```js // load nodejs compatible require var ace_require = require; require = undefined; @@ -190,7 +190,9 @@ var Honey = { 'requirePath': ['..'] }; // walk up to js folder, see Honey docs importScripts("../lib/require.js"); var antlr4_require = require; require = ace_require; +``` Now it's safe to load antlr, and the parsers generated for your language. Assuming that your language files (generated or hand-built) are in a folder with an index.js file that calls require for each file, your parser loading code can be as simple as follows: +```js // load antlr4 and myLanguage var antlr4, mylanguage; try { @@ -200,10 +202,12 @@ try { } finally { require = ace_require; } +``` Please note the try-finally construct. ANTLR uses 'require' synchronously so it's perfectly safe to ignore the ACE 'require' while running ANTLR code. ACE itself does not guarantee synchronous execution, so you are much safer always switching 'require' back to 'ace_require'. Now detecting deep syntax errors in your code is a task for your ANTLR listener or visitor or whatever piece of code you've delegated this to. We're not going to describe this here, since it would require some knowledge of your language. However, detecting grammar syntax errors is something ANTLR does beautifully (isn't that why you went for ANTLR in the first place?). So what we will illustrate here is how to report grammar syntax errors. I have no doubt that from there, you will be able to extend the validator to suit your specific needs. Whenever ANTLR encounters an unexpected token, it fires an error. By default, the error is routed to an error listener which simply writes to the console. What we need to do is replace this listener by our own listener, se we can route errors to the ACE editor. First, let's create such a listener: +```js // class for gathering errors and posting them to ACE editor var AnnotatingErrorListener = function(annotations) { antlr4.error.ErrorListener.call(this); @@ -222,9 +226,9 @@ AnnotatingErrorListener.prototype.syntaxError = function(recognizer, offendingSy type: "error" }); }; - +``` With this, all that remains to be done is plug the listener in when we parse the code. Here is how I do it: - +```js var validate = function(input) { var stream = new antlr4.InputStream(input); var lexer = new mylanguage.MyLexer(stream); @@ -237,6 +241,7 @@ var validate = function(input) { parser.parseMyRule(); return annotations; }; +``` You know what? That's it! You now have an ACE editor that does syntax validation using ANTLR! I hope you find this useful, and simple enough to get started. What I did not address here is packaging, not something I'm an expert at. The good news is that it makes development simple, since I don't have to run any compilation process. I just edit my code, reload my editor page, and check how it goes. Now wait, hey! How do you debug this? Well, as usual, using Chrome, since neither Firefox or Safari are able to debug worker code. What a shame... diff --git a/doc/actions.md b/doc/actions.md index 4ca01f9ec..91b6de1e4 100644 --- a/doc/actions.md +++ b/doc/actions.md @@ -2,7 +2,7 @@ In Chapter 10, Attributes and Actions, we learned how to embed actions within grammars and looked at the most common token and rule attributes. This section summarizes the important syntax and semantics from that chapter and provides a complete list of all available attributes. (You can learn more about actions in the grammar from the free excerpt on listeners and actions.) -Actions are blocks of text written in the target language and enclosed in curly braces. The recognizer triggers them according to their locations within the grammar. For example, the following rule emits found a decl after the parser has seen a valid declaration: +Actions are blocks of text written in the target language and enclosed in curly braces. The recognizer triggers them according to their locations within the grammar. For example, the following rule emits "found a decl" after the parser has seen a valid declaration: ``` decl: type ID ';' {System.out.println("found a decl");} ; diff --git a/doc/adding-tests.md b/doc/adding-tests.md index 769383d31..4da6d4093 100644 --- a/doc/adding-tests.md +++ b/doc/adding-tests.md @@ -63,7 +63,22 @@ TestTemplates ::= [ ... ``` -For every name mentioned, you will find a `.stg` file with the actual test. E.g., `Sets/StarSet.stg`: +For every name mentioned, you will find a `.stg` file with the actual test template. E.g., `Sets/StarSet.stg`. + +Each `.stg` file descripes the following mandatory elements for the test: + - the test type: "Parser" or "Lexer" + - some ANTLR options, such as "Debug" + - the grammar + - the start rule + - the input i.e. the text to parse + - the expected output + - the expected errors + +The grammar can itself contain template expressions such as . +The test generator replaces these with the corresponding values from the target language template (see below). +It then generates a unit test in which the grammar, the input and the expected output and errors are inlined. + +Here is an example test template: ``` TestType() ::= "Parser" @@ -92,6 +107,7 @@ a : ('a'|'b')* 'c' {} ; >> ``` + ### Cross-language actions embedded within grammars To get: diff --git a/doc/building-antlr.md b/doc/building-antlr.md index f6780d970..b054438d1 100644 --- a/doc/building-antlr.md +++ b/doc/building-antlr.md @@ -13,14 +13,14 @@ The first step is to get the Java source code from the ANTLR 4 repository at git ```bash $ cd /tmp -/tmp $ git clone git@github.com:antlr/antlr4.git +/tmp $ git clone https://github.com/antlr/antlr4.git Cloning into 'antlr4'... -remote: Counting objects: 43273, done. -remote: Compressing objects: 100% (57/57), done. -remote: Total 43273 (delta 26), reused 0 (delta 0) -Receiving objects: 100% (43273/43273), 18.76 MiB | 1.60 MiB/s, done. -Resolving deltas: 100% (22419/22419), done. +remote: Counting objects: 61480, done. +remote: Total 61480 (delta 0), reused 0 (delta 0), pack-reused 61480 +Receiving objects: 100% (61480/61480), 31.24 MiB | 7.18 MiB/s, done. +Resolving deltas: 100% (32970/32970), done. Checking connectivity... done. +Checking out files: 100% (1427/1427), done. ``` # Compile diff --git a/doc/creating-a-language-target.md b/doc/creating-a-language-target.md index c926796bd..fe6150cb0 100644 --- a/doc/creating-a-language-target.md +++ b/doc/creating-a-language-target.md @@ -8,7 +8,7 @@ Creating a new target involves the following key elements: 1. For the tool, create class *X*Target as a subclass of class `Target` in package `org.antlr.v4.codegen.target`. This class describes language specific details about escape characters and strings and so on. There is very little to do here typically. 1. Create *X*.stg in directory tool/resources/org/antlr/v4/tool/templates/codegen/*X*/*X*.stg. This is a [StringTemplate](http://www.stringtemplate.org/) group file (`.stg`) that tells ANTLR how to express all of the parsing elements needed to generate code. You will see templates called `ParserFile`, `Parser`, `Lexer`, `CodeBlockForAlt`, `AltBlock`, etc... Each of these must be described how to build the indicated chunk of code. Your best bet is to find the closest existing target, copy that template file, and tweak to suit. -1. Create a runtime library to support the parsers generated by ANTLR. Under directory runtime/*X*, you are in complete control of the directory structure is dictated by common usage of that target language. For example, Java has: `runtime/Java/lib` and `runtime/Java/src` directories. Under `src`, you will find a directory structure for package `org.antlr.v4.runtime` and below. +1. Create a runtime library to support the parsers generated by ANTLR. Under directory runtime/*X*, you are in complete control of the directory structure as dictated by common usage of that target language. For example, Java has: `runtime/Java/lib` and `runtime/Java/src` directories. Under `src`, you will find a directory structure for package `org.antlr.v4.runtime` and below. 1. Create a template file for runtime tests. All you have to do is provide a few simple templates that indicate how to print values and declare variables. Our runtime test mechanism in dir `runtime-testsuite` will automatically generate code in a new target and check the results. All it needs to know is how to generate a test rig (i.e., a `main` program), how to define various class fields, compare members and so on. You must create a *X* directory underneath `runtime-testsuite/resources/org/antlr/v4/test/runtime`. Again, your best bet is to copy the templates from the closest language to your target and tweak it to suit. ## Getting started @@ -19,4 +19,4 @@ Creating a new target involves the following key elements: ```bash $ mvn compile ``` -That should proceed with success. See [Building ANTLR](building-antlr.md) for more details. (That link does not currently work as I have that documentation in a branch. see https://github.com/parrt/antlr4/blob/move-doc-to-repo/doc/building-antlr.md for now.) \ No newline at end of file +That should proceed with success. See [Building ANTLR](building-antlr.md) for more details. (That link does not currently work as I have that documentation in a branch. see https://github.com/parrt/antlr4/blob/move-doc-to-repo/doc/building-antlr.md for now.) diff --git a/doc/faq/general.md b/doc/faq/general.md index fb9c386f5..9189ac9ab 100644 --- a/doc/faq/general.md +++ b/doc/faq/general.md @@ -68,9 +68,9 @@ expr : expr '*' expr ; ``` -ANTLR 4 automatically constructs parse trees for you and abstract syntax tree (AST) construction is no longer an option. See also What if I need ASTs not parse trees for a compiler, for example? +ANTLR 4 automatically constructs parse trees for you and abstract syntax tree (AST) construction is no longer an option. See also [What if I need ASTs not parse trees for a compiler, for example?](https://github.com/antlr/antlr4/blob/master/doc/faq/parse-trees.md#what-if-i-need-asts-not-parse-trees-for-a-compiler-for-example). -Another big difference is that we discourage the use of actions directly within the grammar because ANTLR 4 automatically generates [listeners and visitors](https://raw.githubusercontent.com/antlr/antlr4/master/doc/listeners.md) for you to use that trigger method calls when some phrases of interest are recognized during a tree walk after parsing. See also [Parse Tree Matching and XPath](https://raw.githubusercontent.com/antlr/antlr4/master/doc/tree-matching.md). +Another big difference is that we discourage the use of actions directly within the grammar because ANTLR 4 automatically generates [listeners and visitors](https://github.com/antlr/antlr4/blob/master/doc/listeners.md) for you to use that trigger method calls when some phrases of interest are recognized during a tree walk after parsing. See also [Parse Tree Matching and XPath](https://github.com/antlr/antlr4/blob/master/doc/tree-matching.md). Semantic predicates are still allowed in both the parser and lexer rules as our actions. For efficiency sake keep semantic predicates to the right edge of lexical rules. diff --git a/doc/grammars.md b/doc/grammars.md index 3a8e77b60..c40d974b6 100644 --- a/doc/grammars.md +++ b/doc/grammars.md @@ -83,7 +83,7 @@ $ grun MyELang stat If there were any `tokens` specifications, the main grammar would merge the token sets. Any named actions such as `@members` would be merged. In general, you should avoid named actions and actions within rules in imported grammars since that limits their reuse. ANTLR also ignores any options in imported grammars. -Imported grammars can also import other grammars. ANTLR pursues all imported grammars in a depth-first fashion. If two or more imported grammars define ruler, ANTLR chooses the first version of `r` it finds. In the following diagram, ANTLR examines grammars in the following order `Nested`, `G1`, `G3`, `G2`. +Imported grammars can also import other grammars. ANTLR pursues all imported grammars in a depth-first fashion. If two or more imported grammars define rule `r`, ANTLR chooses the first version of `r` it finds. In the following diagram, ANTLR examines grammars in the following order `Nested`, `G1`, `G3`, `G2`. diff --git a/doc/left-recursion.md b/doc/left-recursion.md index a08f4357d..3430e10e9 100644 --- a/doc/left-recursion.md +++ b/doc/left-recursion.md @@ -1,6 +1,6 @@ # Left-recursive rules -The most natural expression of a some common language constructs is left recursive. For example C declarators and arithmetic expressions. Unfortunately, left recursive specifications of arithmetic expressions are typically ambiguous but much easier to write out than the multiple levels required in a typical top-down grammar. Here is a sample ANTLR 4 grammar with a left recursive expression rule: +The most natural expression of some common language constructs is left recursive. For example C declarators and arithmetic expressions. Unfortunately, left recursive specifications of arithmetic expressions are typically ambiguous but much easier to write out than the multiple levels required in a typical top-down grammar. Here is a sample ANTLR 4 grammar with a left recursive expression rule: ``` stat: expr '=' expr ';' // e.g., x=y; or x=f(x); diff --git a/doc/lexer-rules.md b/doc/lexer-rules.md index 2141b4303..adda9e8b0 100644 --- a/doc/lexer-rules.md +++ b/doc/lexer-rules.md @@ -171,7 +171,7 @@ error(126): P.g4:3:4: cannot create implicit token for string literal '&' in non ## Lexer Rule Actions -An ANTLR lexer creates a Token object after matching a lexical rule. Each request for a token starts in Lexer.nextToken, which calls emit once it has identified a token.emit collects information from the current state of the lexer to build the token. It accesses fields `_type`, `_text`, `_channel`, `_tokenStartCharIndex`, `_tokenStartLine`, and `_tokenStartCharPositionInLine`. You can set the state of these with the various setter methods such as `setType`. For example, the following rule turns `enum` into an identifier if `enumIsKeyword` is false. +An ANTLR lexer creates a Token object after matching a lexical rule. Each request for a token starts in `Lexer.nextToken`, which calls `emit` once it has identified a token. `emit` collects information from the current state of the lexer to build the token. It accesses fields `_type`, `_text`, `_channel`, `_tokenStartCharIndex`, `_tokenStartLine`, and `_tokenStartCharPositionInLine`. You can set the state of these with the various setter methods such as `setType`. For example, the following rule turns `enum` into an identifier if `enumIsKeyword` is false. ``` ENUM : 'enum' {if (!enumIsKeyword) setType(Identifier);} ; @@ -255,7 +255,8 @@ WS : [ \r\t\n]+ -> skip ; ``` For multiple 'type()' commands, only the rightmost has an effect. -channel() + +### channel() ``` BLOCK_COMMENT diff --git a/doc/listeners.md b/doc/listeners.md index 94b105771..c3bcad9c1 100644 --- a/doc/listeners.md +++ b/doc/listeners.md @@ -8,7 +8,7 @@ By default, ANTLR-generated parsers build a data structure called a parse tree o The interior nodes of the parse tree are phrase names that group and identify their children. The root node is the most abstract phrase name, in this case `stat` (short for statement). The leaves of a parse tree are always the input tokens. Parse trees sit between a language recognizer and an interpreter or translator implementation. They are extremely effective data structures because they contain all of the input and complete knowledge of how the parser grouped the symbols into phrases. Better yet, they are easy to understand and the parser generates them automatically (unless you turn them off with `parser.setBuildParseTree(false)`). -Because we specify phrase structure with a set of rules, parse tree subtree roots correspond to grammar rule names. ANTLR has a ParseTreeWalker that knows how to walk these parse trees and trigger events in listener implementation objects that you can create. ANTLR the tool generates listener interfaces for you also unless you, unless you turn that off with a commandline option. You can also have it generate visitors. For example from a Java.g4 grammar, ANTLR generates: +Because we specify phrase structure with a set of rules, parse tree subtree roots correspond to grammar rule names. ANTLR has a ParseTreeWalker that knows how to walk these parse trees and trigger events in listener implementation objects that you can create. The ANTLR tool generates listener interfaces for you also, unless you turn that off with a commandline option. You can also have it generate visitors. For example from a Java.g4 grammar, ANTLR generates: ```java public interface JavaListener extends ParseTreeListener { @@ -27,11 +27,10 @@ Assuming you've created a listener object called `MyListener`, here is how to ca JavaLexer lexer = new JavaLexer(input); CommonTokenStream tokens = new CommonTokenStream(lexer); JavaParser parser = new JavaParser(tokens); -ParserRuleContext tree = parser.compilationUnit(); // parse - -ParseTreeWalker walker = new ParseTreeWalker(); // create standard walker +JavaParser.CompilationUnitContext tree = parser.compilationUnit(); // parse a compilationUnit + MyListener extractor = new MyListener(parser); -walker.walk(extractor, tree); // initiate walk of tree with listener +ParseTreeWalker.DEFAULT.walk(extractor, tree); // initiate walk of tree with listener in use of default walker ``` Listeners and visitors are great because they keep application-specific code out of grammars, making grammars easier to read and preventing them from getting entangled with a particular application. diff --git a/doc/options.md b/doc/options.md index 1b2c2591b..7ce277551 100644 --- a/doc/options.md +++ b/doc/options.md @@ -12,6 +12,55 @@ where a value can be an identifier, a qualified identifier (for example, a.b.c), All grammars can use the following options. In combined grammars, all options except language pertain only to the generated parser. Options may be set either within the grammar file using the options syntax (described above) or when invoking ANTLR on the command line, using the `-D` option. (see Section 15.9, [ANTLR Tool Command Line Options](tool-options.md).) The following examples demonstrate both mechanisms; note that `-D` overrides options within the grammar. +* `superClass`. Set the superclass of the generated parser or lexer. For combined grammars, it sets the superclass of the parser. +``` +$ cat Hi.g4 +grammar Hi; +a : 'hi' ; +$ antlr4 -DsuperClass=XX Hi.g4 +$ grep 'public class' HiParser.java +public class HiParser extends XX { +$ grep 'public class' HiLexer.java +public class HiLexer extends Lexer { +``` +* `language` Generate code in the indicated language, if ANTLR is able to do so. Otherwise, you will see an error message like this: +``` +$ antlr4 -Dlanguage=C MyGrammar.g4 +error(31): ANTLR cannot generate C code as of version 4.0 +``` +* `tokenVocab` ANTLR assigns token type numbers to the tokens as it encounters them in a file. To use different token type values, such as with a separate lexer, use this option to have ANTLR pull in the tokens file. ANTLR generates a tokens file from each grammar. +``` +$ cat SomeLexer.g4 +lexer grammar SomeLexer; +ID : [a-z]+ ; +$ cat R.g4 +parser grammar R; +options {tokenVocab=SomeLexer;} +tokens {A,B,C} // normally, these would be token types 1, 2, 3 +a : ID ; +$ antlr4 SomeLexer.g4 +$ cat SomeLexer.tokens +ID=1 +$ antlr4 R.g4 +$ cat R.tokens +A=2 +B=3 +C=4 +ID=1 +``` +* `TokenLabelType` ANTLR normally uses type Token when it generates variables referencing tokens. If you have passed a TokenFactory to your parser and lexer so that they create custom tokens, you should set this option to your specific type. This ensures that the context objects know your type for fields and method return values. +``` +$ cat T2.g4 +grammar T2; +options {TokenLabelType=MyToken;} +a : x=ID ; +$ antlr4 T2.g4 +$ grep MyToken T2Parser.java + public MyToken x; +``` +* `contextSuperClass`. Specify the super class of parse tree internal nodes. Default is `ParserRuleContext`. Should derive from ultimately `RuleContext` at minimum. +Java target can use `contextSuperClass=org.antlr.v4.runtime.RuleContextWithAltNum` for convenience. It adds a backing field for `altNumber`, the alt matched for the associated rule node. + ## Rule Options There are currently no valid rule-level options, but the tool still supports the following syntax for future use: @@ -25,7 +74,7 @@ options {...} ## Rule Element Options -Token options have the form `T` as we saw in Section 5.4, [Dealing with Precedence, Left Recursion, and Associativity](http://pragprog.com/book/tpantlr2/the-definitive-antlr-4-reference). The only token option is assocand it accepts values left and right. Here’s a sample grammar with a left-recursive expression rule that specifies a token option on the `^` exponent operator token: +Token options have the form `T` as we saw in Section 5.4, [Dealing with Precedence, Left Recursion, and Associativity](http://pragprog.com/book/tpantlr2/the-definitive-antlr-4-reference). The only token option is `assoc`, and it accepts values `left` and `right`. Here’s a sample grammar with a left-recursive expression rule that specifies a token option on the `^` exponent operator token: ``` grammar ExprLR; @@ -40,7 +89,7 @@ INT : '0'..'9'+ ; WS : [ \n]+ -> skip ; ``` -Semantic predicates also accept an option, per [Catching failed semantic predicates](http://pragprog.com/book/tpantlr2/the-definitive-antlr-4-reference). The only valid option is the fail option, which takes either a string literal in double-quotes or an action that evaluates to a string. The string literal or string result from the action should be the message to emit upon predicate failure. +Semantic predicates also accept an option, per [Catching failed semantic predicates](http://pragprog.com/book/tpantlr2/the-definitive-antlr-4-reference). The only valid option is the `fail` option, which takes either a string literal in double-quotes or an action that evaluates to a string. The string literal or string result from the action should be the message to emit upon predicate failure. ``` ints[int max] diff --git a/doc/predicates.md b/doc/predicates.md index af1d278e7..09998c425 100644 --- a/doc/predicates.md +++ b/doc/predicates.md @@ -29,7 +29,7 @@ expr: {istype()}? ID '(' expr ')' // ctor-style typecast ; ``` -The parser will only predict an expr from stat when `istype()||isfunc()` evaluates to true. This makes sense because the parser should only choose to match an expression if the upcoming `ID` is a type name or function name. It wouldn't make sense to just test one of the predicates in this case. Note that, when the parser gets to expritself, the parsing decision tests the predicates individually, one for each alternative. +The parser will only predict an expr from stat when `istype()||isfunc()` evaluates to true. This makes sense because the parser should only choose to match an expression if the upcoming `ID` is a type name or function name. It wouldn't make sense to just test one of the predicates in this case. Note that, when the parser gets to `expr` itself, the parsing decision tests the predicates individually, one for each alternative. If multiple predicates occur in a sequence, the parser joins them with the `&&` operator. For example, consider changing `stat` to include a predicate before the call `toexpr`: @@ -72,7 +72,7 @@ stat: {System.out.println("goto"); allowgoto=true;} {java5}? 'goto' ID ';' If we can't execute the action during prediction, we shouldn't evaluate the `{java5}?` predicate because it depends on that action. -The prediction process also can't see through token references. Token references have the side effect of advancing the input one symbol. A predicate that tested the current input symbol would find itself out of sync if the parser shifted it over the token reference. For example, in the following grammar, the predicates expectgetCurrentToken to return an ID token. +The prediction process also can't see through token references. Token references have the side effect of advancing the input one symbol. A predicate that tested the current input symbol would find itself out of sync if the parser shifted it over the token reference. For example, in the following grammar, the predicates expect `getCurrentToken` to return an `ID` token. ``` stat: '{' decl '}' diff --git a/doc/releasing-antlr.md b/doc/releasing-antlr.md index 8131f8c9b..05815086e 100644 --- a/doc/releasing-antlr.md +++ b/doc/releasing-antlr.md @@ -2,15 +2,16 @@ ## Github -Create a release candidate tag 4.x-rc-1 or full 4.5 tag - -```bash -git tag -a 4.5 -m 'ANTLR final release 4.5' -git push origin 4.5 -``` - Create a pre-release or full release at github; [Example 4.5-rc-1](https://github.com/antlr/antlr4/releases/tag/4.5-rc-1). +Wack any existing tag as mvn will create one and it fails if already there. + +``` +$ git tag -d 4.5.2 +$ git push origin :refs/tags/4.5.2 +$ git push upstream :refs/tags/4.5.2 +``` + ## Bump version Edit the repository looking for 4.5 or whatever and update it. Bump version in the following files: @@ -20,7 +21,7 @@ Edit the repository looking for 4.5 or whatever and update it. Bump version in t * runtime/Python2/src/antlr4/Recognizer.py * runtime/Python3/setup.py * runtime/Python3/src/antlr4/Recognizer.py - * runtime/CSharp/Antlr4.Runtime/Properties/AssemblyInfo.cs + * runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Properties/AssemblyInfo.cs * runtime/JavaScript/src/antlr4/package.json * runtime/JavaScript/src/antlr4/Recognizer.js * tool/src/org/antlr/v4/codegen/target/CSharpTarget.java @@ -89,8 +90,10 @@ The maven deploy lifecycle phased deploys the artifacts and the poms for the ANT mvn deploy -DskipTests ``` +With JDK 1.7 (not 6 or 8), do this: + ```bash -mvn release:prepare +mvn release:prepare -Darguments="-DskipTests" ``` It will start out by asking you the version number: @@ -111,7 +114,7 @@ What is the new development version for "ANTLR 4"? (org.antlr:antlr4-master) 4.5 Maven will go through your pom.xml files to update versions from 4.5.2-SNAPSHOT to 4.5.2 for release and then to 4.5.3-SNAPSHOT after release, which is done with: ```bash -mvn release:perform +mvn release:perform -Darguments="-DskipTests" ``` Maven will use git to push pom.xml changes. (big smile) @@ -132,16 +135,21 @@ cp ~/.m2/repository/org/antlr/antlr4/4.5.2/antlr4-4.5.2.jar ~/antlr/sites/websit cd ~/antlr/sites/website-antlr4/download git add antlr-4.5.2-complete.jar git add antlr-runtime-4.5.2.jar -git commit -a -m 'add 4.5.2 jars' -git push origin gh-pages ``` Update on site: * download.html * index.html +* api/index.html +* download/index.html * scripts/topnav.js +``` +git commit -a -m 'add 4.5.2 jars' +git push origin gh-pages +``` + ## Deploying Targets ### JavaScript diff --git a/doc/targets.md b/doc/targets.md index 00058a4af..170431d93 100644 --- a/doc/targets.md +++ b/doc/targets.md @@ -18,7 +18,6 @@ The [ANTLR v4 book](http://pragprog.com/book/tpantlr2/the-definitive-antlr-4-ref New features generally appear in the Java target and then migrate to the other targets, but these other targets don't always get updated in the same overall tool release. This section tries to identify features added to Java that have not been added to the other targets. |Feature|Java|C♯|JavaScript|Python2|Python3|Swift|C++| -|-|-|-|-|-|-|-|-| +|---|---|---|---|---|---|---|---| |Ambiguous tree construction|4.5.1|-|-|-|-|-|-| - - + diff --git a/doc/tree-matching.md b/doc/tree-matching.md index 5b7d94b58..f4c6d278c 100644 --- a/doc/tree-matching.md +++ b/doc/tree-matching.md @@ -29,7 +29,7 @@ ParseTreeMatch m = p.match(t); if ( m.succeeded() ) {...} ``` -We can also test for specific expressions or token values. For example, the following checks to see if t is an expression consisting of an identifier added to 0: +We can also test for specific expressions or token values. For example, the following checks to see if `t` is an expression consisting of an identifier added to 0: ```java ParseTree t = ...; // assume t is an expression diff --git a/pom.xml b/pom.xml index 92ca59769..e8cb3f6b6 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.antlr antlr4-master - 4.5.2-SNAPSHOT + 4.5.4-SNAPSHOT pom ANTLR 4 diff --git a/runtime-testsuite/pom.xml b/runtime-testsuite/pom.xml index 38c7b6779..11c7541df 100644 --- a/runtime-testsuite/pom.xml +++ b/runtime-testsuite/pom.xml @@ -4,7 +4,7 @@ org.antlr antlr4-master - 4.5.2-SNAPSHOT + 4.5.4-SNAPSHOT antlr4-runtime-testsuite ANTLR 4 Runtime Test Generator diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/csharp/CSharp.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/csharp/CSharp.test.stg index 17bfda1ae..1a8642a35 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/csharp/CSharp.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/csharp/CSharp.test.stg @@ -333,6 +333,19 @@ ParseTreeWalker walker = new ParseTreeWalker(); walker.Walk(new LeafListener(), ); >> +TreeNodeWithAltNumField(X) ::= << +@parser::members { +public class MyRuleNode : ParserRuleContext { + public int altNum; + public MyRuleNode(ParserRuleContext parent, int invokingStateNumber): base(parent, invokingStateNumber) + { + } + public override int getAltNumber() { return altNum; } + public override void setAltNumber(int altNum) { this.altNum = altNum; } +} +} +>> + TokenGetterListener(X) ::= << public class LeafListener : TBaseListener { public override void ExitA(TParser.AContext ctx) { diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/java/Java.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/java/Java.test.stg index 210b1682e..c63e306c8 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/java/Java.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/java/Java.test.stg @@ -344,6 +344,19 @@ ParseTreeWalker walker = new ParseTreeWalker(); walker.walk(new LeafListener(), ); >> +TreeNodeWithAltNumField(X) ::= << +@parser::members { +public static class MyRuleNode extends ParserRuleContext { + public int altNum; + public MyRuleNode(ParserRuleContext parent, int invokingStateNumber) { + super(parent, invokingStateNumber); + } + @Override public int getAltNumber() { return altNum; } + @Override public void setAltNumber(int altNum) { this.altNum = altNum; } +} +} +>> + TokenGetterListener(X) ::= << public static class LeafListener extends TBaseListener { public void exitA(TParser.AContext ctx) { diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/javascript/chrome/Chrome.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/javascript/chrome/Chrome.test.stg index 4022ca552..fdc16e29a 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/javascript/chrome/Chrome.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/javascript/chrome/Chrome.test.stg @@ -328,6 +328,20 @@ var walker = new antlr4.tree.ParseTreeWalker(); walker.walk(new this.LeafListener(), ); >> +TreeNodeWithAltNumField(X) ::= << + +@parser::header { +MyRuleNode = function(parent, invokingState) { + antlr4.ParserRuleContext.call(this, parent, invokingState); + this.altNum = 0; + return this; +}; + +MyRuleNode.prototype = Object.create(antlr4.ParserRuleContext.prototype); +MyRuleNode.prototype.constructor = MyRuleNode; +} +>> + TokenGetterListener(X) ::= << this.LeafListener = function() { this.exitA = function(ctx) { diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/javascript/explorer/Explorer.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/javascript/explorer/Explorer.test.stg index 69f89741b..0f2c5dc90 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/javascript/explorer/Explorer.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/javascript/explorer/Explorer.test.stg @@ -328,6 +328,20 @@ var walker = new antlr4.tree.ParseTreeWalker(); walker.walk(new this.LeafListener(), ); >> +TreeNodeWithAltNumField(X) ::= << + +@parser::header { +MyRuleNode = function(parent, invokingState) { + antlr4.ParserRuleContext.call(this, parent, invokingState); + this.altNum = 0; + return this; +}; + +MyRuleNode.prototype = Object.create(antlr4.ParserRuleContext.prototype); +MyRuleNode.prototype.constructor = MyRuleNode; +} +>> + TokenGetterListener(X) ::= << this.LeafListener = function() { this.exitA = function(ctx) { diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/javascript/firefox/Firefox.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/javascript/firefox/Firefox.test.stg index d814179a1..d50d2e7b9 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/javascript/firefox/Firefox.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/javascript/firefox/Firefox.test.stg @@ -330,6 +330,20 @@ var walker = new antlr4.tree.ParseTreeWalker(); walker.walk(new this.LeafListener(), ); >> +TreeNodeWithAltNumField(X) ::= << + +@parser::header { +MyRuleNode = function(parent, invokingState) { + antlr4.ParserRuleContext.call(this, parent, invokingState); + this.altNum = 0; + return this; +}; + +MyRuleNode.prototype = Object.create(antlr4.ParserRuleContext.prototype); +MyRuleNode.prototype.constructor = MyRuleNode; +} +>> + TokenGetterListener(X) ::= << this.LeafListener = function() { this.exitA = function(ctx) { diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/javascript/node/Node.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/javascript/node/Node.test.stg index 3cc616318..8a11ff670 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/javascript/node/Node.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/javascript/node/Node.test.stg @@ -328,6 +328,24 @@ var walker = new antlr4.tree.ParseTreeWalker(); walker.walk(new this.LeafListener(), ); >> +TreeNodeWithAltNumField(X) ::= << + +@parser::header { +MyRuleNode = function(parent, invokingState) { + antlr4.ParserRuleContext.call(this, parent, invokingState); + + this.altNum = 0; + return this; +}; + +MyRuleNode.prototype = Object.create(antlr4.ParserRuleContext.prototype); +MyRuleNode.prototype.constructor = MyRuleNode; +MyRuleNode.prototype.getAltNumber = function() { return this.altNum; } +MyRuleNode.prototype.setAltNumber = function(altNumber) { this.altNum = altNumber; } + +} +>> + TokenGetterListener(X) ::= << this.LeafListener = function() { this.exitA = function(ctx) { diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/javascript/safari/Safari.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/javascript/safari/Safari.test.stg index d5d6f8d76..2b1b08c99 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/javascript/safari/Safari.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/javascript/safari/Safari.test.stg @@ -328,6 +328,21 @@ var walker = new antlr4.tree.ParseTreeWalker(); walker.walk(new this.LeafListener(), ); >> +TreeNodeWithAltNumField(X) ::= << + +@parser::header { +MyRuleNode = function(parent, invokingState) { + antlr4.ParserRuleContext.call(this, parent, invokingState); + this.altNum = 0; + return this; +}; + +MyRuleNode.prototype = Object.create(antlr4.ParserRuleContext.prototype); +MyRuleNode.prototype.constructor = MyRuleNode; +} +>> + + TokenGetterListener(X) ::= << this.LeafListener = function() { this.exitA = function(ctx) { diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/python2/Python2.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/python2/Python2.test.stg index 13aea5457..b9664b1fe 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/python2/Python2.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/python2/Python2.test.stg @@ -318,6 +318,19 @@ walker = ParseTreeWalker() walker.walk(TParser.LeafListener(), ) >> +TreeNodeWithAltNumField(X) ::= << +@parser::members { +class MyRuleNode(ParserRuleContext): + def __init__(self, parent = None, invokingStateNumber = None ): + super(Parser.MyRuleNode, self).__init__(parent, invokingStateNumber) + self.altNum = 0; + def getAltNumber(self): + return self.altNum + def setAltNumber(self, altNum): + self.altNum = altNum +} +>> + TokenGetterListener(X) ::= << if __name__ is not None and "." in __name__: from .Listener import Listener diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/python3/Python3.test.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/python3/Python3.test.stg index 73ba0197f..5cd74b50c 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/python3/Python3.test.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/python3/Python3.test.stg @@ -320,6 +320,19 @@ walker = ParseTreeWalker() walker.walk(TParser.LeafListener(), ) >> +TreeNodeWithAltNumField(X) ::= << +@parser::members { +class MyRuleNode(ParserRuleContext): + def __init__(self, parent:ParserRuleContext = None, invokingStateNumber:int = None ): + super(Parser.MyRuleNode, self).__init__(parent, invokingStateNumber) + self.altNum = 0; + def getAltNumber(self): + return self.altNum + def setAltNumber(self, altNum): + self.altNum = altNum +} +>> + TokenGetterListener(X) ::= << class LeafListener(MockListener): def exitA(self, ctx): diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/Index.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/Index.stg index 5d2166e71..617914d9d 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/Index.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/Index.stg @@ -59,6 +59,8 @@ TestTemplates ::= [ "MultipleAlternativesWithCommonLabel_2": [], "MultipleAlternativesWithCommonLabel_3": [], "MultipleAlternativesWithCommonLabel_4": [], + "PrefixAndOtherAlt_1": [], + "PrefixAndOtherAlt_2": [], "PrefixOpWithActionAndLabel_1": [], "PrefixOpWithActionAndLabel_2": [], "PrefixOpWithActionAndLabel_3": [], diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/PrefixAndOtherAlt.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/PrefixAndOtherAlt.stg new file mode 100644 index 000000000..37d022826 --- /dev/null +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/PrefixAndOtherAlt.stg @@ -0,0 +1,24 @@ +TestType() ::= "Parser" + +Options ::= [ + "Debug": false +] + +Grammar ::= [ + "T": {} +] + +Rule() ::= "s" + +grammar(grammarName) ::= << +grammar ; +s @after {} : expr EOF ; +expr : literal + | op expr + | expr op expr + ; +literal : '-'? Integer ; +op : '+' | '-' ; +Integer : [0-9]+ ; +WS : (' '|'\n') -> skip ; +>> \ No newline at end of file diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/PrefixAndOtherAlt_1.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/PrefixAndOtherAlt_1.stg new file mode 100644 index 000000000..553b557f0 --- /dev/null +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/PrefixAndOtherAlt_1.stg @@ -0,0 +1,9 @@ +import "PrefixAndOtherAlt.stg" + +Input() ::= "-1" + +Output() ::= << +(s (expr (literal - 1)) \)<\n> +>> + +Errors() ::= "" \ No newline at end of file diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/PrefixAndOtherAlt_2.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/PrefixAndOtherAlt_2.stg new file mode 100644 index 000000000..de4cea9c5 --- /dev/null +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/PrefixAndOtherAlt_2.stg @@ -0,0 +1,9 @@ +import "PrefixAndOtherAlt.stg" + +Input() ::= "-1 + -1" + +Output() ::= << +(s (expr (expr (literal - 1)) (op +) (expr (literal - 1))) \)<\n> +>> + +Errors() ::= "" \ No newline at end of file diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/ParseTrees/AltNum.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/ParseTrees/AltNum.stg new file mode 100644 index 000000000..124112c1f --- /dev/null +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/ParseTrees/AltNum.stg @@ -0,0 +1,40 @@ +TestType() ::= "Parser" + +Grammar ::= [ + "T": {} +] + +Input() ::= "xyz" + +Rule() ::= "s" + +Output() ::= << +(a:3 x (b:2 y) z)<\n> +>> + +Errors() ::= "" + +grammar(grammarName) ::= << +grammar ; + +options { contextSuperClass=MyRuleNode; } + + + + +s +@init { + +} +@after { + +} + : r=a ; + +a : 'f' + | 'g' + | 'x' b 'z' + ; +b : 'e' {} | 'y' + ; +>> diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/ParseTrees/Index.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/ParseTrees/Index.stg index 5f71a146c..a8bc31d2d 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/ParseTrees/Index.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/ParseTrees/Index.stg @@ -6,5 +6,6 @@ TestTemplates ::= [ "RuleRef": [], "ExtraToken": [], "NoViableAlt": [], - "Sync": [] + "Sync": [], + "AltNum": [] ] diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseTest.java index 23eb6eca4..14848a7aa 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseTest.java @@ -482,6 +482,9 @@ public abstract class BaseTest { throw new RuntimeException("C# runtime project file not found!"); } String runtimeProjPath = runtimeProj.getPath(); + if(isWindows()){ + runtimeProjPath = runtimeProjPath.replaceFirst("/", ""); + } XPathExpression exp = XPathFactory.newInstance().newXPath() .compile("/Project/ItemGroup/ProjectReference[@Include='" + runtimeName + "']"); Element node = (Element)exp.evaluate(prjXml, XPathConstants.NODE); diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/TestLeftRecursion.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/TestLeftRecursion.java index 595123e38..b3b91fdaf 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/TestLeftRecursion.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/TestLeftRecursion.java @@ -1801,6 +1801,50 @@ public class TestLeftRecursion extends BaseTest { assertEquals("(prog (statement (letterA a)) (statement (letterA a)) )\n", found); assertNull(this.stderrDuringParse); + } + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ + @Test + public void testPrefixAndOtherAlt_1() throws Exception { + mkdir(tmpdir); + StringBuilder grammarBuilder = new StringBuilder(223); + grammarBuilder.append("grammar T;\n"); + grammarBuilder.append("s @after {Console.WriteLine($ctx.ToStringTree(this));} : expr EOF ; \n"); + grammarBuilder.append("expr : literal\n"); + grammarBuilder.append(" | op expr\n"); + grammarBuilder.append(" | expr op expr\n"); + grammarBuilder.append(" ;\n"); + grammarBuilder.append("literal : '-'? Integer ;\n"); + grammarBuilder.append("op : '+' | '-' ;\n"); + grammarBuilder.append("Integer : [0-9]+ ;\n"); + grammarBuilder.append("WS : (' '|'\\n') -> skip ;"); + String grammar = grammarBuilder.toString(); + String input ="-1"; + String found = execParser("T.g4", grammar, "TParser", "TLexer", "s", input, false); + assertEquals("(s (expr (literal - 1)) )\n", found); + assertNull(this.stderrDuringParse); + + } + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ + @Test + public void testPrefixAndOtherAlt_2() throws Exception { + mkdir(tmpdir); + StringBuilder grammarBuilder = new StringBuilder(223); + grammarBuilder.append("grammar T;\n"); + grammarBuilder.append("s @after {Console.WriteLine($ctx.ToStringTree(this));} : expr EOF ; \n"); + grammarBuilder.append("expr : literal\n"); + grammarBuilder.append(" | op expr\n"); + grammarBuilder.append(" | expr op expr\n"); + grammarBuilder.append(" ;\n"); + grammarBuilder.append("literal : '-'? Integer ;\n"); + grammarBuilder.append("op : '+' | '-' ;\n"); + grammarBuilder.append("Integer : [0-9]+ ;\n"); + grammarBuilder.append("WS : (' '|'\\n') -> skip ;"); + String grammar = grammarBuilder.toString(); + String input ="-1 + -1"; + String found = execParser("T.g4", grammar, "TParser", "TLexer", "s", input, false); + assertEquals("(s (expr (expr (literal - 1)) (op +) (expr (literal - 1))) )\n", found); + assertNull(this.stderrDuringParse); + } /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ @Test diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/TestParseTrees.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/TestParseTrees.java index 4f6e1b322..3ff7bb4e1 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/TestParseTrees.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/TestParseTrees.java @@ -2,7 +2,6 @@ package org.antlr.v4.test.runtime.csharp; import org.junit.Test; -import org.junit.Ignore; @SuppressWarnings("unused") public class TestParseTrees extends BaseTest { @@ -52,6 +51,49 @@ public class TestParseTrees extends BaseTest { assertEquals("(a y)\n", found); assertNull(this.stderrDuringParse); + } + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ + @Test + public void testAltNum() throws Exception { + mkdir(tmpdir); + StringBuilder grammarBuilder = new StringBuilder(547); + grammarBuilder.append("grammar T;\n"); + grammarBuilder.append("\n"); + grammarBuilder.append("options { contextSuperClass=MyRuleNode; }\n"); + grammarBuilder.append("\n"); + grammarBuilder.append("@parser::members {\n"); + grammarBuilder.append("public class MyRuleNode : ParserRuleContext {\n"); + grammarBuilder.append(" public int altNum;\n"); + grammarBuilder.append(" public MyRuleNode(ParserRuleContext parent, int invokingStateNumber): base(parent, invokingStateNumber)\n"); + grammarBuilder.append(" {\n"); + grammarBuilder.append(" }\n"); + grammarBuilder.append(" public override int getAltNumber() { return altNum; }\n"); + grammarBuilder.append(" public override void setAltNumber(int altNum) { this.altNum = altNum; }\n"); + grammarBuilder.append("}\n"); + grammarBuilder.append("}\n"); + grammarBuilder.append("\n"); + grammarBuilder.append("\n"); + grammarBuilder.append("s\n"); + grammarBuilder.append("@init {\n"); + grammarBuilder.append("this.BuildParseTree = true;\n"); + grammarBuilder.append("}\n"); + grammarBuilder.append("@after {\n"); + grammarBuilder.append("Console.WriteLine($r.ctx.ToStringTree(this));\n"); + grammarBuilder.append("}\n"); + grammarBuilder.append(" : r=a ;\n"); + grammarBuilder.append("\n"); + grammarBuilder.append("a : 'f'\n"); + grammarBuilder.append(" | 'g'\n"); + grammarBuilder.append(" | 'x' b 'z'\n"); + grammarBuilder.append(" ;\n"); + grammarBuilder.append("b : 'e' {} | 'y'\n"); + grammarBuilder.append(" ;"); + String grammar = grammarBuilder.toString(); + String input ="xyz"; + String found = execParser("T.g4", grammar, "TParser", "TLexer", "s", input, false); + assertEquals("(a:3 x (b:2 y) z)\n", found); + assertNull(this.stderrDuringParse); + } /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ @Test diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/java/TestLeftRecursion.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/java/TestLeftRecursion.java index 6606f5ba0..2f72610f3 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/java/TestLeftRecursion.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/java/TestLeftRecursion.java @@ -2007,6 +2007,58 @@ public class TestLeftRecursion extends BaseTest { } + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ + @Test + public void testPrefixAndOtherAlt_1() throws Exception { + mkdir(tmpdir); + + StringBuilder grammarBuilder = new StringBuilder(224); + grammarBuilder.append("grammar T;\n"); + grammarBuilder.append("s @after {System.out.println($ctx.toStringTree(this));} : expr EOF ; \n"); + grammarBuilder.append("expr : literal\n"); + grammarBuilder.append(" | op expr\n"); + grammarBuilder.append(" | expr op expr\n"); + grammarBuilder.append(" ;\n"); + grammarBuilder.append("literal : '-'? Integer ;\n"); + grammarBuilder.append("op : '+' | '-' ;\n"); + grammarBuilder.append("Integer : [0-9]+ ;\n"); + grammarBuilder.append("WS : (' '|'\\n') -> skip ;"); + String grammar = grammarBuilder.toString(); + + + String input ="-1"; + String found = execParser("T.g4", grammar, "TParser", "TLexer", "s", input, false); + assertEquals("(s (expr (literal - 1)) )\n", found); + assertNull(this.stderrDuringParse); + + } + + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ + @Test + public void testPrefixAndOtherAlt_2() throws Exception { + mkdir(tmpdir); + + StringBuilder grammarBuilder = new StringBuilder(224); + grammarBuilder.append("grammar T;\n"); + grammarBuilder.append("s @after {System.out.println($ctx.toStringTree(this));} : expr EOF ; \n"); + grammarBuilder.append("expr : literal\n"); + grammarBuilder.append(" | op expr\n"); + grammarBuilder.append(" | expr op expr\n"); + grammarBuilder.append(" ;\n"); + grammarBuilder.append("literal : '-'? Integer ;\n"); + grammarBuilder.append("op : '+' | '-' ;\n"); + grammarBuilder.append("Integer : [0-9]+ ;\n"); + grammarBuilder.append("WS : (' '|'\\n') -> skip ;"); + String grammar = grammarBuilder.toString(); + + + String input ="-1 + -1"; + String found = execParser("T.g4", grammar, "TParser", "TLexer", "s", input, false); + assertEquals("(s (expr (expr (literal - 1)) (op +) (expr (literal - 1))) )\n", found); + assertNull(this.stderrDuringParse); + + } + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ @Test public void testPrefixOpWithActionAndLabel_1() throws Exception { diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/java/TestParseTrees.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/java/TestParseTrees.java index 7067a2b53..7a0a944df 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/java/TestParseTrees.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/java/TestParseTrees.java @@ -1,10 +1,10 @@ /* This file is generated by TestGenerator, any edits will be overwritten by the next generation. */ package org.antlr.v4.test.runtime.java; -import org.junit.Ignore; import org.junit.Test; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; public class TestParseTrees extends BaseTest { @@ -62,6 +62,53 @@ public class TestParseTrees extends BaseTest { } + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ + @Test + public void testAltNum() throws Exception { + mkdir(tmpdir); + + StringBuilder grammarBuilder = new StringBuilder(562); + grammarBuilder.append("grammar T;\n"); + grammarBuilder.append("\n"); + grammarBuilder.append("options { contextSuperClass=MyRuleNode; }\n"); + grammarBuilder.append("\n"); + grammarBuilder.append("@parser::members {\n"); + grammarBuilder.append("public static class MyRuleNode extends ParserRuleContext {\n"); + grammarBuilder.append(" public int altNum;\n"); + grammarBuilder.append(" public MyRuleNode(ParserRuleContext parent, int invokingStateNumber) {\n"); + grammarBuilder.append(" super(parent, invokingStateNumber);\n"); + grammarBuilder.append(" }\n"); + grammarBuilder.append(" @Override public int getAltNumber() { return altNum; }\n"); + grammarBuilder.append(" @Override public void setAltNumber(int altNum) { this.altNum = altNum; }\n"); + grammarBuilder.append("}\n"); + grammarBuilder.append("}\n"); + grammarBuilder.append("\n"); + grammarBuilder.append("\n"); + grammarBuilder.append("s\n"); + grammarBuilder.append("@init {\n"); + grammarBuilder.append("setBuildParseTree(true);\n"); + grammarBuilder.append("}\n"); + grammarBuilder.append("@after {\n"); + grammarBuilder.append("System.out.println($r.ctx.toStringTree(this));\n"); + grammarBuilder.append("}\n"); + grammarBuilder.append(" : r=a ;\n"); + grammarBuilder.append("\n"); + grammarBuilder.append("a : 'f'\n"); + grammarBuilder.append(" | 'g'\n"); + grammarBuilder.append(" | 'x' b 'z'\n"); + grammarBuilder.append(" ;\n"); + grammarBuilder.append("b : 'e' {} | 'y'\n"); + grammarBuilder.append(" ;"); + String grammar = grammarBuilder.toString(); + + + String input ="xyz"; + String found = execParser("T.g4", grammar, "TParser", "TLexer", "s", input, false); + assertEquals("(a:3 x (b:2 y) z)\n", found); + assertNull(this.stderrDuringParse); + + } + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ @Test public void testExtraToken() throws Exception { diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/node/BaseTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/node/BaseTest.java index 59b68d14c..8db0f0716 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/node/BaseTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/node/BaseTest.java @@ -445,9 +445,16 @@ public abstract class BaseTest { if ( runtimeSrc==null ) { throw new RuntimeException("Cannot find JavaScript runtime"); } + if(isWindows()){ + return runtimeSrc.getPath().replaceFirst("/", ""); + } return runtimeSrc.getPath(); } + private boolean isWindows() { + return System.getProperty("os.name").toLowerCase().contains("windows"); + } + public void testErrors(String[] pairs, boolean printTree) { for (int i = 0; i < pairs.length; i += 2) { String input = pairs[i]; diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/node/TestLeftRecursion.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/node/TestLeftRecursion.java index 53fb15efb..2b392fc44 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/node/TestLeftRecursion.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/node/TestLeftRecursion.java @@ -1905,6 +1905,54 @@ public class TestLeftRecursion extends BaseTest { assertEquals("(prog (statement (letterA a)) (statement (letterA a)) )\n", found); assertNull(this.stderrDuringParse); + } + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ + @Test + public void testPrefixAndOtherAlt_1() throws Exception { + mkdir(tmpdir); + StringBuilder grammarBuilder = new StringBuilder(223); + grammarBuilder.append("grammar T;\n"); + grammarBuilder.append("s @after {console.log($ctx.toStringTree(null, this));} : expr EOF ; \n"); + grammarBuilder.append("expr : literal\n"); + grammarBuilder.append(" | op expr\n"); + grammarBuilder.append(" | expr op expr\n"); + grammarBuilder.append(" ;\n"); + grammarBuilder.append("literal : '-'? Integer ;\n"); + grammarBuilder.append("op : '+' | '-' ;\n"); + grammarBuilder.append("Integer : [0-9]+ ;\n"); + grammarBuilder.append("WS : (' '|'\\n') -> skip ;"); + String grammar = grammarBuilder.toString(); + String input ="-1"; + String found = execParser("T.g4", grammar, "TParser", "TLexer", + "TListener", "TVisitor", + "s", input, false); + assertEquals("(s (expr (literal - 1)) )\n", found); + assertNull(this.stderrDuringParse); + + } + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ + @Test + public void testPrefixAndOtherAlt_2() throws Exception { + mkdir(tmpdir); + StringBuilder grammarBuilder = new StringBuilder(223); + grammarBuilder.append("grammar T;\n"); + grammarBuilder.append("s @after {console.log($ctx.toStringTree(null, this));} : expr EOF ; \n"); + grammarBuilder.append("expr : literal\n"); + grammarBuilder.append(" | op expr\n"); + grammarBuilder.append(" | expr op expr\n"); + grammarBuilder.append(" ;\n"); + grammarBuilder.append("literal : '-'? Integer ;\n"); + grammarBuilder.append("op : '+' | '-' ;\n"); + grammarBuilder.append("Integer : [0-9]+ ;\n"); + grammarBuilder.append("WS : (' '|'\\n') -> skip ;"); + String grammar = grammarBuilder.toString(); + String input ="-1 + -1"; + String found = execParser("T.g4", grammar, "TParser", "TLexer", + "TListener", "TVisitor", + "s", input, false); + assertEquals("(s (expr (expr (literal - 1)) (op +) (expr (literal - 1))) )\n", found); + assertNull(this.stderrDuringParse); + } /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ @Test diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/node/TestParseTrees.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/node/TestParseTrees.java index c1f9afa92..2dd5b992d 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/node/TestParseTrees.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/node/TestParseTrees.java @@ -1,10 +1,10 @@ /* This file is generated by TestGenerator, any edits will be overwritten by the next generation. */ package org.antlr.v4.test.runtime.javascript.node; -import org.junit.Ignore; import org.junit.Test; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; @SuppressWarnings("unused") public class TestParseTrees extends BaseTest { @@ -58,6 +58,55 @@ public class TestParseTrees extends BaseTest { assertEquals("(a y)\n", found); assertNull(this.stderrDuringParse); + } + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ + @Test + public void testAltNum() throws Exception { + mkdir(tmpdir); + StringBuilder grammarBuilder = new StringBuilder(663); + grammarBuilder.append("grammar T;\n"); + grammarBuilder.append("\n"); + grammarBuilder.append("options { contextSuperClass=MyRuleNode; }\n"); + grammarBuilder.append("\n"); + grammarBuilder.append("@parser::header {\n"); + grammarBuilder.append("MyRuleNode = function(parent, invokingState) {\n"); + grammarBuilder.append(" antlr4.ParserRuleContext.call(this, parent, invokingState);\n"); + grammarBuilder.append("\n"); + grammarBuilder.append(" this.altNum = 0;\n"); + grammarBuilder.append(" return this;\n"); + grammarBuilder.append("};\n"); + grammarBuilder.append("\n"); + grammarBuilder.append("MyRuleNode.prototype = Object.create(antlr4.ParserRuleContext.prototype);\n"); + grammarBuilder.append("MyRuleNode.prototype.constructor = MyRuleNode;\n"); + grammarBuilder.append("MyRuleNode.prototype.getAltNumber = function() { return this.altNum; }\n"); + grammarBuilder.append("MyRuleNode.prototype.setAltNumber = function(altNumber) { this.altNum = altNumber; }\n"); + grammarBuilder.append("\n"); + grammarBuilder.append("}\n"); + grammarBuilder.append("\n"); + grammarBuilder.append("\n"); + grammarBuilder.append("s\n"); + grammarBuilder.append("@init {\n"); + grammarBuilder.append("this.buildParseTrees = true;\n"); + grammarBuilder.append("}\n"); + grammarBuilder.append("@after {\n"); + grammarBuilder.append("console.log($r.ctx.toStringTree(null, this));\n"); + grammarBuilder.append("}\n"); + grammarBuilder.append(" : r=a ;\n"); + grammarBuilder.append("\n"); + grammarBuilder.append("a : 'f'\n"); + grammarBuilder.append(" | 'g'\n"); + grammarBuilder.append(" | 'x' b 'z'\n"); + grammarBuilder.append(" ;\n"); + grammarBuilder.append("b : 'e' {} | 'y'\n"); + grammarBuilder.append(" ;"); + String grammar = grammarBuilder.toString(); + String input ="xyz"; + String found = execParser("T.g4", grammar, "TParser", "TLexer", + "TListener", "TVisitor", + "s", input, false); + assertEquals("(a:3 x (b:2 y) z)\n", found); + assertNull(this.stderrDuringParse); + } /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ @Test diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/python/BasePythonTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/python/BasePythonTest.java index ab91d9b02..f221f3008 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/python/BasePythonTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/python/BasePythonTest.java @@ -583,9 +583,16 @@ public abstract class BasePythonTest { if ( runtimeSrc==null ) { throw new RuntimeException("Cannot find "+targetName+" runtime"); } + if(isWindows()){ + return runtimeSrc.getPath().replaceFirst("/", ""); + } return runtimeSrc.getPath(); } + private boolean isWindows() { + return System.getProperty("os.name").toLowerCase().contains("windows"); + } + public void testErrors(String[] pairs, boolean printTree) { for (int i = 0; i < pairs.length; i+=2) { String input = pairs[i]; diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/python2/TestLeftRecursion.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/python2/TestLeftRecursion.java index 8e07e462a..d65523920 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/python2/TestLeftRecursion.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/python2/TestLeftRecursion.java @@ -2058,6 +2058,60 @@ public class TestLeftRecursion extends BasePython2Test { } + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ + @Test + public void testPrefixAndOtherAlt_1() throws Exception { + mkdir(tmpdir); + + StringBuilder grammarBuilder = new StringBuilder(216); + grammarBuilder.append("grammar T;\n"); + grammarBuilder.append("s @after {print($ctx.toStringTree(recog=self))} : expr EOF ; \n"); + grammarBuilder.append("expr : literal\n"); + grammarBuilder.append(" | op expr\n"); + grammarBuilder.append(" | expr op expr\n"); + grammarBuilder.append(" ;\n"); + grammarBuilder.append("literal : '-'? Integer ;\n"); + grammarBuilder.append("op : '+' | '-' ;\n"); + grammarBuilder.append("Integer : [0-9]+ ;\n"); + grammarBuilder.append("WS : (' '|'\\n') -> skip ;"); + String grammar = grammarBuilder.toString(); + + + String input ="-1"; + String found = execParser("T.g4", grammar, "TParser", "TLexer", "TListener", "TVisitor", "s", input, false); + + assertEquals("(s (expr (literal - 1)) )\n", found); + assertNull(this.stderrDuringParse); + + } + + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ + @Test + public void testPrefixAndOtherAlt_2() throws Exception { + mkdir(tmpdir); + + StringBuilder grammarBuilder = new StringBuilder(216); + grammarBuilder.append("grammar T;\n"); + grammarBuilder.append("s @after {print($ctx.toStringTree(recog=self))} : expr EOF ; \n"); + grammarBuilder.append("expr : literal\n"); + grammarBuilder.append(" | op expr\n"); + grammarBuilder.append(" | expr op expr\n"); + grammarBuilder.append(" ;\n"); + grammarBuilder.append("literal : '-'? Integer ;\n"); + grammarBuilder.append("op : '+' | '-' ;\n"); + grammarBuilder.append("Integer : [0-9]+ ;\n"); + grammarBuilder.append("WS : (' '|'\\n') -> skip ;"); + String grammar = grammarBuilder.toString(); + + + String input ="-1 + -1"; + String found = execParser("T.g4", grammar, "TParser", "TLexer", "TListener", "TVisitor", "s", input, false); + + assertEquals("(s (expr (expr (literal - 1)) (op +) (expr (literal - 1))) )\n", found); + assertNull(this.stderrDuringParse); + + } + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ @Test public void testPrefixOpWithActionAndLabel_1() throws Exception { diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/python2/TestParseTrees.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/python2/TestParseTrees.java index b94a2959e..da8e26f07 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/python2/TestParseTrees.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/python2/TestParseTrees.java @@ -1,9 +1,10 @@ /* This file is generated by TestGenerator, any edits will be overwritten by the next generation. */ package org.antlr.v4.test.runtime.python2; -import org.junit.Ignore; import org.junit.Test; -import static org.junit.Assert.*; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; @SuppressWarnings("unused") public class TestParseTrees extends BasePython2Test { @@ -64,6 +65,54 @@ public class TestParseTrees extends BasePython2Test { } + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ + @Test + public void testAltNum() throws Exception { + mkdir(tmpdir); + + StringBuilder grammarBuilder = new StringBuilder(562); + grammarBuilder.append("grammar T;\n"); + grammarBuilder.append("\n"); + grammarBuilder.append("options { contextSuperClass=MyRuleNode; }\n"); + grammarBuilder.append("\n"); + grammarBuilder.append("@parser::members {\n"); + grammarBuilder.append("class MyRuleNode(ParserRuleContext):\n"); + grammarBuilder.append(" def __init__(self, parent = None, invokingStateNumber = None ):\n"); + grammarBuilder.append(" super(TParser.MyRuleNode, self).__init__(parent, invokingStateNumber)\n"); + grammarBuilder.append(" self.altNum = 0;\n"); + grammarBuilder.append(" def getAltNumber(self):\n"); + grammarBuilder.append(" return self.altNum\n"); + grammarBuilder.append(" def setAltNumber(self, altNum):\n"); + grammarBuilder.append(" self.altNum = altNum\n"); + grammarBuilder.append("}\n"); + grammarBuilder.append("\n"); + grammarBuilder.append("\n"); + grammarBuilder.append("s\n"); + grammarBuilder.append("@init {\n"); + grammarBuilder.append("self._buildParseTrees = True\n"); + grammarBuilder.append("}\n"); + grammarBuilder.append("@after {\n"); + grammarBuilder.append("print($r.ctx.toStringTree(recog=self))\n"); + grammarBuilder.append("}\n"); + grammarBuilder.append(" : r=a ;\n"); + grammarBuilder.append("\n"); + grammarBuilder.append("a : 'f'\n"); + grammarBuilder.append(" | 'g'\n"); + grammarBuilder.append(" | 'x' b 'z'\n"); + grammarBuilder.append(" ;\n"); + grammarBuilder.append("b : 'e' {} | 'y'\n"); + grammarBuilder.append(" ;"); + String grammar = grammarBuilder.toString(); + + + String input ="xyz"; + String found = execParser("T.g4", grammar, "TParser", "TLexer", "TListener", "TVisitor", "s", input, false); + + assertEquals("(a:3 x (b:2 y) z)\n", found); + assertNull(this.stderrDuringParse); + + } + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ @Test public void testExtraToken() throws Exception { diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/python3/BasePython3Test.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/python3/BasePython3Test.java index faf2d763a..fdcbdabb4 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/python3/BasePython3Test.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/python3/BasePython3Test.java @@ -41,8 +41,8 @@ public abstract class BasePython3Test extends BasePythonTest { @Override protected String getPythonExecutable() { - return "python3.4"; - } + return "python3.5"; + } // force 3.5 @Override protected void writeLexerTestFile(String lexerName, boolean showDFA) { diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/python3/TestLeftRecursion.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/python3/TestLeftRecursion.java index 3b856ebe9..72e8341a7 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/python3/TestLeftRecursion.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/python3/TestLeftRecursion.java @@ -2058,6 +2058,60 @@ public class TestLeftRecursion extends BasePython3Test { } + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ + @Test + public void testPrefixAndOtherAlt_1() throws Exception { + mkdir(tmpdir); + + StringBuilder grammarBuilder = new StringBuilder(216); + grammarBuilder.append("grammar T;\n"); + grammarBuilder.append("s @after {print($ctx.toStringTree(recog=self))} : expr EOF ; \n"); + grammarBuilder.append("expr : literal\n"); + grammarBuilder.append(" | op expr\n"); + grammarBuilder.append(" | expr op expr\n"); + grammarBuilder.append(" ;\n"); + grammarBuilder.append("literal : '-'? Integer ;\n"); + grammarBuilder.append("op : '+' | '-' ;\n"); + grammarBuilder.append("Integer : [0-9]+ ;\n"); + grammarBuilder.append("WS : (' '|'\\n') -> skip ;"); + String grammar = grammarBuilder.toString(); + + + String input ="-1"; + String found = execParser("T.g4", grammar, "TParser", "TLexer", "TListener", "TVisitor", "s", input, false); + + assertEquals("(s (expr (literal - 1)) )\n", found); + assertNull(this.stderrDuringParse); + + } + + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ + @Test + public void testPrefixAndOtherAlt_2() throws Exception { + mkdir(tmpdir); + + StringBuilder grammarBuilder = new StringBuilder(216); + grammarBuilder.append("grammar T;\n"); + grammarBuilder.append("s @after {print($ctx.toStringTree(recog=self))} : expr EOF ; \n"); + grammarBuilder.append("expr : literal\n"); + grammarBuilder.append(" | op expr\n"); + grammarBuilder.append(" | expr op expr\n"); + grammarBuilder.append(" ;\n"); + grammarBuilder.append("literal : '-'? Integer ;\n"); + grammarBuilder.append("op : '+' | '-' ;\n"); + grammarBuilder.append("Integer : [0-9]+ ;\n"); + grammarBuilder.append("WS : (' '|'\\n') -> skip ;"); + String grammar = grammarBuilder.toString(); + + + String input ="-1 + -1"; + String found = execParser("T.g4", grammar, "TParser", "TLexer", "TListener", "TVisitor", "s", input, false); + + assertEquals("(s (expr (expr (literal - 1)) (op +) (expr (literal - 1))) )\n", found); + assertNull(this.stderrDuringParse); + + } + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ @Test public void testPrefixOpWithActionAndLabel_1() throws Exception { diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/python3/TestParseTrees.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/python3/TestParseTrees.java index cb4d137e6..5d1b0da2d 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/python3/TestParseTrees.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/python3/TestParseTrees.java @@ -1,9 +1,10 @@ /* This file is generated by TestGenerator, any edits will be overwritten by the next generation. */ package org.antlr.v4.test.runtime.python3; -import org.junit.Ignore; import org.junit.Test; -import static org.junit.Assert.*; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; @SuppressWarnings("unused") public class TestParseTrees extends BasePython3Test { @@ -64,6 +65,54 @@ public class TestParseTrees extends BasePython3Test { } + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ + @Test + public void testAltNum() throws Exception { + mkdir(tmpdir); + + StringBuilder grammarBuilder = new StringBuilder(584); + grammarBuilder.append("grammar T;\n"); + grammarBuilder.append("\n"); + grammarBuilder.append("options { contextSuperClass=MyRuleNode; }\n"); + grammarBuilder.append("\n"); + grammarBuilder.append("@parser::members {\n"); + grammarBuilder.append("class MyRuleNode(ParserRuleContext):\n"); + grammarBuilder.append(" def __init__(self, parent:ParserRuleContext = None, invokingStateNumber:int = None ):\n"); + grammarBuilder.append(" super(TParser.MyRuleNode, self).__init__(parent, invokingStateNumber)\n"); + grammarBuilder.append(" self.altNum = 0;\n"); + grammarBuilder.append(" def getAltNumber(self):\n"); + grammarBuilder.append(" return self.altNum\n"); + grammarBuilder.append(" def setAltNumber(self, altNum):\n"); + grammarBuilder.append(" self.altNum = altNum\n"); + grammarBuilder.append("}\n"); + grammarBuilder.append("\n"); + grammarBuilder.append("\n"); + grammarBuilder.append("s\n"); + grammarBuilder.append("@init {\n"); + grammarBuilder.append("self._buildParseTrees = True\n"); + grammarBuilder.append("}\n"); + grammarBuilder.append("@after {\n"); + grammarBuilder.append("print($r.ctx.toStringTree(recog=self))\n"); + grammarBuilder.append("}\n"); + grammarBuilder.append(" : r=a ;\n"); + grammarBuilder.append("\n"); + grammarBuilder.append("a : 'f'\n"); + grammarBuilder.append(" | 'g'\n"); + grammarBuilder.append(" | 'x' b 'z'\n"); + grammarBuilder.append(" ;\n"); + grammarBuilder.append("b : 'e' {} | 'y'\n"); + grammarBuilder.append(" ;"); + String grammar = grammarBuilder.toString(); + + + String input ="xyz"; + String found = execParser("T.g4", grammar, "TParser", "TLexer", "TListener", "TVisitor", "s", input, false); + + assertEquals("(a:3 x (b:2 y) z)\n", found); + assertNull(this.stderrDuringParse); + + } + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ @Test public void testExtraToken() throws Exception { diff --git a/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Parser.cs b/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Parser.cs index 8d13cf52f..9df0b4270 100644 --- a/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Parser.cs +++ b/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Parser.cs @@ -352,7 +352,7 @@ namespace Antlr4.Runtime /// for a newly constructed parser. /// /// - /// + /// /// /// if a complete parse tree will be constructed while /// parsing, otherwise @@ -378,14 +378,14 @@ namespace Antlr4.Runtime /// by default for a newly constructed parser. /// /// - /// + /// /// /// to trim the capacity of the /// /// list to its size after a rule is parsed. /// /// - /// + /// /// /// if the /// @@ -649,9 +649,9 @@ namespace Antlr4.Runtime } } - public override IIntStream InputStream + public override IIntStream InputStream { - get + get { return _input; } @@ -659,11 +659,11 @@ namespace Antlr4.Runtime public ITokenStream TokenStream { - get + get { return _input; } - set + set { this._input = null; Reset (); @@ -842,6 +842,7 @@ namespace Antlr4.Runtime public virtual void EnterOuterAlt(ParserRuleContext localctx, int altNum) { + localctx.setAltNumber(altNum); // if we have new localctx, make sure we replace existing ctx // that is previous child of parse tree if (_buildParseTrees && _ctx != localctx) @@ -1004,7 +1005,7 @@ namespace Antlr4.Runtime /// /// the symbol type to check /// - /// + /// /// /// if /// diff --git a/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Properties/AssemblyInfo.cs b/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Properties/AssemblyInfo.cs index 15edab194..af3305927 100644 --- a/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Properties/AssemblyInfo.cs +++ b/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Properties/AssemblyInfo.cs @@ -67,8 +67,8 @@ using System.Runtime.InteropServices; // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("4.5.1.0")] +[assembly: AssemblyVersion("4.5.3.0")] #if !COMPACT -[assembly: AssemblyFileVersion("4.5.1.0")] -[assembly: AssemblyInformationalVersion("4.5.1-dev")] +[assembly: AssemblyFileVersion("4.5.3.0")] +[assembly: AssemblyInformationalVersion("4.5.3.0")] #endif diff --git a/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/RuleContext.cs b/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/RuleContext.cs index 6cb5fe96d..54a968ffa 100644 --- a/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/RuleContext.cs +++ b/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/RuleContext.cs @@ -30,6 +30,7 @@ using System.Collections.Generic; using System.Text; using Antlr4.Runtime; +using Antlr4.Runtime.Atn; using Antlr4.Runtime.Misc; using Antlr4.Runtime.Sharpen; using Antlr4.Runtime.Tree; @@ -139,7 +140,7 @@ namespace Antlr4.Runtime { return _parent; } - set + set { _parent = value; } @@ -216,6 +217,23 @@ namespace Antlr4.Runtime } } + /* For rule associated with this parse tree internal node, return + * the outer alternative number used to match the input. Default + * implementation does not compute nor store this alt num. Create + * a subclass of ParserRuleContext with backing field and set + * option contextSuperClass. + * to set it. + */ + public virtual int getAltNumber() { return Atn.ATN.InvalidAltNumber; } + + /* Set the outer alternative number for this context node. Default + * implementation does nothing to avoid backing field overhead for + * trees that don't need it. Create + * a subclass of ParserRuleContext with backing field and set + * option contextSuperClass. + */ + public virtual void setAltNumber(int altNumber) { } + public virtual IParseTree GetChild(int i) { return null; diff --git a/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Tree/Trees.cs b/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Tree/Trees.cs index bb70f7329..c77866920 100644 --- a/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Tree/Trees.cs +++ b/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Tree/Trees.cs @@ -30,6 +30,7 @@ using System.Collections.Generic; using System.Text; using Antlr4.Runtime; +using Antlr4.Runtime.Atn; using Antlr4.Runtime.Misc; using Antlr4.Runtime.Sharpen; using Antlr4.Runtime.Tree; @@ -111,10 +112,14 @@ namespace Antlr4.Runtime.Tree { if (ruleNames != null) { - if (t is IRuleNode) + if (t is RuleContext) { - int ruleIndex = ((IRuleNode)t).RuleContext.RuleIndex; + int ruleIndex = ((RuleContext)t).RuleIndex; string ruleName = ruleNames[ruleIndex]; + int altNumber = ((RuleContext)t).getAltNumber(); + if ( altNumber!=Atn.ATN.InvalidAltNumber ) { + return ruleName+":"+altNumber; + } return ruleName; } else diff --git a/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Vocabulary.cs b/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Vocabulary.cs index 0cd01c589..0f7b8057e 100644 --- a/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Vocabulary.cs +++ b/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Vocabulary.cs @@ -67,6 +67,8 @@ namespace Antlr4.Runtime [NotNull] private readonly string[] displayNames; + private readonly int maxTokenType; + /// /// Constructs a new instance of /// @@ -126,6 +128,19 @@ namespace Antlr4.Runtime this.literalNames = literalNames != null ? literalNames : EmptyNames; this.symbolicNames = symbolicNames != null ? symbolicNames : EmptyNames; this.displayNames = displayNames != null ? displayNames : EmptyNames; + this.maxTokenType = + System.Math.Max(this.displayNames.Length, + System.Math.Max(this.literalNames.Length, this.symbolicNames.Length)) - 1; + + } + + /// + /// Returns the highest token type value. It can be used to iterate from + /// zero to that number, inclusively, thus querying all stored entries. + /// + public virtual int getMaxTokenType() + { + return maxTokenType; } [return: Nullable] diff --git a/runtime/Java/pom.xml b/runtime/Java/pom.xml index b9c026525..9824ffdd4 100644 --- a/runtime/Java/pom.xml +++ b/runtime/Java/pom.xml @@ -3,7 +3,7 @@ org.antlr antlr4-master - 4.5.2-SNAPSHOT + 4.5.4-SNAPSHOT ../../pom.xml antlr4-runtime diff --git a/runtime/Java/src/org/antlr/v4/runtime/Parser.java b/runtime/Java/src/org/antlr/v4/runtime/Parser.java index eb299ecd1..eda34a13b 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Parser.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Parser.java @@ -34,7 +34,6 @@ import org.antlr.v4.runtime.atn.ATNDeserializationOptions; import org.antlr.v4.runtime.atn.ATNDeserializer; import org.antlr.v4.runtime.atn.ATNSimulator; import org.antlr.v4.runtime.atn.ATNState; -import org.antlr.v4.runtime.atn.AmbiguityInfo; import org.antlr.v4.runtime.atn.ParseInfo; import org.antlr.v4.runtime.atn.ParserATNSimulator; import org.antlr.v4.runtime.atn.PredictionMode; @@ -51,7 +50,6 @@ import org.antlr.v4.runtime.tree.pattern.ParseTreePattern; import org.antlr.v4.runtime.tree.pattern.ParseTreePatternMatcher; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -649,6 +647,7 @@ public abstract class Parser extends Recognizer { } public void enterOuterAlt(ParserRuleContext localctx, int altNum) { + localctx.setAltNumber(altNum); // if we have new localctx, make sure we replace existing ctx // that is previous child of parse tree if ( _buildParseTrees && _ctx != localctx ) { diff --git a/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java b/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java index 6cde47e67..010396625 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java @@ -29,20 +29,17 @@ */ package org.antlr.v4.runtime; +import org.antlr.v4.runtime.atn.ATN; import org.antlr.v4.runtime.misc.Interval; import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.runtime.tree.ParseTreeVisitor; import org.antlr.v4.runtime.tree.RuleNode; import org.antlr.v4.runtime.tree.Trees; -import javax.print.PrintException; -import javax.swing.*; -import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.concurrent.Future; -/** /** A rule context is a record of a single rule invocation. +/** A rule context is a record of a single rule invocation. * * We form a stack of these context objects using the parent * pointer. A parent pointer of null indicates that the current @@ -169,6 +166,27 @@ public class RuleContext implements RuleNode { public int getRuleIndex() { return -1; } + /** For rule associated with this parse tree internal node, return + * the outer alternative number used to match the input. Default + * implementation does not compute nor store this alt num. Create + * a subclass of ParserRuleContext with backing field and set + * option contextSuperClass. + * to set it. + * + * @since 4.5.3 + */ + public int getAltNumber() { return ATN.INVALID_ALT_NUMBER; } + + /** Set the outer alternative number for this context node. Default + * implementation does nothing to avoid backing field overhead for + * trees that don't need it. Create + * a subclass of ParserRuleContext with backing field and set + * option contextSuperClass. + * + * @since 4.5.3 + */ + public void setAltNumber(int altNumber) { } + @Override public ParseTree getChild(int i) { return null; diff --git a/runtime/Java/src/org/antlr/v4/runtime/RuleContextWithAltNum.java b/runtime/Java/src/org/antlr/v4/runtime/RuleContextWithAltNum.java new file mode 100644 index 000000000..63d3b22ce --- /dev/null +++ b/runtime/Java/src/org/antlr/v4/runtime/RuleContextWithAltNum.java @@ -0,0 +1,24 @@ +package org.antlr.v4.runtime; + +import org.antlr.v4.runtime.atn.ATN; + +/** A handy class for use with + * + * options {contextSuperClass=org.antlr.v4.runtime.RuleContextWithAltNum;} + * + * that provides a backing field / impl for the outer alternative number + * matched for an internal parse tree node. + * + * I'm only putting into Java runtime as I'm certain I'm the only one that + * will really every use this. + */ +public class RuleContextWithAltNum extends ParserRuleContext { + public int altNum; + public RuleContextWithAltNum() { altNum = ATN.INVALID_ALT_NUMBER; } + + public RuleContextWithAltNum(ParserRuleContext parent, int invokingStateNumber) { + super(parent, invokingStateNumber); + } + @Override public int getAltNumber() { return altNum; } + @Override public void setAltNumber(int altNum) { this.altNum = altNum; } +} diff --git a/runtime/Java/src/org/antlr/v4/runtime/RuntimeMetaData.java b/runtime/Java/src/org/antlr/v4/runtime/RuntimeMetaData.java index bf67cf029..b8bf0d589 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/RuntimeMetaData.java +++ b/runtime/Java/src/org/antlr/v4/runtime/RuntimeMetaData.java @@ -91,7 +91,7 @@ public class RuntimeMetaData { * omitted. * */ - public static final String VERSION = "4.5.1"; + public static final String VERSION = "4.5.3"; /** * Gets the currently executing version of the ANTLR 4 runtime library. diff --git a/runtime/Java/src/org/antlr/v4/runtime/Vocabulary.java b/runtime/Java/src/org/antlr/v4/runtime/Vocabulary.java index 0b6386c8e..2df537718 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Vocabulary.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Vocabulary.java @@ -37,6 +37,13 @@ package org.antlr.v4.runtime; * @author Sam Harwell */ public interface Vocabulary { + /** + * Returns the highest token type value. It can be used to iterate from + * zero to that number, inclusively, thus querying all stored entries. + * @return the highest token type value + */ + int getMaxTokenType(); + /** * Gets the string literal associated with a token type. The string returned * by this method, when not {@code null}, can be used unaltered in a parser @@ -85,7 +92,7 @@ public interface Vocabulary { * *
    *
  • Tokens created by lexer rules.
    • - *
  • Tokens defined in a {@code tokens{}} block in a lexer or parser + *
  • Tokens defined in a tokens{} block in a lexer or parser * grammar.
    • *
  • The implicitly defined {@code EOF} token, which has the token type * {@link Token#EOF}.
    • diff --git a/runtime/Java/src/org/antlr/v4/runtime/VocabularyImpl.java b/runtime/Java/src/org/antlr/v4/runtime/VocabularyImpl.java index ff5beefe2..93566bdcb 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/VocabularyImpl.java +++ b/runtime/Java/src/org/antlr/v4/runtime/VocabularyImpl.java @@ -57,6 +57,8 @@ public class VocabularyImpl implements Vocabulary { private final String[] displayNames; + private final int maxTokenType; + /** * Constructs a new instance of {@link VocabularyImpl} from the specified * literal and symbolic token names. @@ -94,6 +96,10 @@ public class VocabularyImpl implements Vocabulary { this.literalNames = literalNames != null ? literalNames : EMPTY_NAMES; this.symbolicNames = symbolicNames != null ? symbolicNames : EMPTY_NAMES; this.displayNames = displayNames != null ? displayNames : EMPTY_NAMES; + // See note here on -1 part: https://github.com/antlr/antlr4/pull/1146 + this.maxTokenType = + Math.max(this.displayNames.length, + Math.max(this.literalNames.length, this.symbolicNames.length)) - 1; } /** @@ -143,6 +149,11 @@ public class VocabularyImpl implements Vocabulary { return new VocabularyImpl(literalNames, symbolicNames, tokenNames); } + @Override + public int getMaxTokenType() { + return maxTokenType; + } + @Override public String getLiteralName(int tokenType) { if (tokenType >= 0 && tokenType < literalNames.length) { diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSerializer.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSerializer.java index 4c31ac92f..9183eef61 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSerializer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSerializer.java @@ -424,6 +424,12 @@ public class ATNSerializer { .append(ATNState.serializationNames.get(stype)).append(" ") .append(ruleIndex).append(arg).append("\n"); } + // this code is meant to model the form of ATNDeserializer.deserialize, + // since both need to be updated together whenever a change is made to + // the serialization format. The "dead" code is only used in debugging + // and testing scenarios, so the form you see here was kept for + // improved maintainability. + // start int numNonGreedyStates = ATNDeserializer.toInt(data[p++]); for (int i = 0; i < numNonGreedyStates; i++) { int stateNumber = ATNDeserializer.toInt(data[p++]); @@ -432,6 +438,7 @@ public class ATNSerializer { for (int i = 0; i < numPrecedenceStates; i++) { int stateNumber = ATNDeserializer.toInt(data[p++]); } + // finish int nrules = ATNDeserializer.toInt(data[p++]); for (int i=0; i(); diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java b/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java index 011e8f26f..991ee2b35 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java @@ -30,9 +30,6 @@ package org.antlr.v4.runtime.misc; -import java.awt.*; -import java.awt.event.WindowAdapter; -import java.awt.event.WindowEvent; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -151,38 +148,6 @@ public class Utils { return data; } - public static void waitForClose(final Window window) throws InterruptedException { - final Object lock = new Object(); - - Thread t = new Thread() { - @Override - public void run() { - synchronized (lock) { - while (window.isVisible()) { - try { - lock.wait(500); - } catch (InterruptedException e) { - } - } - } - } - }; - - t.start(); - - window.addWindowListener(new WindowAdapter() { - @Override - public void windowClosing(WindowEvent arg0) { - synchronized (lock) { - window.setVisible(false); - lock.notify(); - } - } - }); - - t.join(); - } - /** Convert array of strings to string→index map. Useful for * converting rulenames to name→ruleindex map. */ diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java b/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java index c58ee3dbe..3cb86683b 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/Trees.java @@ -33,7 +33,9 @@ package org.antlr.v4.runtime.tree; import org.antlr.v4.runtime.CommonToken; import org.antlr.v4.runtime.Parser; import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.RuleContext; import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.atn.ATN; import org.antlr.v4.runtime.misc.Interval; import org.antlr.v4.runtime.misc.Predicate; import org.antlr.v4.runtime.misc.Utils; @@ -91,9 +93,13 @@ public class Trees { public static String getNodeText(Tree t, List ruleNames) { if ( ruleNames!=null ) { - if ( t instanceof RuleNode ) { - int ruleIndex = ((RuleNode)t).getRuleContext().getRuleIndex(); + if ( t instanceof RuleContext ) { + int ruleIndex = ((RuleContext)t).getRuleContext().getRuleIndex(); String ruleName = ruleNames.get(ruleIndex); + int altNumber = ((RuleContext) t).getAltNumber(); + if ( altNumber!=ATN.INVALID_ALT_NUMBER ) { + return ruleName+":"+altNumber; + } return ruleName; } else if ( t instanceof ErrorNode) { diff --git a/runtime/JavaScript/README.md b/runtime/JavaScript/README.md index f35326d6d..5205fd423 100644 --- a/runtime/JavaScript/README.md +++ b/runtime/JavaScript/README.md @@ -8,6 +8,6 @@ This runtime has been tested in Node.js, Safari, Firefox, Chrome and IE. See www.antlr.org for more information on ANTLR -See https://raw.githubusercontent.com/antlr/antlr4/master/doc/javascript-target.md for more information on using ANTLR in JavaScript +See https://github.com/antlr/antlr4/blob/master/doc/javascript-target.md for more information on using ANTLR in JavaScript diff --git a/runtime/JavaScript/src/antlr4/BufferedTokenStream.js b/runtime/JavaScript/src/antlr4/BufferedTokenStream.js index 43baa8c37..217d67221 100644 --- a/runtime/JavaScript/src/antlr4/BufferedTokenStream.js +++ b/runtime/JavaScript/src/antlr4/BufferedTokenStream.js @@ -321,7 +321,7 @@ BufferedTokenStream.prototype.getHiddenTokensToRight = function(tokenIndex, channel = -1; } this.lazyInit(); - if (this.tokenIndex < 0 || tokenIndex >= this.tokens.length) { + if (tokenIndex < 0 || tokenIndex >= this.tokens.length) { throw "" + tokenIndex + " not in 0.." + this.tokens.length - 1; } var nextOnChannel = this.nextTokenOnChannel(tokenIndex + 1, diff --git a/runtime/JavaScript/src/antlr4/LL1Analyzer.js b/runtime/JavaScript/src/antlr4/LL1Analyzer.js index 65de2c0fc..d0bd3074c 100644 --- a/runtime/JavaScript/src/antlr4/LL1Analyzer.js +++ b/runtime/JavaScript/src/antlr4/LL1Analyzer.js @@ -157,7 +157,7 @@ LL1Analyzer.prototype.LOOK = function(s, stopState, ctx) { // is {@code null}. /// LL1Analyzer.prototype._LOOK = function(s, stopState , ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF) { - var c = new ATNConfig({state:s, alt:0}, ctx); + var c = new ATNConfig({state:s, alt:0, context: ctx}, null); if (lookBusy.contains(c)) { return; } diff --git a/runtime/JavaScript/src/antlr4/Parser.js b/runtime/JavaScript/src/antlr4/Parser.js index d691129c8..466e09107 100644 --- a/runtime/JavaScript/src/antlr4/Parser.js +++ b/runtime/JavaScript/src/antlr4/Parser.js @@ -471,6 +471,7 @@ Parser.prototype.exitRule = function() { }; Parser.prototype.enterOuterAlt = function(localctx, altNum) { + localctx.setAltNumber(altNum); // if we have new localctx, make sure we replace existing ctx // that is previous child of parse tree if (this.buildParseTrees && this._ctx !== localctx) { diff --git a/runtime/JavaScript/src/antlr4/README.md b/runtime/JavaScript/src/antlr4/README.md index 9ff7e5cfc..5205fd423 100644 --- a/runtime/JavaScript/src/antlr4/README.md +++ b/runtime/JavaScript/src/antlr4/README.md @@ -8,6 +8,6 @@ This runtime has been tested in Node.js, Safari, Firefox, Chrome and IE. See www.antlr.org for more information on ANTLR -See https://theantlrguy.atlassian.net/wiki/display/ANTLR4/JavaScript+Target for more information on using ANTLR in JavaScript +See https://github.com/antlr/antlr4/blob/master/doc/javascript-target.md for more information on using ANTLR in JavaScript diff --git a/runtime/JavaScript/src/antlr4/Recognizer.js b/runtime/JavaScript/src/antlr4/Recognizer.js index 14ad65b13..e2e20c6f2 100644 --- a/runtime/JavaScript/src/antlr4/Recognizer.js +++ b/runtime/JavaScript/src/antlr4/Recognizer.js @@ -45,7 +45,7 @@ Recognizer.ruleIndexMapCache = {}; Recognizer.prototype.checkVersion = function(toolVersion) { - var runtimeVersion = "4.5.1"; + var runtimeVersion = "4.5.3"; if (runtimeVersion!==toolVersion) { console.log("ANTLR runtime and generated code versions disagree: "+runtimeVersion+"!="+toolVersion); } diff --git a/runtime/JavaScript/src/antlr4/RuleContext.js b/runtime/JavaScript/src/antlr4/RuleContext.js index 7299357b9..c3631bab6 100644 --- a/runtime/JavaScript/src/antlr4/RuleContext.js +++ b/runtime/JavaScript/src/antlr4/RuleContext.js @@ -51,6 +51,7 @@ var RuleNode = require('./tree/Tree').RuleNode; var INVALID_INTERVAL = require('./tree/Tree').INVALID_INTERVAL; +var INVALID_ALT_NUMBER = require('./atn/ATN').INVALID_ALT_NUMBER; function RuleContext(parent, invokingState) { RuleNode.call(this); @@ -113,6 +114,21 @@ RuleContext.prototype.getText = function() { } }; +// For rule associated with this parse tree internal node, return +// the outer alternative number used to match the input. Default +// implementation does not compute nor store this alt num. Create +// a subclass of ParserRuleContext with backing field and set +// option contextSuperClass. +// to set it. +RuleContext.prototype.getAltNumber = function() { return INVALID_ALT_NUMBER; } + +// Set the outer alternative number for this context node. Default +// implementation does nothing to avoid backing field overhead for +// trees that don't need it. Create +// a subclass of ParserRuleContext with backing field and set +// option contextSuperClass. +RuleContext.prototype.setAltNumber = function(altNumber) { } + RuleContext.prototype.getChild = function(i) { return null; }; diff --git a/runtime/JavaScript/src/antlr4/atn/ATNConfig.js b/runtime/JavaScript/src/antlr4/atn/ATNConfig.js index 0b13737ed..988729dc7 100644 --- a/runtime/JavaScript/src/antlr4/atn/ATNConfig.js +++ b/runtime/JavaScript/src/antlr4/atn/ATNConfig.js @@ -50,7 +50,7 @@ function checkParams(params, isCfg) { } else { var props = {}; props.state = params.state || null; - props.alt = params.alt || null; + props.alt = (params.alt === undefined) ? null : params.alt; props.context = params.context || null; props.semanticContext = params.semanticContext || null; if(isCfg) { diff --git a/runtime/JavaScript/src/antlr4/atn/LexerActionExecutor.js b/runtime/JavaScript/src/antlr4/atn/LexerActionExecutor.js index 17af9f016..5b9e7a7b1 100644 --- a/runtime/JavaScript/src/antlr4/atn/LexerActionExecutor.js +++ b/runtime/JavaScript/src/antlr4/atn/LexerActionExecutor.js @@ -42,7 +42,7 @@ function LexerActionExecutor(lexerActions) { this.lexerActions = lexerActions === null ? [] : lexerActions; // Caches the result of {@link //hashCode} since the hash code is an element // of the performance-critical {@link LexerATNConfig//hashCode} operation. - this.hashString = lexerActions.toString(); // "".join([str(la) for la in + this._hashString = lexerActions.toString(); // "".join([str(la) for la in // lexerActions])) return this; } @@ -172,7 +172,7 @@ LexerActionExecutor.prototype.execute = function(lexer, input, startIndex) { }; LexerActionExecutor.prototype.hashString = function() { - return this.hashString; + return this._hashString; }; LexerActionExecutor.prototype.equals = function(other) { @@ -180,9 +180,18 @@ LexerActionExecutor.prototype.equals = function(other) { return true; } else if (!(other instanceof LexerActionExecutor)) { return false; + } else if (this._hashString != other._hashString) { + return false; + } else if (this.lexerActions.length != other.lexerActions.length) { + return false; } else { - return this.hashString === other.hashString && - this.lexerActions === other.lexerActions; + var numActions = this.lexerActions.length + for (var idx = 0; idx < numActions; ++idx) { + if (!this.lexerActions[idx].equals(other.lexerActions[idx])) { + return false; + } + } + return true; } }; diff --git a/runtime/JavaScript/src/antlr4/atn/ParserATNSimulator.js b/runtime/JavaScript/src/antlr4/atn/ParserATNSimulator.js index f405f991a..f96fae9c1 100644 --- a/runtime/JavaScript/src/antlr4/atn/ParserATNSimulator.js +++ b/runtime/JavaScript/src/antlr4/atn/ParserATNSimulator.js @@ -1301,10 +1301,10 @@ ParserATNSimulator.prototype.closureCheckingStopState = function(config, configs } continue; } - returnState = this.atn.states[config.context.getReturnState(i)]; - newContext = config.context.getParent(i); // "pop" return state + var returnState = this.atn.states[config.context.getReturnState(i)]; + var newContext = config.context.getParent(i); // "pop" return state var parms = {state:returnState, alt:config.alt, context:newContext, semanticContext:config.semanticContext}; - c = new ATNConfig(parms, null); + var c = new ATNConfig(parms, null); // While we have context to pop back from, we may have // gotten that context AFTER having falling off a rule. // Make sure we track that we are now out of context. diff --git a/runtime/JavaScript/src/antlr4/atn/PredictionMode.js b/runtime/JavaScript/src/antlr4/atn/PredictionMode.js index 6d717ee9b..c04d502ba 100644 --- a/runtime/JavaScript/src/antlr4/atn/PredictionMode.js +++ b/runtime/JavaScript/src/antlr4/atn/PredictionMode.js @@ -38,6 +38,9 @@ var BitSet = require('./../Utils').BitSet; var AltDict = require('./../Utils').AltDict; var ATN = require('./ATN').ATN; var RuleStopState = require('./ATNState').RuleStopState; +var ATNConfigSet = require('./ATNConfigSet').ATNConfigSet; +var ATNConfig = require('./ATNConfig').ATNConfig; +var SemanticContext = require('./SemanticContext').SemanticContext; function PredictionMode() { return this; @@ -580,4 +583,4 @@ PredictionMode.getSingleViableAlt = function(altsets) { return result; }; -exports.PredictionMode = PredictionMode; \ No newline at end of file +exports.PredictionMode = PredictionMode; diff --git a/runtime/JavaScript/src/antlr4/dfa/DFAState.js b/runtime/JavaScript/src/antlr4/dfa/DFAState.js index 481a3f67e..a1d6c9945 100644 --- a/runtime/JavaScript/src/antlr4/dfa/DFAState.js +++ b/runtime/JavaScript/src/antlr4/dfa/DFAState.js @@ -30,6 +30,8 @@ /// var ATNConfigSet = require('./../atn/ATNConfigSet').ATNConfigSet; +var Utils = require('./../Utils'); +var Set = Utils.Set; // Map a predicate to a predicted alternative./// @@ -163,4 +165,4 @@ DFAState.prototype.hashString = function() { }; exports.DFAState = DFAState; -exports.PredPrediction = PredPrediction; \ No newline at end of file +exports.PredPrediction = PredPrediction; diff --git a/runtime/JavaScript/src/antlr4/package.json b/runtime/JavaScript/src/antlr4/package.json index f192da1c0..dfa05e29e 100644 --- a/runtime/JavaScript/src/antlr4/package.json +++ b/runtime/JavaScript/src/antlr4/package.json @@ -1,6 +1,6 @@ { "name": "antlr4", - "version": "4.5.1", + "version": "4.5.3", "description": "JavaScript runtime for ANTLR4", "main": "src/antlr4/index.js", "repository": "antlr/antlr4.git", diff --git a/runtime/JavaScript/src/antlr4/tree/Tree.js b/runtime/JavaScript/src/antlr4/tree/Tree.js index 9295d3034..03581c740 100644 --- a/runtime/JavaScript/src/antlr4/tree/Tree.js +++ b/runtime/JavaScript/src/antlr4/tree/Tree.js @@ -95,6 +95,13 @@ ParseTreeVisitor.prototype.visit = function(ctx) { } }; +ParseTreeVisitor.prototype.visitTerminal = function(node) { +}; + +ParseTreeVisitor.prototype.visitErrorNode = function(node) { +}; + + var visitAtom = function(visitor, ctx) { if (ctx.parser === undefined) { //is terminal return; diff --git a/runtime/JavaScript/src/antlr4/tree/Trees.js b/runtime/JavaScript/src/antlr4/tree/Trees.js index 38a039b2e..d20258e73 100644 --- a/runtime/JavaScript/src/antlr4/tree/Trees.js +++ b/runtime/JavaScript/src/antlr4/tree/Trees.js @@ -34,6 +34,8 @@ var RuleNode = require('./Tree').RuleNode; var ErrorNode = require('./Tree').ErrorNode; var TerminalNode = require('./Tree').TerminalNode; var ParserRuleContext = require('./../ParserRuleContext').ParserRuleContext; +var RuleContext = require('./../RuleContext').RuleContext; +var INVALID_ALT_NUMBER = require('./../atn/ATN').INVALID_ALT_NUMBER; /** A set of utility routines useful for all kinds of ANTLR trees. */ @@ -75,8 +77,12 @@ Trees.getNodeText = function(t, ruleNames, recog) { ruleNames = recog.ruleNames; } if(ruleNames!==null) { - if (t instanceof RuleNode) { - return ruleNames[t.getRuleContext().ruleIndex]; + if (t instanceof RuleContext) { + var altNumber = t.getAltNumber(); + if ( altNumber!=INVALID_ALT_NUMBER ) { + return ruleNames[t.ruleIndex]+":"+altNumber; + } + return ruleNames[t.ruleIndex]; } else if ( t instanceof ErrorNode) { return t.toString(); } else if(t instanceof TerminalNode) { @@ -115,7 +121,7 @@ Trees.getAncestors = function(t) { } return ancestors; }; - + Trees.findAllTokenNodes = function(t, ttype) { return Trees.findAllNodes(t, ttype, true); }; diff --git a/runtime/JavaScript/src/antlr4/tree/index.js b/runtime/JavaScript/src/antlr4/tree/index.js index ce8844c57..e6b760ed0 100644 --- a/runtime/JavaScript/src/antlr4/tree/index.js +++ b/runtime/JavaScript/src/antlr4/tree/index.js @@ -1,6 +1,6 @@ var Tree = require('./Tree'); -exports.Trees = require('./Tree').Trees; +exports.Trees = require('./Trees').Trees; exports.RuleNode = Tree.RuleNode; exports.ParseTreeListener = Tree.ParseTreeListener; exports.ParseTreeVisitor = Tree.ParseTreeVisitor; -exports.ParseTreeWalker = Tree.ParseTreeWalker; \ No newline at end of file +exports.ParseTreeWalker = Tree.ParseTreeWalker; diff --git a/runtime/JavaScript/src/lib/require.js b/runtime/JavaScript/src/lib/require.js index 9a8069c35..d14210d24 100644 --- a/runtime/JavaScript/src/lib/require.js +++ b/runtime/JavaScript/src/lib/require.js @@ -62,7 +62,13 @@ // anchor element as parser in that case. Thes breaks web worker support, // but we don't care since these browsers also don't support web workers. - var parser = URL ? new URL(location.href) : document.createElement('A'); + try { + var parser = new URL(location.href); + } + catch (e) { + console.warn("Honey: falling back to DOM workaround for URL parser ("+e+")"); + parser = document.createElement('A'); + } // INFO Module cache // Contains getter functions for the exports objects of all the loaded @@ -81,7 +87,7 @@ delete cache.foo; } catch (e) { - console.warn("Honey: falling back to DOM workaround for defineProperty ("+e+")"); + console.warn("Honey: falling back to DOM workaround for cache object ("+e+")"); cache = document.createElement('DIV'); } diff --git a/runtime/Python2/setup.py b/runtime/Python2/setup.py index 0eeb03c5f..b4815753e 100644 --- a/runtime/Python2/setup.py +++ b/runtime/Python2/setup.py @@ -2,12 +2,12 @@ from distutils.core import setup setup( name='antlr4-python2-runtime', - version='4.5.2', + version='4.5.3', packages=['antlr4', 'antlr4.atn', 'antlr4.dfa', 'antlr4.tree', 'antlr4.error', 'antlr4.xpath'], package_dir={'': 'src'}, url='http://www.antlr.org', license='BSD', author='Eric Vergnaud, Terence Parr, Sam Harwell', author_email='eric.vergnaud@wanadoo.fr', - description='ANTLR 4.5.2 runtime for Python 2.7.6' + description='ANTLR 4.5.3 runtime for Python 2.7.6' ) diff --git a/runtime/Python2/src/antlr4/IntervalSet.py b/runtime/Python2/src/antlr4/IntervalSet.py index e28a556d4..3a55a2631 100644 --- a/runtime/Python2/src/antlr4/IntervalSet.py +++ b/runtime/Python2/src/antlr4/IntervalSet.py @@ -97,16 +97,10 @@ class IntervalSet(object): if self.intervals is None: return False else: - for i in self.intervals: - if item in i: - return True - return False + return any(item in i for i in self.intervals) def __len__(self): - xlen = 0 - for i in self.intervals: - xlen += len(i) - return xlen + return sum(len(i) for i in self.intervals) def removeRange(self, v): if v.start==v.stop-1: @@ -126,7 +120,7 @@ class IntervalSet(object): # check for included range, remove it elif v.start<=i.start and v.stop>=i.stop: self.intervals.pop(k) - k = k - 1 # need another pass + k -= 1 # need another pass # check for lower boundary elif v.start 0: @@ -382,6 +382,7 @@ class Parser (Recognizer): self._ctx = self._ctx.parentCtx def enterOuterAlt(self, localctx, altNum): + localctx.setAltNumber(altNum) # if we have new localctx, make sure we replace existing ctx # that is previous child of parse tree if self.buildParseTrees and self._ctx != localctx: diff --git a/runtime/Python2/src/antlr4/PredictionContext.py b/runtime/Python2/src/antlr4/PredictionContext.py index ba28eca7b..44d382818 100644 --- a/runtime/Python2/src/antlr4/PredictionContext.py +++ b/runtime/Python2/src/antlr4/PredictionContext.py @@ -30,6 +30,7 @@ #/ from io import StringIO from antlr4.RuleContext import RuleContext +from antlr4.atn.ATN import ATN from antlr4.atn.ATNState import ATNState @@ -98,7 +99,7 @@ def calculateHashCode(parent, returnState): def calculateListsHashCode(parents, returnStates ): h = 0 - for parent, returnState in parents, returnStates: + for parent, returnState in zip(parents, returnStates): h = hash((h, calculateHashCode(parent, returnState))) return h @@ -254,6 +255,10 @@ class ArrayPredictionContext(PredictionContext): buf.write(u"]") return buf.getvalue() + def __hash__(self): + return self.cachedHashCode + + # Convert a {@link RuleContext} tree to a {@link PredictionContext} graph. # Return {@link #EMPTY} if {@code outerContext} is empty or null. @@ -328,18 +333,18 @@ def merge(a, b, rootIsWildcard, mergeCache): #/ def mergeSingletons(a, b, rootIsWildcard, mergeCache): if mergeCache is not None: - previous = mergeCache.get(a,b) + previous = mergeCache.get((a,b), None) if previous is not None: return previous - previous = mergeCache.get(b,a) + previous = mergeCache.get((b,a), None) if previous is not None: return previous - rootMerge = mergeRoot(a, b, rootIsWildcard) - if rootMerge is not None: + merged = mergeRoot(a, b, rootIsWildcard) + if merged is not None: if mergeCache is not None: - mergeCache.put(a, b, rootMerge) - return rootMerge + mergeCache[(a, b)] = merged + return merged if a.returnState==b.returnState: parent = merge(a.parentCtx, b.parentCtx, rootIsWildcard, mergeCache) @@ -352,10 +357,10 @@ def mergeSingletons(a, b, rootIsWildcard, mergeCache): # merge parents x and y, giving array node with x,y then remainders # of those graphs. dup a, a' points at merged array # new joined parent so create new singleton pointing to it, a' - a_ = SingletonPredictionContext.create(parent, a.returnState) + merged = SingletonPredictionContext.create(parent, a.returnState) if mergeCache is not None: - mergeCache.put(a, b, a_) - return a_ + mergeCache[(a, b)] = merged + return merged else: # a != b payloads differ # see if we can collapse parents due to $+x parents if local ctx singleParent = None @@ -365,26 +370,24 @@ def mergeSingletons(a, b, rootIsWildcard, mergeCache): # sort payloads and use same parent payloads = [ a.returnState, b.returnState ] if a.returnState > b.returnState: - payloads[0] = b.returnState - payloads[1] = a.returnState + payloads = [ b.returnState, a.returnState ] parents = [singleParent, singleParent] - a_ = ArrayPredictionContext(parents, payloads) + merged = ArrayPredictionContext(parents, payloads) if mergeCache is not None: - mergeCache.put(a, b, a_) - return a_ + mergeCache[(a, b)] = merged + return merged # parents differ and can't merge them. Just pack together # into array; can't merge. # ax + by = [ax,by] payloads = [ a.returnState, b.returnState ] parents = [ a.parentCtx, b.parentCtx ] if a.returnState > b.returnState: # sort by payload - payloads[0] = b.returnState - payloads[1] = a.returnState + payloads = [ b.returnState, a.returnState ] parents = [ b.parentCtx, a.parentCtx ] - a_ = ArrayPredictionContext(parents, payloads) + merged = ArrayPredictionContext(parents, payloads) if mergeCache is not None: - mergeCache.put(a, b, a_) - return a_ + mergeCache[(a, b)] = merged + return merged # @@ -466,10 +469,10 @@ def mergeRoot(a, b, rootIsWildcard): #/ def mergeArrays(a, b, rootIsWildcard, mergeCache): if mergeCache is not None: - previous = mergeCache.get(a,b) + previous = mergeCache.get((a,b), None) if previous is not None: return previous - previous = mergeCache.get(b,a) + previous = mergeCache.get((b,a), None) if previous is not None: return previous @@ -478,8 +481,8 @@ def mergeArrays(a, b, rootIsWildcard, mergeCache): j = 0 # walks b k = 0 # walks target M array - mergedReturnStates = [] * (len(a.returnState) + len( b.returnStates)) - mergedParents = [] * len(mergedReturnStates) + mergedReturnStates = [None] * (len(a.returnStates) + len( b.returnStates)) + mergedParents = [None] * len(mergedReturnStates) # walk and merge to yield mergedParents, mergedReturnStates while itrg, src->trg, we keep track of the previous trg to # avoid looking up the DFA state again, which is expensive. @@ -223,8 +223,8 @@ class LexerATNSimulator(ATNSimulator): return None target = s.edges[t - self.MIN_DFA_EDGE] - if self.debug and target is not None: - print("reuse state "+s.stateNumber+ " edge to "+target.stateNumber) + if LexerATNSimulator.debug and target is not None: + print("reuse state", str(s.stateNumber), "edge to", str(target.stateNumber)) return target @@ -280,8 +280,8 @@ class LexerATNSimulator(ATNSimulator): if currentAltReachedAcceptState and cfg.passedThroughNonGreedyDecision: continue - if self.debug: - print("testing %s at %s\n", self.getTokenName(t), cfg.toString(self.recog, True)) + if LexerATNSimulator.debug: + print("testing", self.getTokenName(t), "at", str(cfg)) for trans in cfg.state.transitions: # for each transition target = self.getReachableTarget(trans, t) @@ -298,8 +298,8 @@ class LexerATNSimulator(ATNSimulator): skipAlt = cfg.alt def accept(self, input, lexerActionExecutor, startIndex, index, line, charPos): - if self.debug: - print("ACTION %s\n", lexerActionExecutor) + if LexerATNSimulator.debug: + print("ACTION", lexerActionExecutor) # seek to after last char in token input.seek(index) @@ -334,15 +334,15 @@ class LexerATNSimulator(ATNSimulator): # {@code false}. def closure(self, input, config, configs, currentAltReachedAcceptState, speculative, treatEofAsEpsilon): - if self.debug: - print("closure("+config.toString(self.recog, True)+")") + if LexerATNSimulator.debug: + print("closure(" + str(config) + ")") if isinstance( config.state, RuleStopState ): - if self.debug: + if LexerATNSimulator.debug: if self.recog is not None: - print("closure at %s rule stop %s\n", self.recog.getRuleNames()[config.state.ruleIndex], config) + print("closure at", self.recog.symbolicNames[config.state.ruleIndex], "rule stop", str(config)) else: - print("closure at rule stop %s\n", config) + print("closure at rule stop", str(config)) if config.context is None or config.context.hasEmptyPath(): if config.context is None or config.context.isEmpty(): @@ -404,7 +404,7 @@ class LexerATNSimulator(ATNSimulator): # states reached by traversing predicates. Since this is when we # test them, we cannot cash the DFA state target of ID. - if self.debug: + if LexerATNSimulator.debug: print("EVAL rule "+ str(t.ruleIndex) + ":" + str(t.predIndex)) configs.hasSemanticContext = True if self.evaluatePredicate(input, t.ruleIndex, t.predIndex, speculative): @@ -516,7 +516,7 @@ class LexerATNSimulator(ATNSimulator): # Only track edges within the DFA bounds return to - if self.debug: + if LexerATNSimulator.debug: print("EDGE " + str(from_) + " -> " + str(to) + " upon "+ chr(tk)) if from_.edges is None: @@ -535,11 +535,7 @@ class LexerATNSimulator(ATNSimulator): def addDFAState(self, configs): proposed = DFAState(configs=configs) - firstConfigWithRuleStopState = None - for c in configs: - if isinstance(c.state, RuleStopState): - firstConfigWithRuleStopState = c - break + firstConfigWithRuleStopState = next((cfg for cfg in configs if isinstance(cfg.state, RuleStopState)), None) if firstConfigWithRuleStopState is not None: proposed.isAcceptState = True diff --git a/runtime/Python2/src/antlr4/atn/ParserATNSimulator.py b/runtime/Python2/src/antlr4/atn/ParserATNSimulator.py index dea6b7e14..a979b40c2 100755 --- a/runtime/Python2/src/antlr4/atn/ParserATNSimulator.py +++ b/runtime/Python2/src/antlr4/atn/ParserATNSimulator.py @@ -308,7 +308,7 @@ class ParserATNSimulator(ATNSimulator): pass def adaptivePredict(self, input, decision, outerContext): - if self.debug or self.debug_list_atn_decisions: + if ParserATNSimulator.debug or ParserATNSimulator.debug_list_atn_decisions: print("adaptivePredict decision " + str(decision) + " exec LA(1)==" + self.getLookaheadName(input) + " line " + str(input.LT(1).line) + ":" + @@ -336,10 +336,10 @@ class ParserATNSimulator(ATNSimulator): if s0 is None: if outerContext is None: outerContext = ParserRuleContext.EMPTY - if self.debug or self.debug_list_atn_decisions: + if ParserATNSimulator.debug or ParserATNSimulator.debug_list_atn_decisions: print("predictATN decision " + str(dfa.decision) + " exec LA(1)==" + self.getLookaheadName(input) + - ", outerContext=" + outerContext.toString(self.parser)) + ", outerContext=" + outerContext.toString(self.parser.literalNames, None)) # If this is not a precedence DFA, we check the ATN start state # to determine if this ATN start state is the decision for the @@ -368,8 +368,8 @@ class ParserATNSimulator(ATNSimulator): dfa.s0 = s0 alt = self.execATN(dfa, s0, input, index, outerContext) - if self.debug: - print("DFA after predictATN: " + dfa.toString(self.parser.tokenNames)) + if ParserATNSimulator.debug: + print("DFA after predictATN: " + dfa.toString(self.parser.literalNames)) return alt finally: self._dfa = None @@ -408,14 +408,14 @@ class ParserATNSimulator(ATNSimulator): # conflict + preds # def execATN(self, dfa, s0, input, startIndex, outerContext ): - if self.debug or self.debug_list_atn_decisions: + if ParserATNSimulator.debug or ParserATNSimulator.debug_list_atn_decisions: print("execATN decision " + str(dfa.decision) + " exec LA(1)==" + self.getLookaheadName(input) + " line " + str(input.LT(1).line) + ":" + str(input.LT(1).column)) previousD = s0 - if self.debug: + if ParserATNSimulator.debug: print("s0 = " + str(s0)) t = input.LA(1) @@ -445,7 +445,7 @@ class ParserATNSimulator(ATNSimulator): # IF PREDS, MIGHT RESOLVE TO SINGLE ALT => SLL (or syntax error) conflictingAlts = None if D.predicates is not None: - if self.debug: + if ParserATNSimulator.debug: print("DFA state has preds in DFA sim LL failover") conflictIndex = input.index if conflictIndex != startIndex: @@ -453,7 +453,7 @@ class ParserATNSimulator(ATNSimulator): conflictingAlts = self.evalSemanticContext(D.predicates, outerContext, True) if len(conflictingAlts)==1: - if self.debug: + if ParserATNSimulator.debug: print("Full LL avoided") return min(conflictingAlts) @@ -462,7 +462,7 @@ class ParserATNSimulator(ATNSimulator): # context occurs with the index at the correct spot input.seek(conflictIndex) - if self.dfa_debug: + if ParserATNSimulator.dfa_debug: print("ctx sensitive state " + str(outerContext) +" in " + str(D)) fullCtx = True s0_closure = self.computeStartState(dfa.atnStartState, outerContext, fullCtx) @@ -534,7 +534,7 @@ class ParserATNSimulator(ATNSimulator): predictedAlt = self.getUniqueAlt(reach) - if self.debug: + if ParserATNSimulator.debug: altSubSets = PredictionMode.getConflictingAltSubsets(reach) print("SLL altSubSets=" + str(altSubSets) + ", configs=" + str(reach) + ", predict=" + str(predictedAlt) + ", allSubsetsConflict=" + @@ -586,8 +586,8 @@ class ParserATNSimulator(ATNSimulator): input, startIndex, outerContext): - if self.debug or self.debug_list_atn_decisions: - print("execATNWithFullContext "+s0) + if ParserATNSimulator.debug or ParserATNSimulator.debug_list_atn_decisions: + print("execATNWithFullContext", str(s0)) fullCtx = True foundExactAmbig = False reach = None @@ -616,7 +616,7 @@ class ParserATNSimulator(ATNSimulator): raise e altSubSets = PredictionMode.getConflictingAltSubsets(reach) - if self.debug: + if ParserATNSimulator.debug: print("LL altSubSets=" + str(altSubSets) + ", predict=" + str(PredictionMode.getUniqueAlt(altSubSets)) + ", resolvesToJustOneViableAlt=" + str(PredictionMode.resolvesToJustOneViableAlt(altSubSets))) @@ -685,7 +685,7 @@ class ParserATNSimulator(ATNSimulator): return predictedAlt def computeReachSet(self, closure, t, fullCtx): - if self.debug: + if ParserATNSimulator.debug: print("in computeReachSet, starting closure: " + str(closure)) if self.mergeCache is None: @@ -707,7 +707,7 @@ class ParserATNSimulator(ATNSimulator): # First figure out where we can reach on input t for c in closure: - if self.debug: + if ParserATNSimulator.debug: print("testing " + self.getTokenName(t) + " at " + str(c)) if isinstance(c.state, RuleStopState): @@ -967,7 +967,7 @@ class ParserATNSimulator(ATNSimulator): # nonambig alts are null in altToPred if nPredAlts==0: altToPred = None - if self.debug: + if ParserATNSimulator.debug: print("getPredsForAmbigAlts result " + str_list(altToPred)) return altToPred @@ -1093,11 +1093,11 @@ class ParserATNSimulator(ATNSimulator): break continue predicateEvaluationResult = pair.pred.eval(self.parser, outerContext) - if self.debug or self.dfa_debug: + if ParserATNSimulator.debug or ParserATNSimulator.dfa_debug: print("eval pred " + str(pair) + "=" + str(predicateEvaluationResult)) if predicateEvaluationResult: - if self.debug or self.dfa_debug: + if ParserATNSimulator.debug or ParserATNSimulator.dfa_debug: print("PREDICT " + str(pair.alt)) predictions.add(pair.alt) if not complete: @@ -1119,8 +1119,8 @@ class ParserATNSimulator(ATNSimulator): def closureCheckingStopState(self, config, configs, closureBusy, collectPredicates, fullCtx, depth, treatEofAsEpsilon): - if self.debug: - print("closure(" + config.toString(self.parser,True) + ")") + if ParserATNSimulator.debug: + print("closure(" + str(config) + ")") if isinstance(config.state, RuleStopState): # We hit rule end. If we have context info, use it @@ -1134,7 +1134,7 @@ class ParserATNSimulator(ATNSimulator): continue else: # we have no context info, just chase follow links (if greedy) - if self.debug: + if ParserATNSimulator.debug: print("FALLING off rule " + self.getRuleName(config.state.ruleIndex)) self.closure_(config, configs, closureBusy, collectPredicates, fullCtx, depth, treatEofAsEpsilon) @@ -1154,7 +1154,7 @@ class ParserATNSimulator(ATNSimulator): return else: # else if we have no context info, just chase follow links (if greedy) - if self.debug: + if ParserATNSimulator.debug: print("FALLING off rule " + self.getRuleName(config.state.ruleIndex)) self.closure_(config, configs, closureBusy, collectPredicates, fullCtx, depth, treatEofAsEpsilon) @@ -1196,7 +1196,7 @@ class ParserATNSimulator(ATNSimulator): c.reachesIntoOuterContext += 1 configs.dipsIntoOuterContext = True # TODO: can remove? only care when we add to set per middle of this method newDepth -= 1 - if self.debug: + if ParserATNSimulator.debug: print("dips into outer ctx: " + str(c)) elif isinstance(t, RuleTransition): # latch when newDepth goes negative - once we step out of the entry context we can't return @@ -1237,12 +1237,12 @@ class ParserATNSimulator(ATNSimulator): return m(self, config, t, collectPredicates, inContext, fullCtx, treatEofAsEpsilon) def actionTransition(self, config, t): - if self.debug: + if ParserATNSimulator.debug: print("ACTION edge " + str(t.ruleIndex) + ":" + str(t.actionIndex)) return ATNConfig(state=t.target, config=config) def precedenceTransition(self, config, pt, collectPredicates, inContext, fullCtx): - if self.debug: + if ParserATNSimulator.debug: print("PRED (collectPredicates=" + str(collectPredicates) + ") " + str(pt.precedence) + ">=_p, ctx dependent=true") if self.parser is not None: @@ -1267,12 +1267,12 @@ class ParserATNSimulator(ATNSimulator): else: c = ATNConfig(state=pt.target, config=config) - if self.debug: + if ParserATNSimulator.debug: print("config from pred transition=" + str(c)) return c def predTransition(self, config, pt, collectPredicates, inContext, fullCtx): - if self.debug: + if ParserATNSimulator.debug: print("PRED (collectPredicates=" + str(collectPredicates) + ") " + str(pt.ruleIndex) + ":" + str(pt.predIndex) + ", ctx dependent=" + str(pt.isCtxDependent)) if self.parser is not None: @@ -1297,12 +1297,12 @@ class ParserATNSimulator(ATNSimulator): else: c = ATNConfig(state=pt.target, config=config) - if self.debug: + if ParserATNSimulator.debug: print("config from pred transition=" + str(c)) return c def ruleTransition(self, config, t): - if self.debug: + if ParserATNSimulator.debug: print("CALL rule " + self.getRuleName(t.target.ruleIndex) + ", ctx=" + str(config.context)) returnState = t.followState newContext = SingletonPredictionContext.create(config.context, returnState.stateNumber) @@ -1360,13 +1360,12 @@ class ParserATNSimulator(ATNSimulator): def getTokenName(self, t): if t==Token.EOF: return u"EOF" - if self.parser is not None and self.parser.tokenNames is not None: - if t >= len(self.parser.tokenNames): - print(str(t) + " ttype out of range: " + str_list(self.parser.tokenNames)) - print(str_list(self.parser.getInputStream().getTokens())) - else: - return self.parser.tokensNames[t] + u"<" + unicode(t) + ">" - return unicode(t) + if self.parser is not None and \ + self.parser.literalNames is not None and \ + t < len(self.parser.literalNames): + return self.parser.literalNames[t] + u"<" + unicode(t) + ">" + else: + return unicode(t) def getLookaheadName(self, input): return self.getTokenName(input.LA(1)) @@ -1421,7 +1420,7 @@ class ParserATNSimulator(ATNSimulator): # on {@code to} # def addDFAEdge(self, dfa, from_, t, to): - if self.debug: + if ParserATNSimulator.debug: print("EDGE " + str(from_) + " -> " + str(to) + " upon " + self.getTokenName(t)) if to is None: @@ -1435,8 +1434,8 @@ class ParserATNSimulator(ATNSimulator): from_.edges = [None] * (self.atn.maxTokenType + 2) from_.edges[t+1] = to # connect - if self.debug: - names = None if self.parser is None else self.parser.tokenNames + if ParserATNSimulator.debug: + names = None if self.parser is None else self.parser.literalNames print("DFA=\n" + dfa.toString(names)) return to @@ -1470,12 +1469,12 @@ class ParserATNSimulator(ATNSimulator): D.configs.optimizeConfigs(self) D.configs.setReadonly(True) dfa.states[D] = D - if self.debug: + if ParserATNSimulator.debug: print("adding new DFA state: " + str(D)) return D def reportAttemptingFullContext(self, dfa, conflictingAlts, configs, startIndex, stopIndex): - if self.debug or self.retry_debug: + if ParserATNSimulator.debug or ParserATNSimulator.retry_debug: interval = range(startIndex, stopIndex + 1) print("reportAttemptingFullContext decision=" + str(dfa.decision) + ":" + str(configs) + ", input=" + self.parser.getTokenStream().getText(interval)) @@ -1483,7 +1482,7 @@ class ParserATNSimulator(ATNSimulator): self.parser.getErrorListenerDispatch().reportAttemptingFullContext(self.parser, dfa, startIndex, stopIndex, conflictingAlts, configs) def reportContextSensitivity(self, dfa, prediction, configs, startIndex, stopIndex): - if self.debug or self.retry_debug: + if ParserATNSimulator.debug or ParserATNSimulator.retry_debug: interval = range(startIndex, stopIndex + 1) print("reportContextSensitivity decision=" + str(dfa.decision) + ":" + str(configs) + ", input=" + self.parser.getTokenStream().getText(interval)) @@ -1493,7 +1492,7 @@ class ParserATNSimulator(ATNSimulator): # If context sensitive parsing, we know it's ambiguity not conflict# def reportAmbiguity(self, dfa, D, startIndex, stopIndex, exact, ambigAlts, configs ): - if self.debug or self.retry_debug: + if ParserATNSimulator.debug or ParserATNSimulator.retry_debug: # ParserATNPathFinder finder = new ParserATNPathFinder(parser, atn); # int i = 1; # for (Transition t : dfa.atnStartState.transitions) { diff --git a/runtime/Python2/src/antlr4/atn/PredictionMode.py b/runtime/Python2/src/antlr4/atn/PredictionMode.py index c01641972..0980ba773 100644 --- a/runtime/Python2/src/antlr4/atn/PredictionMode.py +++ b/runtime/Python2/src/antlr4/atn/PredictionMode.py @@ -232,10 +232,7 @@ class PredictionMode(object): # {@link RuleStopState}, otherwise {@code false} @classmethod def hasConfigInRuleStopState(cls, configs): - for c in configs: - if isinstance(c.state, RuleStopState): - return True - return False + return any(isinstance(cfg.state, RuleStopState) for cfg in configs) # Checks if all configurations in {@code configs} are in a # {@link RuleStopState}. Configurations meeting this condition have reached @@ -247,10 +244,7 @@ class PredictionMode(object): # {@link RuleStopState}, otherwise {@code false} @classmethod def allConfigsInRuleStopStates(cls, configs): - for config in configs: - if not isinstance(config.state, RuleStopState): - return False - return True + return all(isinstance(cfg.state, RuleStopState) for cfg in configs) # # Full LL prediction termination. @@ -419,10 +413,7 @@ class PredictionMode(object): # @classmethod def hasNonConflictingAltSet(cls, altsets): - for alts in altsets: - if len(alts)==1: - return True - return False + return any(len(alts) == 1 for alts in altsets) # # Determines if any single alternative subset in {@code altsets} contains @@ -434,10 +425,7 @@ class PredictionMode(object): # @classmethod def hasConflictingAltSet(cls, altsets): - for alts in altsets: - if len(alts)>1: - return True - return False + return any(len(alts) > 1 for alts in altsets) # # Determines if every alternative subset in {@code altsets} is equivalent. @@ -448,13 +436,9 @@ class PredictionMode(object): # @classmethod def allSubsetsEqual(cls, altsets): - first = None - for alts in altsets: - if first is None: - first = alts - elif not alts==first: - return False - return True + if not altsets: + return True + return all(alts == altsets[0] for alts in altsets[1:]) # # Returns the unique alternative predicted by all alternative subsets in @@ -467,9 +451,8 @@ class PredictionMode(object): def getUniqueAlt(cls, altsets): all = cls.getAlts(altsets) if len(all)==1: - return all[0] - else: - return ATN.INVALID_ALT_NUMBER + return all.pop() + return ATN.INVALID_ALT_NUMBER # Gets the complete set of represented alternatives for a collection of # alternative subsets. This method returns the union of each {@link BitSet} @@ -480,10 +463,7 @@ class PredictionMode(object): # @classmethod def getAlts(cls, altsets): - all = set() - for alts in altsets: - all = all | alts - return all + return set.union(*altsets) # # This function gets the conflicting alt subsets from a configuration set. @@ -527,11 +507,7 @@ class PredictionMode(object): @classmethod def hasStateAssociatedWithOneAlt(cls, configs): - x = cls.getStateToAltMap(configs) - for alts in x.values(): - if len(alts)==1: - return True - return False + return any(len(alts) == 1 for alts in cls.getStateToAltMap(configs).values()) @classmethod def getSingleViableAlt(cls, altsets): diff --git a/runtime/Python2/src/antlr4/atn/SemanticContext.py b/runtime/Python2/src/antlr4/atn/SemanticContext.py index 021d4770c..b8fe7b1a0 100644 --- a/runtime/Python2/src/antlr4/atn/SemanticContext.py +++ b/runtime/Python2/src/antlr4/atn/SemanticContext.py @@ -115,14 +115,7 @@ def orContext(a, b): return result def filterPrecedencePredicates(collection): - result = [] - for context in collection: - if isinstance(context, PrecedencePredicate): - if result is None: - result = [] - result.append(context) - return result - + return [context for context in collection if isinstance(context, PrecedencePredicate)] class Predicate(SemanticContext): @@ -187,13 +180,11 @@ class AND(SemanticContext): def __init__(self, a, b): operands = set() if isinstance( a, AND): - for o in a.opnds: - operands.add(o) + operands.update(a.opnds) else: operands.add(a) if isinstance( b, AND): - for o in b.opnds: - operands.add(o) + operands.update(b.opnds) else: operands.add(b) @@ -203,7 +194,7 @@ class AND(SemanticContext): reduced = min(precedencePredicates) operands.add(reduced) - self.opnds = [ o for o in operands ] + self.opnds = list(operands) def __eq__(self, other): if self is other: @@ -227,10 +218,7 @@ class AND(SemanticContext): # unordered.

    # def eval(self, parser, outerContext): - for opnd in self.opnds: - if not opnd.eval(parser, outerContext): - return False - return True + return all(opnd.eval(parser, outerContext) for opnd in self.opnds) def evalPrecedence(self, parser, outerContext): differs = False @@ -277,13 +265,11 @@ class OR (SemanticContext): def __init__(self, a, b): operands = set() if isinstance( a, OR): - for o in a.opnds: - operands.add(o) + operands.update(a.opnds) else: operands.add(a) if isinstance( b, OR): - for o in b.opnds: - operands.add(o) + operands.update(b.opnds) else: operands.add(b) @@ -291,10 +277,10 @@ class OR (SemanticContext): if len(precedencePredicates)>0: # interested in the transition with the highest precedence s = sorted(precedencePredicates) - reduced = s[len(s)-1] + reduced = s[-1] operands.add(reduced) - self.opnds = [ o for o in operands ] + self.opnds = list(operands) def __eq__(self, other): if self is other: @@ -315,10 +301,7 @@ class OR (SemanticContext): # unordered.

    # def eval(self, parser, outerContext): - for opnd in self.opnds: - if opnd.eval(parser, outerContext): - return True - return False + return any(opnd.eval(parser, outerContext) for opnd in self.opnds) def evalPrecedence(self, parser, outerContext): differs = False diff --git a/runtime/Python2/src/antlr4/dfa/DFAState.py b/runtime/Python2/src/antlr4/dfa/DFAState.py index 85ccac106..7045f7c22 100644 --- a/runtime/Python2/src/antlr4/dfa/DFAState.py +++ b/runtime/Python2/src/antlr4/dfa/DFAState.py @@ -105,14 +105,9 @@ class DFAState(object): # Get the set of all alts mentioned by all ATN configurations in this # DFA state. def getAltSet(self): - alts = set() if self.configs is not None: - for c in self.configs: - alts.add(c.alt) - if len(alts)==0: - return None - else: - return alts + return set(cfg.alt for cfg in self.configs) or None + return None def __hash__(self): return hash(self.configs) diff --git a/runtime/Python2/src/antlr4/tree/Trees.py b/runtime/Python2/src/antlr4/tree/Trees.py index 61722faa9..00a322b45 100644 --- a/runtime/Python2/src/antlr4/tree/Trees.py +++ b/runtime/Python2/src/antlr4/tree/Trees.py @@ -32,6 +32,8 @@ # A set of utility routines useful for all kinds of ANTLR trees.# from io import StringIO +import antlr4 +from antlr4.atn.ATN import ATN from antlr4.Token import Token from antlr4.Utils import escapeWhitespace from antlr4.tree.Tree import RuleNode, ErrorNode, TerminalNode @@ -65,7 +67,9 @@ class Trees(object): ruleNames = recog.ruleNames if ruleNames is not None: if isinstance(t, RuleNode): - return ruleNames[t.getRuleContext().getRuleIndex()] + if t.getAltNumber()!=ATN.INVALID_ALT_NUMBER: + return ruleNames[t.getRuleIndex()]+":"+str(t.getAltNumber()) + return ruleNames[t.getRuleIndex()] elif isinstance( t, ErrorNode): return unicode(t) elif isinstance(t, TerminalNode): @@ -125,8 +129,7 @@ class Trees(object): @classmethod def descendants(cls, t): - nodes = [] - nodes.append(t) + nodes = [t] for i in range(0, t.getChildCount()): nodes.extend(cls.descendants(t.getChild(i))) return nodes diff --git a/runtime/Python2/src/antlr4/xpath/XPath.py b/runtime/Python2/src/antlr4/xpath/XPath.py index eee87da6f..e138d3cdf 100644 --- a/runtime/Python2/src/antlr4/xpath/XPath.py +++ b/runtime/Python2/src/antlr4/xpath/XPath.py @@ -289,12 +289,7 @@ class XPathRuleElement(XPathElement): def evaluate(self, t): # return all children of t that match nodeName - nodes = [] - for c in Trees.getChildren(t): - if isinstance(c, ParserRuleContext ): - if (c.ruleIndex == self.ruleIndex ) == (not self.invert): - nodes.append(c) - return nodes + return [c for c in Trees.getChildren(t) if isinstance(c, ParserRuleContext) and (c.ruleIndex == self.ruleIndex) == (not self.invert)] class XPathTokenAnywhereElement(XPathElement): @@ -314,12 +309,8 @@ class XPathTokenElement(XPathElement): def evaluate(self, t): # return all children of t that match nodeName - nodes = [] - for c in Trees.getChildren(t): - if isinstance(c, TerminalNode): - if (c.symbol.type == self.tokenType ) == (not self.invert): - nodes.append(c) - return nodes + return [c for c in Trees.getChildren(t) if isinstance(c, TerminalNode) and (c.symbol.type == self.tokenType) == (not self.invert)] + class XPathWildcardAnywhereElement(XPathElement): @@ -343,4 +334,4 @@ class XPathWildcardElement(XPathElement): if self.invert: return list() # !* is weird but valid (empty) else: - return Trees.getChildren(t) \ No newline at end of file + return Trees.getChildren(t) diff --git a/runtime/Python3/setup.py b/runtime/Python3/setup.py index d9de10715..2de08f223 100644 --- a/runtime/Python3/setup.py +++ b/runtime/Python3/setup.py @@ -2,12 +2,12 @@ from distutils.core import setup setup( name='antlr4-python3-runtime', - version='4.5.2', + version='4.5.3', packages=['antlr4', 'antlr4.atn', 'antlr4.dfa', 'antlr4.tree', 'antlr4.error', 'antlr4.xpath'], package_dir={'': 'src'}, url='http://www.antlr.org', license='BSD', author='Eric Vergnaud, Terence Parr, Sam Harwell', author_email='eric.vergnaud@wanadoo.fr', - description='ANTLR 4.5.2 runtime for Python 3.4.0' + description='ANTLR 4.5.3 runtime for Python 3.4.0' ) diff --git a/runtime/Python3/src/antlr4/IntervalSet.py b/runtime/Python3/src/antlr4/IntervalSet.py index e5f535f45..980509821 100644 --- a/runtime/Python3/src/antlr4/IntervalSet.py +++ b/runtime/Python3/src/antlr4/IntervalSet.py @@ -84,16 +84,10 @@ class IntervalSet(object): if self.intervals is None: return False else: - for i in self.intervals: - if item in i: - return True - return False + return any(item in i for i in self.intervals) def __len__(self): - xlen = 0 - for i in self.intervals: - xlen += len(i) - return xlen + return sum(len(i) for i in self.intervals) def removeRange(self, v): if v.start==v.stop-1: @@ -113,7 +107,7 @@ class IntervalSet(object): # check for included range, remove it elif v.start<=i.start and v.stop>=i.stop: self.intervals.pop(k) - k = k - 1 # need another pass + k -= 1 # need another pass # check for lower boundary elif v.start 0: @@ -389,6 +389,7 @@ class Parser (Recognizer): self._ctx = self._ctx.parentCtx def enterOuterAlt(self, localctx:ParserRuleContext, altNum:int): + localctx.setAltNumber(altNum) # if we have new localctx, make sure we replace existing ctx # that is previous child of parse tree if self.buildParseTrees and self._ctx != localctx: diff --git a/runtime/Python3/src/antlr4/PredictionContext.py b/runtime/Python3/src/antlr4/PredictionContext.py index a6e643ea4..0cfc8896c 100644 --- a/runtime/Python3/src/antlr4/PredictionContext.py +++ b/runtime/Python3/src/antlr4/PredictionContext.py @@ -95,7 +95,7 @@ def calculateHashCode(parent:PredictionContext, returnState:int): def calculateListsHashCode(parents:[], returnStates:[] ): h = 0 - for parent, returnState in parents, returnStates: + for parent, returnState in zip(parents, returnStates): h = hash((h, calculateHashCode(parent, returnState))) return h @@ -242,7 +242,7 @@ class ArrayPredictionContext(PredictionContext): if self.returnStates[i]==PredictionContext.EMPTY_RETURN_STATE: buf.write("$") continue - buf.write(self.returnStates[i]) + buf.write(str(self.returnStates[i])) if self.parents[i] is not None: buf.write(' ') buf.write(str(self.parents[i])) @@ -251,6 +251,10 @@ class ArrayPredictionContext(PredictionContext): buf.write("]") return buf.getvalue() + def __hash__(self): + return self.cachedHashCode + + # Convert a {@link RuleContext} tree to a {@link PredictionContext} graph. # Return {@link #EMPTY} if {@code outerContext} is empty or null. @@ -325,18 +329,18 @@ def merge(a:PredictionContext, b:PredictionContext, rootIsWildcard:bool, mergeCa #/ def mergeSingletons(a:SingletonPredictionContext, b:SingletonPredictionContext, rootIsWildcard:bool, mergeCache:dict): if mergeCache is not None: - previous = mergeCache.get(a,b) + previous = mergeCache.get((a,b), None) if previous is not None: return previous - previous = mergeCache.get(b,a) + previous = mergeCache.get((b,a), None) if previous is not None: return previous - rootMerge = mergeRoot(a, b, rootIsWildcard) - if rootMerge is not None: + merged = mergeRoot(a, b, rootIsWildcard) + if merged is not None: if mergeCache is not None: - mergeCache.put(a, b, rootMerge) - return rootMerge + mergeCache[(a, b)] = merged + return merged if a.returnState==b.returnState: parent = merge(a.parentCtx, b.parentCtx, rootIsWildcard, mergeCache) @@ -349,10 +353,10 @@ def mergeSingletons(a:SingletonPredictionContext, b:SingletonPredictionContext, # merge parents x and y, giving array node with x,y then remainders # of those graphs. dup a, a' points at merged array # new joined parent so create new singleton pointing to it, a' - a_ = SingletonPredictionContext.create(parent, a.returnState) + merged = SingletonPredictionContext.create(parent, a.returnState) if mergeCache is not None: - mergeCache.put(a, b, a_) - return a_ + mergeCache[(a, b)] = merged + return merged else: # a != b payloads differ # see if we can collapse parents due to $+x parents if local ctx singleParent = None @@ -362,26 +366,24 @@ def mergeSingletons(a:SingletonPredictionContext, b:SingletonPredictionContext, # sort payloads and use same parent payloads = [ a.returnState, b.returnState ] if a.returnState > b.returnState: - payloads[0] = b.returnState - payloads[1] = a.returnState + payloads = [ b.returnState, a.returnState ] parents = [singleParent, singleParent] - a_ = ArrayPredictionContext(parents, payloads) + merged = ArrayPredictionContext(parents, payloads) if mergeCache is not None: - mergeCache.put(a, b, a_) - return a_ + mergeCache[(a, b)] = merged + return merged # parents differ and can't merge them. Just pack together # into array; can't merge. # ax + by = [ax,by] payloads = [ a.returnState, b.returnState ] parents = [ a.parentCtx, b.parentCtx ] if a.returnState > b.returnState: # sort by payload - payloads[0] = b.returnState - payloads[1] = a.returnState + payloads = [ b.returnState, a.returnState ] parents = [ b.parentCtx, a.parentCtx ] - a_ = ArrayPredictionContext(parents, payloads) + merged = ArrayPredictionContext(parents, payloads) if mergeCache is not None: - mergeCache.put(a, b, a_) - return a_ + mergeCache[(a, b)] = merged + return merged # @@ -463,10 +465,10 @@ def mergeRoot(a:SingletonPredictionContext, b:SingletonPredictionContext, rootIs #/ def mergeArrays(a:ArrayPredictionContext, b:ArrayPredictionContext, rootIsWildcard:bool, mergeCache:dict): if mergeCache is not None: - previous = mergeCache.get(a,b) + previous = mergeCache.get((a,b), None) if previous is not None: return previous - previous = mergeCache.get(b,a) + previous = mergeCache.get((b,a), None) if previous is not None: return previous @@ -475,8 +477,8 @@ def mergeArrays(a:ArrayPredictionContext, b:ArrayPredictionContext, rootIsWildca j = 0 # walks b k = 0 # walks target M array - mergedReturnStates = [] * (len(a.returnState) + len( b.returnStates)) - mergedParents = [] * len(mergedReturnStates) + mergedReturnStates = [None] * (len(a.returnStates) + len( b.returnStates)) + mergedParents = [None] * len(mergedReturnStates) # walk and merge to yield mergedParents, mergedReturnStates while itrg, src->trg, we keep track of the previous trg to # avoid looking up the DFA state again, which is expensive. @@ -228,8 +229,8 @@ class LexerATNSimulator(ATNSimulator): return None target = s.edges[t - self.MIN_DFA_EDGE] - if self.debug and target is not None: - print("reuse state "+s.stateNumber+ " edge to "+target.stateNumber) + if LexerATNSimulator.debug and target is not None: + print("reuse state", str(s.stateNumber), "edge to", str(target.stateNumber)) return target @@ -285,8 +286,8 @@ class LexerATNSimulator(ATNSimulator): if currentAltReachedAcceptState and cfg.passedThroughNonGreedyDecision: continue - if self.debug: - print("testing %s at %s\n", self.getTokenName(t), cfg.toString(self.recog, True)) + if LexerATNSimulator.debug: + print("testing", self.getTokenName(t), "at", str(cfg)) for trans in cfg.state.transitions: # for each transition target = self.getReachableTarget(trans, t) @@ -303,8 +304,8 @@ class LexerATNSimulator(ATNSimulator): skipAlt = cfg.alt def accept(self, input:InputStream, lexerActionExecutor:LexerActionExecutor, startIndex:int, index:int, line:int, charPos:int): - if self.debug: - print("ACTION %s\n", lexerActionExecutor) + if LexerATNSimulator.debug: + print("ACTION", lexerActionExecutor) # seek to after last char in token input.seek(index) @@ -339,15 +340,15 @@ class LexerATNSimulator(ATNSimulator): # {@code false}. def closure(self, input:InputStream, config:LexerATNConfig, configs:ATNConfigSet, currentAltReachedAcceptState:bool, speculative:bool, treatEofAsEpsilon:bool): - if self.debug: - print("closure("+config.toString(self.recog, True)+")") + if LexerATNSimulator.debug: + print("closure(" + str(config) + ")") if isinstance( config.state, RuleStopState ): - if self.debug: + if LexerATNSimulator.debug: if self.recog is not None: - print("closure at %s rule stop %s\n", self.recog.getRuleNames()[config.state.ruleIndex], config) + print("closure at", self.recog.symbolicNames[config.state.ruleIndex], "rule stop", str(config)) else: - print("closure at rule stop %s\n", config) + print("closure at rule stop", str(config)) if config.context is None or config.context.hasEmptyPath(): if config.context is None or config.context.isEmpty(): @@ -410,7 +411,7 @@ class LexerATNSimulator(ATNSimulator): # states reached by traversing predicates. Since this is when we # test them, we cannot cash the DFA state target of ID. - if self.debug: + if LexerATNSimulator.debug: print("EVAL rule "+ str(t.ruleIndex) + ":" + str(t.predIndex)) configs.hasSemanticContext = True if self.evaluatePredicate(input, t.ruleIndex, t.predIndex, speculative): @@ -522,7 +523,7 @@ class LexerATNSimulator(ATNSimulator): # Only track edges within the DFA bounds return to - if self.debug: + if LexerATNSimulator.debug: print("EDGE " + str(from_) + " -> " + str(to) + " upon "+ chr(tk)) if from_.edges is None: @@ -541,11 +542,7 @@ class LexerATNSimulator(ATNSimulator): def addDFAState(self, configs:ATNConfigSet) -> DFAState: proposed = DFAState(configs=configs) - firstConfigWithRuleStopState = None - for c in configs: - if isinstance(c.state, RuleStopState): - firstConfigWithRuleStopState = c - break + firstConfigWithRuleStopState = next((cfg for cfg in configs if isinstance(cfg.state, RuleStopState)), None) if firstConfigWithRuleStopState is not None: proposed.isAcceptState = True diff --git a/runtime/Python3/src/antlr4/atn/ParserATNSimulator.py b/runtime/Python3/src/antlr4/atn/ParserATNSimulator.py index fee18087f..7bcaa3912 100755 --- a/runtime/Python3/src/antlr4/atn/ParserATNSimulator.py +++ b/runtime/Python3/src/antlr4/atn/ParserATNSimulator.py @@ -313,7 +313,7 @@ class ParserATNSimulator(ATNSimulator): pass def adaptivePredict(self, input:TokenStream, decision:int, outerContext:ParserRuleContext): - if self.debug or self.debug_list_atn_decisions: + if ParserATNSimulator.debug or ParserATNSimulator.debug_list_atn_decisions: print("adaptivePredict decision " + str(decision) + " exec LA(1)==" + self.getLookaheadName(input) + " line " + str(input.LT(1).line) + ":" + @@ -341,10 +341,10 @@ class ParserATNSimulator(ATNSimulator): if s0 is None: if outerContext is None: outerContext = ParserRuleContext.EMPTY - if self.debug or self.debug_list_atn_decisions: + if ParserATNSimulator.debug or ParserATNSimulator.debug_list_atn_decisions: print("predictATN decision " + str(dfa.decision) + " exec LA(1)==" + self.getLookaheadName(input) + - ", outerContext=" + outerContext.toString(self.parser)) + ", outerContext=" + outerContext.toString(self.parser.literalNames, None)) # If this is not a precedence DFA, we check the ATN start state # to determine if this ATN start state is the decision for the @@ -373,8 +373,8 @@ class ParserATNSimulator(ATNSimulator): dfa.s0 = s0 alt = self.execATN(dfa, s0, input, index, outerContext) - if self.debug: - print("DFA after predictATN: " + dfa.toString(self.parser.tokenNames)) + if ParserATNSimulator.debug: + print("DFA after predictATN: " + dfa.toString(self.parser.literalNames)) return alt finally: self._dfa = None @@ -413,14 +413,14 @@ class ParserATNSimulator(ATNSimulator): # conflict + preds # def execATN(self, dfa:DFA, s0:DFAState, input:TokenStream, startIndex:int, outerContext:ParserRuleContext ): - if self.debug or self.debug_list_atn_decisions: + if ParserATNSimulator.debug or ParserATNSimulator.debug_list_atn_decisions: print("execATN decision " + str(dfa.decision) + " exec LA(1)==" + self.getLookaheadName(input) + " line " + str(input.LT(1).line) + ":" + str(input.LT(1).column)) previousD = s0 - if self.debug: + if ParserATNSimulator.debug: print("s0 = " + str(s0)) t = input.LA(1) @@ -450,7 +450,7 @@ class ParserATNSimulator(ATNSimulator): # IF PREDS, MIGHT RESOLVE TO SINGLE ALT => SLL (or syntax error) conflictingAlts = None if D.predicates is not None: - if self.debug: + if ParserATNSimulator.debug: print("DFA state has preds in DFA sim LL failover") conflictIndex = input.index if conflictIndex != startIndex: @@ -458,7 +458,7 @@ class ParserATNSimulator(ATNSimulator): conflictingAlts = self.evalSemanticContext(D.predicates, outerContext, True) if len(conflictingAlts)==1: - if self.debug: + if ParserATNSimulator.debug: print("Full LL avoided") return min(conflictingAlts) @@ -467,7 +467,7 @@ class ParserATNSimulator(ATNSimulator): # context occurs with the index at the correct spot input.seek(conflictIndex) - if self.dfa_debug: + if ParserATNSimulator.dfa_debug: print("ctx sensitive state " + str(outerContext) +" in " + str(D)) fullCtx = True s0_closure = self.computeStartState(dfa.atnStartState, outerContext, fullCtx) @@ -539,7 +539,7 @@ class ParserATNSimulator(ATNSimulator): predictedAlt = self.getUniqueAlt(reach) - if self.debug: + if ParserATNSimulator.debug: altSubSets = PredictionMode.getConflictingAltSubsets(reach) print("SLL altSubSets=" + str(altSubSets) + ", configs=" + str(reach) + ", predict=" + str(predictedAlt) + ", allSubsetsConflict=" + @@ -591,8 +591,8 @@ class ParserATNSimulator(ATNSimulator): input:TokenStream, startIndex:int, outerContext:ParserRuleContext): - if self.debug or self.debug_list_atn_decisions: - print("execATNWithFullContext "+s0) + if ParserATNSimulator.debug or ParserATNSimulator.debug_list_atn_decisions: + print("execATNWithFullContext", str(s0)) fullCtx = True foundExactAmbig = False reach = None @@ -621,7 +621,7 @@ class ParserATNSimulator(ATNSimulator): raise e altSubSets = PredictionMode.getConflictingAltSubsets(reach) - if self.debug: + if ParserATNSimulator.debug: print("LL altSubSets=" + str(altSubSets) + ", predict=" + str(PredictionMode.getUniqueAlt(altSubSets)) + ", resolvesToJustOneViableAlt=" + str(PredictionMode.resolvesToJustOneViableAlt(altSubSets))) @@ -690,7 +690,7 @@ class ParserATNSimulator(ATNSimulator): return predictedAlt def computeReachSet(self, closure:ATNConfigSet, t:int, fullCtx:bool): - if self.debug: + if ParserATNSimulator.debug: print("in computeReachSet, starting closure: " + str(closure)) if self.mergeCache is None: @@ -712,7 +712,7 @@ class ParserATNSimulator(ATNSimulator): # First figure out where we can reach on input t for c in closure: - if self.debug: + if ParserATNSimulator.debug: print("testing " + self.getTokenName(t) + " at " + str(c)) if isinstance(c.state, RuleStopState): @@ -972,7 +972,7 @@ class ParserATNSimulator(ATNSimulator): # nonambig alts are null in altToPred if nPredAlts==0: altToPred = None - if self.debug: + if ParserATNSimulator.debug: print("getPredsForAmbigAlts result " + str_list(altToPred)) return altToPred @@ -1098,11 +1098,11 @@ class ParserATNSimulator(ATNSimulator): break continue predicateEvaluationResult = pair.pred.eval(self.parser, outerContext) - if self.debug or self.dfa_debug: + if ParserATNSimulator.debug or ParserATNSimulator.dfa_debug: print("eval pred " + str(pair) + "=" + str(predicateEvaluationResult)) if predicateEvaluationResult: - if self.debug or self.dfa_debug: + if ParserATNSimulator.debug or ParserATNSimulator.dfa_debug: print("PREDICT " + str(pair.alt)) predictions.add(pair.alt) if not complete: @@ -1124,8 +1124,8 @@ class ParserATNSimulator(ATNSimulator): def closureCheckingStopState(self, config:ATNConfig, configs:ATNConfigSet, closureBusy:set, collectPredicates:bool, fullCtx:bool, depth:int, treatEofAsEpsilon:bool): - if self.debug: - print("closure(" + config.toString(self.parser,True) + ")") + if ParserATNSimulator.debug: + print("closure(" + str(config) + ")") if isinstance(config.state, RuleStopState): # We hit rule end. If we have context info, use it @@ -1139,7 +1139,7 @@ class ParserATNSimulator(ATNSimulator): continue else: # we have no context info, just chase follow links (if greedy) - if self.debug: + if ParserATNSimulator.debug: print("FALLING off rule " + self.getRuleName(config.state.ruleIndex)) self.closure_(config, configs, closureBusy, collectPredicates, fullCtx, depth, treatEofAsEpsilon) @@ -1159,7 +1159,7 @@ class ParserATNSimulator(ATNSimulator): return else: # else if we have no context info, just chase follow links (if greedy) - if self.debug: + if ParserATNSimulator.debug: print("FALLING off rule " + self.getRuleName(config.state.ruleIndex)) self.closure_(config, configs, closureBusy, collectPredicates, fullCtx, depth, treatEofAsEpsilon) @@ -1201,7 +1201,7 @@ class ParserATNSimulator(ATNSimulator): c.reachesIntoOuterContext += 1 configs.dipsIntoOuterContext = True # TODO: can remove? only care when we add to set per middle of this method newDepth -= 1 - if self.debug: + if ParserATNSimulator.debug: print("dips into outer ctx: " + str(c)) elif isinstance(t, RuleTransition): # latch when newDepth goes negative - once we step out of the entry context we can't return @@ -1242,12 +1242,12 @@ class ParserATNSimulator(ATNSimulator): return m(self, config, t, collectPredicates, inContext, fullCtx, treatEofAsEpsilon) def actionTransition(self, config:ATNConfig, t:ActionTransition): - if self.debug: + if ParserATNSimulator.debug: print("ACTION edge " + str(t.ruleIndex) + ":" + str(t.actionIndex)) return ATNConfig(state=t.target, config=config) def precedenceTransition(self, config:ATNConfig, pt:PrecedencePredicateTransition, collectPredicates:bool, inContext:bool, fullCtx:bool): - if self.debug: + if ParserATNSimulator.debug: print("PRED (collectPredicates=" + str(collectPredicates) + ") " + str(pt.precedence) + ">=_p, ctx dependent=true") if self.parser is not None: @@ -1272,12 +1272,12 @@ class ParserATNSimulator(ATNSimulator): else: c = ATNConfig(state=pt.target, config=config) - if self.debug: + if ParserATNSimulator.debug: print("config from pred transition=" + str(c)) return c def predTransition(self, config:ATNConfig, pt:PredicateTransition, collectPredicates:bool, inContext:bool, fullCtx:bool): - if self.debug: + if ParserATNSimulator.debug: print("PRED (collectPredicates=" + str(collectPredicates) + ") " + str(pt.ruleIndex) + ":" + str(pt.predIndex) + ", ctx dependent=" + str(pt.isCtxDependent)) if self.parser is not None: @@ -1302,12 +1302,12 @@ class ParserATNSimulator(ATNSimulator): else: c = ATNConfig(state=pt.target, config=config) - if self.debug: + if ParserATNSimulator.debug: print("config from pred transition=" + str(c)) return c def ruleTransition(self, config:ATNConfig, t:RuleTransition): - if self.debug: + if ParserATNSimulator.debug: print("CALL rule " + self.getRuleName(t.target.ruleIndex) + ", ctx=" + str(config.context)) returnState = t.followState newContext = SingletonPredictionContext.create(config.context, returnState.stateNumber) @@ -1365,13 +1365,12 @@ class ParserATNSimulator(ATNSimulator): def getTokenName(self, t:int): if t==Token.EOF: return "EOF" - if self.parser is not None and self.parser.tokenNames is not None: - if t >= len(self.parser.tokenNames): - print(str(t) + " ttype out of range: " + str_list(self.parser.tokenNames)) - print(str_list(self.parser.getInputStream().getTokens())) - else: - return self.parser.tokensNames[t] + "<" + str(t) + ">" - return str(t) + if self.parser is not None and \ + self.parser.literalNames is not None and \ + t < len(self.parser.literalNames): + return self.parser.literalNames[t] + "<" + str(t) + ">" + else: + return str(t) def getLookaheadName(self, input:TokenStream): return self.getTokenName(input.LA(1)) @@ -1426,7 +1425,7 @@ class ParserATNSimulator(ATNSimulator): # on {@code to} # def addDFAEdge(self, dfa:DFA, from_:DFAState, t:int, to:DFAState): - if self.debug: + if ParserATNSimulator.debug: print("EDGE " + str(from_) + " -> " + str(to) + " upon " + self.getTokenName(t)) if to is None: @@ -1440,8 +1439,8 @@ class ParserATNSimulator(ATNSimulator): from_.edges = [None] * (self.atn.maxTokenType + 2) from_.edges[t+1] = to # connect - if self.debug: - names = None if self.parser is None else self.parser.tokenNames + if ParserATNSimulator.debug: + names = None if self.parser is None else self.parser.literalNames print("DFA=\n" + dfa.toString(names)) return to @@ -1475,12 +1474,12 @@ class ParserATNSimulator(ATNSimulator): D.configs.optimizeConfigs(self) D.configs.setReadonly(True) dfa.states[D] = D - if self.debug: + if ParserATNSimulator.debug: print("adding new DFA state: " + str(D)) return D def reportAttemptingFullContext(self, dfa:DFA, conflictingAlts:set, configs:ATNConfigSet, startIndex:int, stopIndex:int): - if self.debug or self.retry_debug: + if ParserATNSimulator.debug or ParserATNSimulator.retry_debug: interval = range(startIndex, stopIndex + 1) print("reportAttemptingFullContext decision=" + str(dfa.decision) + ":" + str(configs) + ", input=" + self.parser.getTokenStream().getText(interval)) @@ -1488,7 +1487,7 @@ class ParserATNSimulator(ATNSimulator): self.parser.getErrorListenerDispatch().reportAttemptingFullContext(self.parser, dfa, startIndex, stopIndex, conflictingAlts, configs) def reportContextSensitivity(self, dfa:DFA, prediction:int, configs:ATNConfigSet, startIndex:int, stopIndex:int): - if self.debug or self.retry_debug: + if ParserATNSimulator.debug or ParserATNSimulator.retry_debug: interval = range(startIndex, stopIndex + 1) print("reportContextSensitivity decision=" + str(dfa.decision) + ":" + str(configs) + ", input=" + self.parser.getTokenStream().getText(interval)) @@ -1498,7 +1497,7 @@ class ParserATNSimulator(ATNSimulator): # If context sensitive parsing, we know it's ambiguity not conflict# def reportAmbiguity(self, dfa:DFA, D:DFAState, startIndex:int, stopIndex:int, exact:bool, ambigAlts:set, configs:ATNConfigSet ): - if self.debug or self.retry_debug: + if ParserATNSimulator.debug or ParserATNSimulator.retry_debug: # ParserATNPathFinder finder = new ParserATNPathFinder(parser, atn); # int i = 1; # for (Transition t : dfa.atnStartState.transitions) { diff --git a/runtime/Python3/src/antlr4/atn/PredictionMode.py b/runtime/Python3/src/antlr4/atn/PredictionMode.py index 0fba6351f..67039c728 100644 --- a/runtime/Python3/src/antlr4/atn/PredictionMode.py +++ b/runtime/Python3/src/antlr4/atn/PredictionMode.py @@ -235,10 +235,7 @@ class PredictionMode(Enum): # {@link RuleStopState}, otherwise {@code false} @classmethod def hasConfigInRuleStopState(cls, configs:ATNConfigSet): - for c in configs: - if isinstance(c.state, RuleStopState): - return True - return False + return any(isinstance(cfg.state, RuleStopState) for cfg in configs) # Checks if all configurations in {@code configs} are in a # {@link RuleStopState}. Configurations meeting this condition have reached @@ -250,10 +247,7 @@ class PredictionMode(Enum): # {@link RuleStopState}, otherwise {@code false} @classmethod def allConfigsInRuleStopStates(cls, configs:ATNConfigSet): - for config in configs: - if not isinstance(config.state, RuleStopState): - return False - return True + return all(isinstance(cfg.state, RuleStopState) for cfg in configs) # # Full LL prediction termination. @@ -422,10 +416,7 @@ class PredictionMode(Enum): # @classmethod def hasNonConflictingAltSet(cls, altsets:list): - for alts in altsets: - if len(alts)==1: - return True - return False + return any(len(alts) == 1 for alts in altsets) # # Determines if any single alternative subset in {@code altsets} contains @@ -437,10 +428,7 @@ class PredictionMode(Enum): # @classmethod def hasConflictingAltSet(cls, altsets:list): - for alts in altsets: - if len(alts)>1: - return True - return False + return any(len(alts) > 1 for alts in altsets) # # Determines if every alternative subset in {@code altsets} is equivalent. @@ -451,13 +439,10 @@ class PredictionMode(Enum): # @classmethod def allSubsetsEqual(cls, altsets:list): - first = None - for alts in altsets: - if first is None: - first = alts - elif not alts==first: - return False - return True + if not altsets: + return True + first = next(iter(altsets)) + return all(alts == first for alts in iter(altsets)) # # Returns the unique alternative predicted by all alternative subsets in @@ -470,9 +455,8 @@ class PredictionMode(Enum): def getUniqueAlt(cls, altsets:list): all = cls.getAlts(altsets) if len(all)==1: - return all[0] - else: - return ATN.INVALID_ALT_NUMBER + return next(iter(all)) + return ATN.INVALID_ALT_NUMBER # Gets the complete set of represented alternatives for a collection of # alternative subsets. This method returns the union of each {@link BitSet} @@ -483,10 +467,7 @@ class PredictionMode(Enum): # @classmethod def getAlts(cls, altsets:list): - all = set() - for alts in altsets: - all = all | alts - return all + return set.union(*altsets) # # This function gets the conflicting alt subsets from a configuration set. @@ -530,11 +511,7 @@ class PredictionMode(Enum): @classmethod def hasStateAssociatedWithOneAlt(cls, configs:ATNConfigSet): - x = cls.getStateToAltMap(configs) - for alts in x.values(): - if len(alts)==1: - return True - return False + return any(len(alts) == 1 for alts in cls.getStateToAltMap(configs).values()) @classmethod def getSingleViableAlt(cls, altsets:list): diff --git a/runtime/Python3/src/antlr4/atn/SemanticContext.py b/runtime/Python3/src/antlr4/atn/SemanticContext.py index c012e35c2..d69165e4d 100644 --- a/runtime/Python3/src/antlr4/atn/SemanticContext.py +++ b/runtime/Python3/src/antlr4/atn/SemanticContext.py @@ -116,13 +116,7 @@ def orContext(a:SemanticContext, b:SemanticContext): return result def filterPrecedencePredicates(collection:list): - result = [] - for context in collection: - if isinstance(context, PrecedencePredicate): - if result is None: - result = [] - result.append(context) - return result + return [context for context in collection if isinstance(context, PrecedencePredicate)] class Predicate(SemanticContext): @@ -188,13 +182,11 @@ class AND(SemanticContext): def __init__(self, a:SemanticContext, b:SemanticContext): operands = set() if isinstance( a, AND ): - for o in a.opnds: - operands.add(o) + operands.update(a.opnds) else: operands.add(a) if isinstance( b, AND ): - for o in b.opnds: - operands.add(o) + operands.update(b.opnds) else: operands.add(b) @@ -204,7 +196,7 @@ class AND(SemanticContext): reduced = min(precedencePredicates) operands.add(reduced) - self.opnds = [ o for o in operands ] + self.opnds = list(operands) def __eq__(self, other): if self is other: @@ -227,11 +219,8 @@ class AND(SemanticContext): # The evaluation of predicates by this context is short-circuiting, but # unordered.

    # - def eval(self, parser:Recognizer , outerContext:RuleContext ): - for opnd in self.opnds: - if not opnd.eval(parser, outerContext): - return False - return True + def eval(self, parser:Recognizer, outerContext:RuleContext): + return all(opnd.eval(parser, outerContext) for opnd in self.opnds) def evalPrecedence(self, parser:Recognizer, outerContext:RuleContext): differs = False @@ -278,13 +267,11 @@ class OR (SemanticContext): def __init__(self, a:SemanticContext, b:SemanticContext): operands = set() if isinstance( a, OR ): - for o in a.opnds: - operands.add(o) + operands.update(a.opnds) else: operands.add(a) if isinstance( b, OR ): - for o in b.opnds: - operands.add(o) + operands.update(b.opnds) else: operands.add(b) @@ -292,10 +279,10 @@ class OR (SemanticContext): if len(precedencePredicates)>0: # interested in the transition with the highest precedence s = sorted(precedencePredicates) - reduced = s[len(s)-1] + reduced = s[-1] operands.add(reduced) - self.opnds = [ o for o in operands ] + self.opnds = list(operands) def __eq__(self, other): if self is other: @@ -316,10 +303,7 @@ class OR (SemanticContext): # unordered.

    # def eval(self, parser:Recognizer, outerContext:RuleContext): - for opnd in self.opnds: - if opnd.eval(parser, outerContext): - return True - return False + return any(opnd.eval(parser, outerContext) for opnd in self.opnds) def evalPrecedence(self, parser:Recognizer, outerContext:RuleContext): differs = False diff --git a/runtime/Python3/src/antlr4/dfa/DFAState.py b/runtime/Python3/src/antlr4/dfa/DFAState.py index 1ab5a0e77..9ad996384 100644 --- a/runtime/Python3/src/antlr4/dfa/DFAState.py +++ b/runtime/Python3/src/antlr4/dfa/DFAState.py @@ -104,14 +104,9 @@ class DFAState(object): # Get the set of all alts mentioned by all ATN configurations in this # DFA state. def getAltSet(self): - alts = set() if self.configs is not None: - for c in self.configs: - alts.add(c.alt) - if len(alts)==0: - return None - else: - return alts + return set(cfg.alt for cfg in self.configs) or None + return None def __hash__(self): return hash(self.configs) diff --git a/runtime/Python3/src/antlr4/tree/Trees.py b/runtime/Python3/src/antlr4/tree/Trees.py index 234034586..42fde282b 100644 --- a/runtime/Python3/src/antlr4/tree/Trees.py +++ b/runtime/Python3/src/antlr4/tree/Trees.py @@ -68,7 +68,9 @@ class Trees(object): ruleNames = recog.ruleNames if ruleNames is not None: if isinstance(t, RuleNode): - return ruleNames[t.getRuleContext().getRuleIndex()] + if t.getAltNumber()!=0: # should use ATN.INVALID_ALT_NUMBER but won't compile + return ruleNames[t.getRuleIndex()]+":"+str(t.getAltNumber()) + return ruleNames[t.getRuleIndex()] elif isinstance( t, ErrorNode): return str(t) elif isinstance(t, TerminalNode): @@ -128,8 +130,7 @@ class Trees(object): @classmethod def descendants(cls, t:ParseTree): - nodes = [] - nodes.append(t) + nodes = [t] for i in range(0, t.getChildCount()): nodes.extend(cls.descendants(t.getChild(i))) return nodes diff --git a/runtime/Python3/src/antlr4/xpath/XPath.py b/runtime/Python3/src/antlr4/xpath/XPath.py index 6b41a943a..6d3d825a5 100644 --- a/runtime/Python3/src/antlr4/xpath/XPath.py +++ b/runtime/Python3/src/antlr4/xpath/XPath.py @@ -290,12 +290,8 @@ class XPathRuleElement(XPathElement): def evaluate(self, t:ParseTree): # return all children of t that match nodeName - nodes = [] - for c in Trees.getChildren(t): - if isinstance(c, ParserRuleContext ): - if (c.ruleIndex == self.ruleIndex ) == (not self.invert): - nodes.append(c) - return nodes + return [c for c in Trees.getChildren(t) if isinstance(c, ParserRuleContext) and (c.ruleIndex == self.ruleIndex) == (not self.invert)] + class XPathTokenAnywhereElement(XPathElement): @@ -315,12 +311,8 @@ class XPathTokenElement(XPathElement): def evaluate(self, t:ParseTree): # return all children of t that match nodeName - nodes = [] - for c in Trees.getChildren(t): - if isinstance(c, TerminalNode): - if (c.symbol.type == self.tokenType ) == (not self.invert): - nodes.append(c) - return nodes + return [c for c in Trees.getChildren(t) if isinstance(c, TerminalNode) and (c.symbol.type == self.tokenType) == (not self.invert)] + class XPathWildcardAnywhereElement(XPathElement): diff --git a/tool-testsuite/pom.xml b/tool-testsuite/pom.xml index dad48a51c..c7b1564d4 100644 --- a/tool-testsuite/pom.xml +++ b/tool-testsuite/pom.xml @@ -4,7 +4,7 @@ org.antlr antlr4-master - 4.5.2-SNAPSHOT + 4.5.4-SNAPSHOT antlr4-tool-testsuite ANTLR 4 Tool Tests diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/InterpreterTreeTextProvider.java b/tool-testsuite/test/org/antlr/v4/test/tool/InterpreterTreeTextProvider.java index 1a555c4ed..bff84ae17 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/InterpreterTreeTextProvider.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/InterpreterTreeTextProvider.java @@ -1,10 +1,9 @@ package org.antlr.v4.test.tool; +import org.antlr.v4.gui.TreeTextProvider; import org.antlr.v4.runtime.tree.ErrorNode; import org.antlr.v4.runtime.tree.Tree; import org.antlr.v4.runtime.tree.Trees; -import org.antlr.v4.gui.TreeTextProvider; -import org.antlr.v4.tool.GrammarInterpreterRuleContext; import java.util.Arrays; import java.util.List; @@ -17,10 +16,6 @@ public class InterpreterTreeTextProvider implements TreeTextProvider { public String getText(Tree node) { if ( node==null ) return "null"; String nodeText = Trees.getNodeText(node, ruleNames); - if ( node instanceof GrammarInterpreterRuleContext) { - GrammarInterpreterRuleContext ctx = (GrammarInterpreterRuleContext) node; - return nodeText+":"+ctx.getOuterAltNum(); - } if ( node instanceof ErrorNode) { return ""; } diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestActionTranslation.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestActionTranslation.java index e3c44bb1f..dfe170418 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestActionTranslation.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestActionTranslation.java @@ -31,6 +31,7 @@ package org.antlr.v4.test.tool; import org.antlr.v4.test.runtime.java.BaseTest; +import org.antlr.v4.tool.Grammar; import org.junit.Test; /** */ @@ -180,6 +181,15 @@ public class TestActionTranslation extends BaseTest { testActions(attributeTemplate, "finally", action, expected); } + @Test public void testEmptyActions() throws Exception { + String gS = + "grammar A;\n"+ + "a[] : 'a' ;\n" + + "c : a[] c[] ;\n"; + Grammar g = new Grammar(gS); + } + + @Test public void testDynamicRuleScopeRefInSubrule() throws Exception { String action = "$a::n;"; } diff --git a/tool/pom.xml b/tool/pom.xml index 073f0cbd1..42e5f825f 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -3,7 +3,7 @@ org.antlr antlr4-master - 4.5.2-SNAPSHOT + 4.5.4-SNAPSHOT antlr4 ANTLR 4 Tool diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/CSharp/CSharp.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/CSharp/CSharp.stg index 0617385b1..6d5710922 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/CSharp/CSharp.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/CSharp/CSharp.stg @@ -30,7 +30,7 @@ // args must be , -ParserFile(file, parser, namedActions) ::= << +ParserFile(file, parser, namedActions, contextSuperClass) ::= << namespace { @@ -839,7 +839,7 @@ CaptureNextTokenType(d) ::= " = TokenStream.La(1);" StructDecl(struct,ctorAttrs,attrs,getters,dispatchMethods,interfaces,extensionMembers, superClass={ParserRuleContext}) ::= << -public partial class : , { +public partial class : ParserRuleContext, { ;}; separator="\n"> }; separator="\n"> public (ParserRuleContext parent, int invokingState) : base(parent, invokingState) { } diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg index 8bd429a0b..af925be25 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg @@ -42,7 +42,7 @@ javaTypeInitMap ::= [ // args must be , -ParserFile(file, parser, namedActions) ::= << +ParserFile(file, parser, namedActions, contextSuperClass) ::= << package ; @@ -765,9 +765,9 @@ ListLabelName(label) ::= "