From 3c08f49a1a62457e2c5ea2333725217358ffeac8 Mon Sep 17 00:00:00 2001 From: Robbie Cooper Date: Wed, 5 Aug 2015 02:18:37 -0400 Subject: [PATCH 01/95] Fix NameError in LexerATNSimulator.reset() Without this line, one gets "NameError: name 'Lexer' is not defined" when calling reset(). This is the same way __init__() assigns self.mode. --- runtime/Python3/src/antlr4/atn/LexerATNSimulator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/runtime/Python3/src/antlr4/atn/LexerATNSimulator.py b/runtime/Python3/src/antlr4/atn/LexerATNSimulator.py index 150743b66..d768cf156 100644 --- a/runtime/Python3/src/antlr4/atn/LexerATNSimulator.py +++ b/runtime/Python3/src/antlr4/atn/LexerATNSimulator.py @@ -130,6 +130,7 @@ class LexerATNSimulator(ATNSimulator): self.startIndex = -1 self.line = 1 self.column = 0 + from antlr4.Lexer import Lexer self.mode = Lexer.DEFAULT_MODE def matchATN(self, input:InputStream): From 3abca793af54d76a516429274887237a4231d2d7 Mon Sep 17 00:00:00 2001 From: Robbie Cooper Date: Thu, 12 Nov 2015 17:18:17 -0500 Subject: [PATCH 02/95] Add cooperra to contributors.txt --- contributors.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contributors.txt b/contributors.txt index 75e3b8b92..45ef1bb9c 100644 --- a/contributors.txt +++ b/contributors.txt @@ -74,4 +74,5 @@ YYYY/MM/DD, github id, Full name, email 2015/05/12, Pursuit92, Josh Chase, jcjoshuachase@gmail.com 2015/05/20, peturingi, Pétur Ingi Egilsson, petur@petur.eu 2015/05/27, jcbrinfo, Jean-Christophe Beaupré, jcbrinfo@users.noreply.github.com -2015/06/29, jvanzyl, Jason van Zyl, jason@takari.io \ No newline at end of file +2015/06/29, jvanzyl, Jason van Zyl, jason@takari.io +2015/11/12, cooperra, Robbie Cooper, cooperra@users.noreply.github.com From c651c83a8b98131a585d3062a7877cd589376e10 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Wed, 9 Dec 2015 22:17:46 +0800 Subject: [PATCH 03/95] fix #1058 --- runtime/JavaScript/src/antlr4/tree/Tree.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/runtime/JavaScript/src/antlr4/tree/Tree.js b/runtime/JavaScript/src/antlr4/tree/Tree.js index 9295d3034..03581c740 100644 --- a/runtime/JavaScript/src/antlr4/tree/Tree.js +++ b/runtime/JavaScript/src/antlr4/tree/Tree.js @@ -95,6 +95,13 @@ ParseTreeVisitor.prototype.visit = function(ctx) { } }; +ParseTreeVisitor.prototype.visitTerminal = function(node) { +}; + +ParseTreeVisitor.prototype.visitErrorNode = function(node) { +}; + + var visitAtom = function(visitor, ctx) { if (ctx.parser === undefined) { //is terminal return; From 797dd823be99d96a8d1061efec39cd35bca8ca55 Mon Sep 17 00:00:00 2001 From: Peter Boyer Date: Tue, 15 Dec 2015 21:44:47 -0500 Subject: [PATCH 04/95] Fix mistake in tokenIndex bounds check undefined < anyInteger returns false --- runtime/JavaScript/src/antlr4/BufferedTokenStream.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/JavaScript/src/antlr4/BufferedTokenStream.js b/runtime/JavaScript/src/antlr4/BufferedTokenStream.js index d8a6ba3c9..812321086 100644 --- a/runtime/JavaScript/src/antlr4/BufferedTokenStream.js +++ b/runtime/JavaScript/src/antlr4/BufferedTokenStream.js @@ -301,7 +301,7 @@ BufferedTokenStream.prototype.getHiddenTokensToRight = function(tokenIndex, channel = -1; } this.lazyInit(); - if (this.tokenIndex < 0 || tokenIndex >= this.tokens.length) { + if (tokenIndex < 0 || tokenIndex >= this.tokens.length) { throw "" + tokenIndex + " not in 0.." + this.tokens.length - 1; } var nextOnChannel = this.nextTokenOnChannel(tokenIndex + 1, From 3876cc600a9aac1660939374bfd15ebdbf22620a Mon Sep 17 00:00:00 2001 From: Sebastian Badur Date: Wed, 16 Dec 2015 20:45:44 +0100 Subject: [PATCH 05/95] Fixing typos in listeners.md - ParserRuleContext does not have type parameters - Confusing sentence cleaned up a little --- doc/listeners.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/listeners.md b/doc/listeners.md index 94b105771..022ec83cb 100644 --- a/doc/listeners.md +++ b/doc/listeners.md @@ -8,7 +8,7 @@ By default, ANTLR-generated parsers build a data structure called a parse tree o The interior nodes of the parse tree are phrase names that group and identify their children. The root node is the most abstract phrase name, in this case `stat` (short for statement). The leaves of a parse tree are always the input tokens. Parse trees sit between a language recognizer and an interpreter or translator implementation. They are extremely effective data structures because they contain all of the input and complete knowledge of how the parser grouped the symbols into phrases. Better yet, they are easy to understand and the parser generates them automatically (unless you turn them off with `parser.setBuildParseTree(false)`). -Because we specify phrase structure with a set of rules, parse tree subtree roots correspond to grammar rule names. ANTLR has a ParseTreeWalker that knows how to walk these parse trees and trigger events in listener implementation objects that you can create. ANTLR the tool generates listener interfaces for you also unless you, unless you turn that off with a commandline option. You can also have it generate visitors. For example from a Java.g4 grammar, ANTLR generates: +Because we specify phrase structure with a set of rules, parse tree subtree roots correspond to grammar rule names. ANTLR has a ParseTreeWalker that knows how to walk these parse trees and trigger events in listener implementation objects that you can create. ANTLR generates listener interfaces for you also, unless you turn that off with a commandline option. You can also have it generate visitors. For example from a Java.g4 grammar, ANTLR generates: ```java public interface JavaListener extends ParseTreeListener { @@ -27,7 +27,7 @@ Assuming you've created a listener object called `MyListener`, here is how to ca JavaLexer lexer = new JavaLexer(input); CommonTokenStream tokens = new CommonTokenStream(lexer); JavaParser parser = new JavaParser(tokens); -ParserRuleContext tree = parser.compilationUnit(); // parse +ParserRuleContext tree = parser.compilationUnit(); // parse ParseTreeWalker walker = new ParseTreeWalker(); // create standard walker MyListener extractor = new MyListener(parser); From a631e3007afae7145cd4ba10a1e62dd192c0ece1 Mon Sep 17 00:00:00 2001 From: sebadur Date: Thu, 17 Dec 2015 01:15:49 +0100 Subject: [PATCH 06/95] Signed to contributors.txt - Also implemented sharwell's suggested changes of originally pull request --- contributors.txt | 3 ++- doc/listeners.md | 9 ++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/contributors.txt b/contributors.txt index df94b18f3..fc7d0d38c 100644 --- a/contributors.txt +++ b/contributors.txt @@ -83,4 +83,5 @@ YYYY/MM/DD, github id, Full name, email 2015/10/12, KvanTTT, Ivan Kochurkin, ivan.kochurkin@gmail.com 2015/10/21, martin-probst, Martin Probst, martin-probst@web.de 2015/10/21, hkff, Walid Benghabrit, walid.benghabrit@mines-nantes.fr -2015/11/25, abego, Udo Borkowski, ub@abego.org \ No newline at end of file +2015/11/25, abego, Udo Borkowski, ub@abego.org +2015/12/17, sebadur, Sebastian Badur, sebadur@users.noreply.github.com \ No newline at end of file diff --git a/doc/listeners.md b/doc/listeners.md index 022ec83cb..c3bcad9c1 100644 --- a/doc/listeners.md +++ b/doc/listeners.md @@ -8,7 +8,7 @@ By default, ANTLR-generated parsers build a data structure called a parse tree o The interior nodes of the parse tree are phrase names that group and identify their children. The root node is the most abstract phrase name, in this case `stat` (short for statement). The leaves of a parse tree are always the input tokens. Parse trees sit between a language recognizer and an interpreter or translator implementation. They are extremely effective data structures because they contain all of the input and complete knowledge of how the parser grouped the symbols into phrases. Better yet, they are easy to understand and the parser generates them automatically (unless you turn them off with `parser.setBuildParseTree(false)`). -Because we specify phrase structure with a set of rules, parse tree subtree roots correspond to grammar rule names. ANTLR has a ParseTreeWalker that knows how to walk these parse trees and trigger events in listener implementation objects that you can create. ANTLR generates listener interfaces for you also, unless you turn that off with a commandline option. You can also have it generate visitors. For example from a Java.g4 grammar, ANTLR generates: +Because we specify phrase structure with a set of rules, parse tree subtree roots correspond to grammar rule names. ANTLR has a ParseTreeWalker that knows how to walk these parse trees and trigger events in listener implementation objects that you can create. The ANTLR tool generates listener interfaces for you also, unless you turn that off with a commandline option. You can also have it generate visitors. For example from a Java.g4 grammar, ANTLR generates: ```java public interface JavaListener extends ParseTreeListener { @@ -27,11 +27,10 @@ Assuming you've created a listener object called `MyListener`, here is how to ca JavaLexer lexer = new JavaLexer(input); CommonTokenStream tokens = new CommonTokenStream(lexer); JavaParser parser = new JavaParser(tokens); -ParserRuleContext tree = parser.compilationUnit(); // parse - -ParseTreeWalker walker = new ParseTreeWalker(); // create standard walker +JavaParser.CompilationUnitContext tree = parser.compilationUnit(); // parse a compilationUnit + MyListener extractor = new MyListener(parser); -walker.walk(extractor, tree); // initiate walk of tree with listener +ParseTreeWalker.DEFAULT.walk(extractor, tree); // initiate walk of tree with listener in use of default walker ``` Listeners and visitors are great because they keep application-specific code out of grammars, making grammars easier to read and preventing them from getting entangled with a particular application. From 6a7cae8ea91f31c021a34f352794324a4654cf22 Mon Sep 17 00:00:00 2001 From: Peter Boyer Date: Wed, 23 Dec 2015 14:33:05 -0600 Subject: [PATCH 07/95] Update contributors.txt --- contributors.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contributors.txt b/contributors.txt index df94b18f3..1097bc53e 100644 --- a/contributors.txt +++ b/contributors.txt @@ -83,4 +83,5 @@ YYYY/MM/DD, github id, Full name, email 2015/10/12, KvanTTT, Ivan Kochurkin, ivan.kochurkin@gmail.com 2015/10/21, martin-probst, Martin Probst, martin-probst@web.de 2015/10/21, hkff, Walid Benghabrit, walid.benghabrit@mines-nantes.fr -2015/11/25, abego, Udo Borkowski, ub@abego.org \ No newline at end of file +2015/11/25, abego, Udo Borkowski, ub@abego.org +2015/12/23, pboyer, Peter Boyer, peter.b.boyer@gmail.com From 0455e0a09f3a060544481355c87cc9d443139fc4 Mon Sep 17 00:00:00 2001 From: David Tymon Date: Thu, 24 Dec 2015 12:23:09 +1100 Subject: [PATCH 08/95] JS: Fix missing 'requires' statements --- runtime/JavaScript/src/antlr4/atn/PredictionMode.js | 5 ++++- runtime/JavaScript/src/antlr4/dfa/DFAState.js | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/runtime/JavaScript/src/antlr4/atn/PredictionMode.js b/runtime/JavaScript/src/antlr4/atn/PredictionMode.js index 6d717ee9b..f192d229f 100644 --- a/runtime/JavaScript/src/antlr4/atn/PredictionMode.js +++ b/runtime/JavaScript/src/antlr4/atn/PredictionMode.js @@ -38,6 +38,9 @@ var BitSet = require('./../Utils').BitSet; var AltDict = require('./../Utils').AltDict; var ATN = require('./ATN').ATN; var RuleStopState = require('./ATNState').RuleStopState; +var ATNConfigSet = require('./ATNConfigSet'); +var ATNConfig = require('./ATNConfig'); +var SemanticContext = require('./SemanticContext').SemanticContext; function PredictionMode() { return this; @@ -580,4 +583,4 @@ PredictionMode.getSingleViableAlt = function(altsets) { return result; }; -exports.PredictionMode = PredictionMode; \ No newline at end of file +exports.PredictionMode = PredictionMode; diff --git a/runtime/JavaScript/src/antlr4/dfa/DFAState.js b/runtime/JavaScript/src/antlr4/dfa/DFAState.js index 481a3f67e..a1d6c9945 100644 --- a/runtime/JavaScript/src/antlr4/dfa/DFAState.js +++ b/runtime/JavaScript/src/antlr4/dfa/DFAState.js @@ -30,6 +30,8 @@ /// var ATNConfigSet = require('./../atn/ATNConfigSet').ATNConfigSet; +var Utils = require('./../Utils'); +var Set = Utils.Set; // Map a predicate to a predicted alternative./// @@ -163,4 +165,4 @@ DFAState.prototype.hashString = function() { }; exports.DFAState = DFAState; -exports.PredPrediction = PredPrediction; \ No newline at end of file +exports.PredPrediction = PredPrediction; From f40ce5a18861e25c0299dd466cec59ef77e9cfff Mon Sep 17 00:00:00 2001 From: David Tymon Date: Thu, 24 Dec 2015 13:01:40 +1100 Subject: [PATCH 09/95] JS: LL1Analyzer passing PredictionContext incorrectly to ATNConfig constructor The PredictionContext should be passed to the ATNConfig constructor in the first argument, the params object. Instead, it is being passed as the second argument which is intended to be the config. --- runtime/JavaScript/src/antlr4/LL1Analyzer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/JavaScript/src/antlr4/LL1Analyzer.js b/runtime/JavaScript/src/antlr4/LL1Analyzer.js index c0270bb55..42d521268 100644 --- a/runtime/JavaScript/src/antlr4/LL1Analyzer.js +++ b/runtime/JavaScript/src/antlr4/LL1Analyzer.js @@ -145,7 +145,7 @@ LL1Analyzer.prototype.LOOK = function(s, stopState, ctx) { // is {@code null}. /// LL1Analyzer.prototype._LOOK = function(s, stopState , ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF) { - var c = new ATNConfig({state:s, alt:0}, ctx); + var c = new ATNConfig({state:s, alt:0, context: ctx}, null); if (lookBusy.contains(c)) { return; } From 812794600de77ad54cc2beb4fed5844d59514da3 Mon Sep 17 00:00:00 2001 From: David Tymon Date: Thu, 24 Dec 2015 13:06:59 +1100 Subject: [PATCH 10/95] JS: LexerActionExecutor cached hash string name clashes with function LexerActionExecutor caches its hash string in a member called 'hashString'. However, the class also has a method with the same name which leads to unexpected results. The member has been renamed to '_hashString' to avoid the name clash. --- runtime/JavaScript/src/antlr4/atn/LexerActionExecutor.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runtime/JavaScript/src/antlr4/atn/LexerActionExecutor.js b/runtime/JavaScript/src/antlr4/atn/LexerActionExecutor.js index 51e882c33..ff020a991 100644 --- a/runtime/JavaScript/src/antlr4/atn/LexerActionExecutor.js +++ b/runtime/JavaScript/src/antlr4/atn/LexerActionExecutor.js @@ -42,7 +42,7 @@ function LexerActionExecutor(lexerActions) { this.lexerActions = lexerActions === null ? [] : lexerActions; // Caches the result of {@link //hashCode} since the hash code is an element // of the performance-critical {@link LexerATNConfig//hashCode} operation. - this.hashString = lexerActions.toString(); // "".join([str(la) for la in + this._hashString = lexerActions.toString(); // "".join([str(la) for la in // lexerActions])) return this; } @@ -158,7 +158,7 @@ LexerActionExecutor.prototype.execute = function(lexer, input, startIndex) { }; LexerActionExecutor.prototype.hashString = function() { - return this.hashString; + return this._hashString; }; LexerActionExecutor.prototype.equals = function(other) { @@ -167,7 +167,7 @@ LexerActionExecutor.prototype.equals = function(other) { } else if (!(other instanceof LexerActionExecutor)) { return false; } else { - return this.hashString === other.hashString && + return this._hashString === other._hashString && this.lexerActions === other.lexerActions; } }; From f77a8c913944b34d2a084159a371f0f9061cd4e5 Mon Sep 17 00:00:00 2001 From: David Tymon Date: Thu, 24 Dec 2015 14:45:07 +1100 Subject: [PATCH 11/95] Sign contributors list --- contributors.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contributors.txt b/contributors.txt index fc7d0d38c..f5fc50adc 100644 --- a/contributors.txt +++ b/contributors.txt @@ -84,4 +84,5 @@ YYYY/MM/DD, github id, Full name, email 2015/10/21, martin-probst, Martin Probst, martin-probst@web.de 2015/10/21, hkff, Walid Benghabrit, walid.benghabrit@mines-nantes.fr 2015/11/25, abego, Udo Borkowski, ub@abego.org -2015/12/17, sebadur, Sebastian Badur, sebadur@users.noreply.github.com \ No newline at end of file +2015/12/17, sebadur, Sebastian Badur, sebadur@users.noreply.github.com +2015/12/24, dtymon, David Tymon, david.tymon@gmail.com From 4a1ec7df3828d2415680dfb6766c79caa3b449bb Mon Sep 17 00:00:00 2001 From: "FUJI Goro (gfx)" Date: Thu, 24 Dec 2015 14:04:58 +0900 Subject: [PATCH 12/95] Remove Utils#waitForClose() It is used nowhere but imports java.awt.*; Android runtime has no java.awt.* so Android SDK build tools say "it includes invalid packages". It's better if antlr4-runtime has no dependency on java.awt.*, esp. it is not used anymore. --- .../src/org/antlr/v4/runtime/misc/Utils.java | 35 ------------------- 1 file changed, 35 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java b/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java index 011e8f26f..991ee2b35 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java @@ -30,9 +30,6 @@ package org.antlr.v4.runtime.misc; -import java.awt.*; -import java.awt.event.WindowAdapter; -import java.awt.event.WindowEvent; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -151,38 +148,6 @@ public class Utils { return data; } - public static void waitForClose(final Window window) throws InterruptedException { - final Object lock = new Object(); - - Thread t = new Thread() { - @Override - public void run() { - synchronized (lock) { - while (window.isVisible()) { - try { - lock.wait(500); - } catch (InterruptedException e) { - } - } - } - } - }; - - t.start(); - - window.addWindowListener(new WindowAdapter() { - @Override - public void windowClosing(WindowEvent arg0) { - synchronized (lock) { - window.setVisible(false); - lock.notify(); - } - } - }); - - t.join(); - } - /** Convert array of strings to string→index map. Useful for * converting rulenames to name→ruleindex map. */ From a1c63234e10c616b5437d23df7b2de882f3604ff Mon Sep 17 00:00:00 2001 From: David Tymon Date: Thu, 24 Dec 2015 19:04:22 +1100 Subject: [PATCH 13/95] JS: Added 'require' statements are missing the class names --- runtime/JavaScript/src/antlr4/atn/PredictionMode.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/JavaScript/src/antlr4/atn/PredictionMode.js b/runtime/JavaScript/src/antlr4/atn/PredictionMode.js index f192d229f..c04d502ba 100644 --- a/runtime/JavaScript/src/antlr4/atn/PredictionMode.js +++ b/runtime/JavaScript/src/antlr4/atn/PredictionMode.js @@ -38,8 +38,8 @@ var BitSet = require('./../Utils').BitSet; var AltDict = require('./../Utils').AltDict; var ATN = require('./ATN').ATN; var RuleStopState = require('./ATNState').RuleStopState; -var ATNConfigSet = require('./ATNConfigSet'); -var ATNConfig = require('./ATNConfig'); +var ATNConfigSet = require('./ATNConfigSet').ATNConfigSet; +var ATNConfig = require('./ATNConfig').ATNConfig; var SemanticContext = require('./SemanticContext').SemanticContext; function PredictionMode() { From c432ce8889dc2782d7bc0fcb480fe1f6010f522f Mon Sep 17 00:00:00 2001 From: Peter Boyer Date: Sun, 3 Jan 2016 21:15:44 -0500 Subject: [PATCH 14/95] Update ace-javascript-target.md --- doc/ace-javascript-target.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/doc/ace-javascript-target.md b/doc/ace-javascript-target.md index 08a512ca7..5f4ea1716 100644 --- a/doc/ace-javascript-target.md +++ b/doc/ace-javascript-target.md @@ -176,13 +176,13 @@ What remains to be done is have our validate function actually validate the inpu To start with, let's load ANTLR and your parser, listener etc.. Easy, since you could write: -``` +```js var antlr4 = require('antlr4/index'); ``` This may work, but it's actually unreliable. The reason is that the require function used by ANTLR, which exactly mimics the NodeJS require function, uses a different syntax than the require function that comes with ACE. So we need to bring in a require function that conforms to the NodeJS syntax. I personally use one that comes from Torben Haase's Honey project, which you can find here. But hey, now we're going to have 2 'require' functions not compatible with each other! Indeed, this is why you need to take special care, as follows: -``` +```js // load nodejs compatible require var ace_require = require; require = undefined; @@ -200,10 +200,12 @@ try { } finally { require = ace_require; } +``` Please note the try-finally construct. ANTLR uses 'require' synchronously so it's perfectly safe to ignore the ACE 'require' while running ANTLR code. ACE itself does not guarantee synchronous execution, so you are much safer always switching 'require' back to 'ace_require'. Now detecting deep syntax errors in your code is a task for your ANTLR listener or visitor or whatever piece of code you've delegated this to. We're not going to describe this here, since it would require some knowledge of your language. However, detecting grammar syntax errors is something ANTLR does beautifully (isn't that why you went for ANTLR in the first place?). So what we will illustrate here is how to report grammar syntax errors. I have no doubt that from there, you will be able to extend the validator to suit your specific needs. Whenever ANTLR encounters an unexpected token, it fires an error. By default, the error is routed to an error listener which simply writes to the console. What we need to do is replace this listener by our own listener, se we can route errors to the ACE editor. First, let's create such a listener: +```js // class for gathering errors and posting them to ACE editor var AnnotatingErrorListener = function(annotations) { antlr4.error.ErrorListener.call(this); @@ -222,9 +224,9 @@ AnnotatingErrorListener.prototype.syntaxError = function(recognizer, offendingSy type: "error" }); }; - +``` With this, all that remains to be done is plug the listener in when we parse the code. Here is how I do it: - +```js var validate = function(input) { var stream = new antlr4.InputStream(input); var lexer = new mylanguage.MyLexer(stream); @@ -236,7 +238,7 @@ var validate = function(input) { parser.addErrorListener(listener); parser.parseMyRule(); return annotations; -}; +};``` You know what? That's it! You now have an ACE editor that does syntax validation using ANTLR! I hope you find this useful, and simple enough to get started. What I did not address here is packaging, not something I'm an expert at. The good news is that it makes development simple, since I don't have to run any compilation process. I just edit my code, reload my editor page, and check how it goes. Now wait, hey! How do you debug this? Well, as usual, using Chrome, since neither Firefox or Safari are able to debug worker code. What a shame... From cceda58ed69316ceac9f23393b9b93cbf1be49c0 Mon Sep 17 00:00:00 2001 From: Peter Boyer Date: Sun, 3 Jan 2016 21:18:09 -0500 Subject: [PATCH 15/95] One more --- doc/ace-javascript-target.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/ace-javascript-target.md b/doc/ace-javascript-target.md index 5f4ea1716..a18bc11d9 100644 --- a/doc/ace-javascript-target.md +++ b/doc/ace-javascript-target.md @@ -190,7 +190,9 @@ var Honey = { 'requirePath': ['..'] }; // walk up to js folder, see Honey docs importScripts("../lib/require.js"); var antlr4_require = require; require = ace_require; +``` Now it's safe to load antlr, and the parsers generated for your language. Assuming that your language files (generated or hand-built) are in a folder with an index.js file that calls require for each file, your parser loading code can be as simple as follows: +```js // load antlr4 and myLanguage var antlr4, mylanguage; try { From 6bb241952a793587de8c132c3690e14766dc6445 Mon Sep 17 00:00:00 2001 From: Peter Boyer Date: Sun, 3 Jan 2016 21:18:32 -0500 Subject: [PATCH 16/95] Update ace-javascript-target.md --- doc/ace-javascript-target.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/ace-javascript-target.md b/doc/ace-javascript-target.md index a18bc11d9..fbba9e9b9 100644 --- a/doc/ace-javascript-target.md +++ b/doc/ace-javascript-target.md @@ -240,7 +240,8 @@ var validate = function(input) { parser.addErrorListener(listener); parser.parseMyRule(); return annotations; -};``` +}; +``` You know what? That's it! You now have an ACE editor that does syntax validation using ANTLR! I hope you find this useful, and simple enough to get started. What I did not address here is packaging, not something I'm an expert at. The good news is that it makes development simple, since I don't have to run any compilation process. I just edit my code, reload my editor page, and check how it goes. Now wait, hey! How do you debug this? Well, as usual, using Chrome, since neither Firefox or Safari are able to debug worker code. What a shame... From 7d478b18ae373cd43789593766effd5b116b2dea Mon Sep 17 00:00:00 2001 From: Josh Chase Date: Wed, 13 Jan 2016 16:24:56 -0800 Subject: [PATCH 17/95] Only run the maven plugin when needed. --- .../java/org/antlr/mojo/antlr4/Antlr4Mojo.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/antlr4-maven-plugin/src/main/java/org/antlr/mojo/antlr4/Antlr4Mojo.java b/antlr4-maven-plugin/src/main/java/org/antlr/mojo/antlr4/Antlr4Mojo.java index e7eb2d87d..d8712ee6a 100644 --- a/antlr4-maven-plugin/src/main/java/org/antlr/mojo/antlr4/Antlr4Mojo.java +++ b/antlr4-maven-plugin/src/main/java/org/antlr/mojo/antlr4/Antlr4Mojo.java @@ -372,6 +372,22 @@ public class Antlr4Mojo extends AbstractMojo { scan.addSourceMapping(mapping); Set grammarFiles = scan.getIncludedSources(sourceDirectory, null); + // We don't want the plugin to run for every grammar, regardless of whether + // it's changed since the last compilation. Check the mtime of the tokens vs + // the grammar file mtime to determine whether we even need to execute. + Set grammarFilesToProcess = new HashSet(); + + for (File grammarFile : grammarFiles) { + String tokensFileName = grammarFile.getName().split("\\.")[0] + ".tokens"; + File outputFile = new File(outputDirectory, tokensFileName); + if ( (! outputFile.exists()) || + outputFile.lastModified() < grammarFile.lastModified() ) { + grammarFilesToProcess.add(grammarFile); + } + } + + grammarFiles = grammarFilesToProcess; + if (grammarFiles.isEmpty()) { getLog().info("No grammars to process"); return Collections.emptyList(); From 9c6e212bc994b0f3fb8adef00abab329ad786d68 Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Sat, 16 Jan 2016 12:53:00 -0800 Subject: [PATCH 18/95] Update creating-a-language-target.md --- doc/creating-a-language-target.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/creating-a-language-target.md b/doc/creating-a-language-target.md index c926796bd..fe6150cb0 100644 --- a/doc/creating-a-language-target.md +++ b/doc/creating-a-language-target.md @@ -8,7 +8,7 @@ Creating a new target involves the following key elements: 1. For the tool, create class *X*Target as a subclass of class `Target` in package `org.antlr.v4.codegen.target`. This class describes language specific details about escape characters and strings and so on. There is very little to do here typically. 1. Create *X*.stg in directory tool/resources/org/antlr/v4/tool/templates/codegen/*X*/*X*.stg. This is a [StringTemplate](http://www.stringtemplate.org/) group file (`.stg`) that tells ANTLR how to express all of the parsing elements needed to generate code. You will see templates called `ParserFile`, `Parser`, `Lexer`, `CodeBlockForAlt`, `AltBlock`, etc... Each of these must be described how to build the indicated chunk of code. Your best bet is to find the closest existing target, copy that template file, and tweak to suit. -1. Create a runtime library to support the parsers generated by ANTLR. Under directory runtime/*X*, you are in complete control of the directory structure is dictated by common usage of that target language. For example, Java has: `runtime/Java/lib` and `runtime/Java/src` directories. Under `src`, you will find a directory structure for package `org.antlr.v4.runtime` and below. +1. Create a runtime library to support the parsers generated by ANTLR. Under directory runtime/*X*, you are in complete control of the directory structure as dictated by common usage of that target language. For example, Java has: `runtime/Java/lib` and `runtime/Java/src` directories. Under `src`, you will find a directory structure for package `org.antlr.v4.runtime` and below. 1. Create a template file for runtime tests. All you have to do is provide a few simple templates that indicate how to print values and declare variables. Our runtime test mechanism in dir `runtime-testsuite` will automatically generate code in a new target and check the results. All it needs to know is how to generate a test rig (i.e., a `main` program), how to define various class fields, compare members and so on. You must create a *X* directory underneath `runtime-testsuite/resources/org/antlr/v4/test/runtime`. Again, your best bet is to copy the templates from the closest language to your target and tweak it to suit. ## Getting started @@ -19,4 +19,4 @@ Creating a new target involves the following key elements: ```bash $ mvn compile ``` -That should proceed with success. See [Building ANTLR](building-antlr.md) for more details. (That link does not currently work as I have that documentation in a branch. see https://github.com/parrt/antlr4/blob/move-doc-to-repo/doc/building-antlr.md for now.) \ No newline at end of file +That should proceed with success. See [Building ANTLR](building-antlr.md) for more details. (That link does not currently work as I have that documentation in a branch. see https://github.com/parrt/antlr4/blob/move-doc-to-repo/doc/building-antlr.md for now.) From 3a733af5f7c011ba750822b2ead40fc4b9de25e9 Mon Sep 17 00:00:00 2001 From: parrt Date: Thu, 28 Jan 2016 10:47:57 -0800 Subject: [PATCH 19/95] Merge branch 'patch-1' of git://github.com/cooperra/antlr4 into cooperra-patch-1 # Conflicts: # contributors.txt --- contributors.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contributors.txt b/contributors.txt index 763a91f07..103a1cf76 100644 --- a/contributors.txt +++ b/contributors.txt @@ -83,6 +83,6 @@ YYYY/MM/DD, github id, Full name, email 2015/10/12, KvanTTT, Ivan Kochurkin, ivan.kochurkin@gmail.com 2015/10/21, martin-probst, Martin Probst, martin-probst@web.de 2015/10/21, hkff, Walid Benghabrit, walid.benghabrit@mines-nantes.fr -2015/11/25, abego, Udo Borkowski, ub@abego.org -2015/12/17, sebadur, Sebastian Badur, sebadur@users.noreply.github.com2015/06/29, jvanzyl, Jason van Zyl, jason@takari.io 2015/11/12, cooperra, Robbie Cooper, cooperra@users.noreply.github.com +2015/11/25, abego, Udo Borkowski, ub@abego.org +2015/12/17, sebadur, Sebastian Badur, sebadur@users.noreply.github.com From ca5f65c73168ce54e49c0450a73f58335fb5a60f Mon Sep 17 00:00:00 2001 From: parrt Date: Thu, 28 Jan 2016 10:55:15 -0800 Subject: [PATCH 20/95] Merge branch 'patch-1' of git://github.com/cooperra/antlr4 into cooperra-patch-1 Fixes #1081 --- contributors.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/contributors.txt b/contributors.txt index dbee1003b..aad64a51c 100644 --- a/contributors.txt +++ b/contributors.txt @@ -86,5 +86,4 @@ YYYY/MM/DD, github id, Full name, email 2015/11/12, cooperra, Robbie Cooper, cooperra@users.noreply.github.com 2015/11/25, abego, Udo Borkowski, ub@abego.org 2015/12/17, sebadur, Sebastian Badur, sebadur@users.noreply.github.com -2015/12/17, sebadur, Sebastian Badur, sebadur@users.noreply.github.com 2015/12/23, pboyer, Peter Boyer, peter.b.boyer@gmail.com From 121aba1bb87c4ba987ea37e08f1b32ebbb6049e9 Mon Sep 17 00:00:00 2001 From: David Tymon Date: Thu, 24 Dec 2015 14:45:07 +1100 Subject: [PATCH 21/95] pull in contrib --- contributors.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/contributors.txt b/contributors.txt index aad64a51c..7d3b44a41 100644 --- a/contributors.txt +++ b/contributors.txt @@ -87,3 +87,4 @@ YYYY/MM/DD, github id, Full name, email 2015/11/25, abego, Udo Borkowski, ub@abego.org 2015/12/17, sebadur, Sebastian Badur, sebadur@users.noreply.github.com 2015/12/23, pboyer, Peter Boyer, peter.b.boyer@gmail.com +2015/12/24, dtymon, David Tymon, david.tymon@gmail.com From 7b4e5082e6346de650fba90dc7c358fbdc5b7245 Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Thu, 28 Jan 2016 20:17:27 -0800 Subject: [PATCH 22/95] tweak contrib --- contributors.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/contributors.txt b/contributors.txt index 2a862de98..7d3b44a41 100644 --- a/contributors.txt +++ b/contributors.txt @@ -88,5 +88,3 @@ YYYY/MM/DD, github id, Full name, email 2015/12/17, sebadur, Sebastian Badur, sebadur@users.noreply.github.com 2015/12/23, pboyer, Peter Boyer, peter.b.boyer@gmail.com 2015/12/24, dtymon, David Tymon, david.tymon@gmail.com -2015/12/17, sebadur, Sebastian Badur, sebadur@users.noreply.github.com -2015/12/24, dtymon, David Tymon, david.tymon@gmail.com From 2d9f2b574b713f5b5d52430db8c48d76687f0bf6 Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Fri, 29 Jan 2016 13:13:12 -0800 Subject: [PATCH 23/95] add travis status --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f1c4d2c22..c1d6902de 100644 --- a/README.md +++ b/README.md @@ -36,8 +36,12 @@ You can buy the book [The Definitive ANTLR 4 Reference](http://amzn.com/19343569 You will find the [Book source code](http://pragprog.com/titles/tpantlr2/source_code) useful. - ## Additional grammars [This repository](https://github.com/antlr/grammars-v4) is a collection of grammars without actions where the root directory name is the all-lowercase name of the language parsed by the grammar. For example, java, cpp, csharp, c, etc... + +Travis Status +--------- + + From b446c50722e389b03fd7e5baea80b371a9717584 Mon Sep 17 00:00:00 2001 From: parrt Date: Fri, 29 Jan 2016 14:13:11 -0800 Subject: [PATCH 24/95] [maven-release-plugin] prepare release 4.5.2 --- antlr4-maven-plugin/pom.xml | 2 +- pom.xml | 2 +- runtime-testsuite/pom.xml | 2 +- runtime/Java/pom.xml | 2 +- tool-testsuite/pom.xml | 2 +- tool/pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/antlr4-maven-plugin/pom.xml b/antlr4-maven-plugin/pom.xml index 4495a2a21..61d1b1b91 100644 --- a/antlr4-maven-plugin/pom.xml +++ b/antlr4-maven-plugin/pom.xml @@ -34,7 +34,7 @@ org.antlr antlr4-master - 4.5.2-SNAPSHOT + 4.5.2 antlr4-maven-plugin maven-plugin diff --git a/pom.xml b/pom.xml index 92ca59769..b08cc99fe 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.antlr antlr4-master - 4.5.2-SNAPSHOT + 4.5.2 pom ANTLR 4 diff --git a/runtime-testsuite/pom.xml b/runtime-testsuite/pom.xml index 40e2f2a3a..4c9da053c 100644 --- a/runtime-testsuite/pom.xml +++ b/runtime-testsuite/pom.xml @@ -4,7 +4,7 @@ org.antlr antlr4-master - 4.5.2-SNAPSHOT + 4.5.2 antlr4-runtime-testsuite ANTLR 4 Runtime Test Generator diff --git a/runtime/Java/pom.xml b/runtime/Java/pom.xml index b9c026525..bcd5c1f62 100644 --- a/runtime/Java/pom.xml +++ b/runtime/Java/pom.xml @@ -3,7 +3,7 @@ org.antlr antlr4-master - 4.5.2-SNAPSHOT + 4.5.2 ../../pom.xml antlr4-runtime diff --git a/tool-testsuite/pom.xml b/tool-testsuite/pom.xml index dad48a51c..b7a1debc0 100644 --- a/tool-testsuite/pom.xml +++ b/tool-testsuite/pom.xml @@ -4,7 +4,7 @@ org.antlr antlr4-master - 4.5.2-SNAPSHOT + 4.5.2 antlr4-tool-testsuite ANTLR 4 Tool Tests diff --git a/tool/pom.xml b/tool/pom.xml index 073f0cbd1..3067a1939 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -3,7 +3,7 @@ org.antlr antlr4-master - 4.5.2-SNAPSHOT + 4.5.2 antlr4 ANTLR 4 Tool From 6b3f9816cb7c8064fc398720769f4eb067296ac3 Mon Sep 17 00:00:00 2001 From: parrt Date: Fri, 29 Jan 2016 14:13:15 -0800 Subject: [PATCH 25/95] [maven-release-plugin] prepare for next development iteration --- antlr4-maven-plugin/pom.xml | 2 +- pom.xml | 2 +- runtime-testsuite/pom.xml | 2 +- runtime/Java/pom.xml | 2 +- tool-testsuite/pom.xml | 2 +- tool/pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/antlr4-maven-plugin/pom.xml b/antlr4-maven-plugin/pom.xml index 61d1b1b91..2ebf43d63 100644 --- a/antlr4-maven-plugin/pom.xml +++ b/antlr4-maven-plugin/pom.xml @@ -34,7 +34,7 @@ org.antlr antlr4-master - 4.5.2 + 4.5.3-SNAPSHOT antlr4-maven-plugin maven-plugin diff --git a/pom.xml b/pom.xml index b08cc99fe..4ff16411c 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.antlr antlr4-master - 4.5.2 + 4.5.3-SNAPSHOT pom ANTLR 4 diff --git a/runtime-testsuite/pom.xml b/runtime-testsuite/pom.xml index 4c9da053c..f9f105200 100644 --- a/runtime-testsuite/pom.xml +++ b/runtime-testsuite/pom.xml @@ -4,7 +4,7 @@ org.antlr antlr4-master - 4.5.2 + 4.5.3-SNAPSHOT antlr4-runtime-testsuite ANTLR 4 Runtime Test Generator diff --git a/runtime/Java/pom.xml b/runtime/Java/pom.xml index bcd5c1f62..c00729bf3 100644 --- a/runtime/Java/pom.xml +++ b/runtime/Java/pom.xml @@ -3,7 +3,7 @@ org.antlr antlr4-master - 4.5.2 + 4.5.3-SNAPSHOT ../../pom.xml antlr4-runtime diff --git a/tool-testsuite/pom.xml b/tool-testsuite/pom.xml index b7a1debc0..cea14b194 100644 --- a/tool-testsuite/pom.xml +++ b/tool-testsuite/pom.xml @@ -4,7 +4,7 @@ org.antlr antlr4-master - 4.5.2 + 4.5.3-SNAPSHOT antlr4-tool-testsuite ANTLR 4 Tool Tests diff --git a/tool/pom.xml b/tool/pom.xml index 3067a1939..bda5bbd53 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -3,7 +3,7 @@ org.antlr antlr4-master - 4.5.2 + 4.5.3-SNAPSHOT antlr4 ANTLR 4 Tool From 6b31ceaa9dc04912a456746c964f35b23d38d97f Mon Sep 17 00:00:00 2001 From: parrt Date: Fri, 29 Jan 2016 15:26:02 -0800 Subject: [PATCH 26/95] [maven-release-plugin] rollback the release of 4.5.2 --- antlr4-maven-plugin/pom.xml | 2 +- pom.xml | 2 +- runtime-testsuite/pom.xml | 2 +- runtime/Java/pom.xml | 2 +- tool-testsuite/pom.xml | 2 +- tool/pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/antlr4-maven-plugin/pom.xml b/antlr4-maven-plugin/pom.xml index 2ebf43d63..4495a2a21 100644 --- a/antlr4-maven-plugin/pom.xml +++ b/antlr4-maven-plugin/pom.xml @@ -34,7 +34,7 @@ org.antlr antlr4-master - 4.5.3-SNAPSHOT + 4.5.2-SNAPSHOT antlr4-maven-plugin maven-plugin diff --git a/pom.xml b/pom.xml index 4ff16411c..92ca59769 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.antlr antlr4-master - 4.5.3-SNAPSHOT + 4.5.2-SNAPSHOT pom ANTLR 4 diff --git a/runtime-testsuite/pom.xml b/runtime-testsuite/pom.xml index f9f105200..40e2f2a3a 100644 --- a/runtime-testsuite/pom.xml +++ b/runtime-testsuite/pom.xml @@ -4,7 +4,7 @@ org.antlr antlr4-master - 4.5.3-SNAPSHOT + 4.5.2-SNAPSHOT antlr4-runtime-testsuite ANTLR 4 Runtime Test Generator diff --git a/runtime/Java/pom.xml b/runtime/Java/pom.xml index c00729bf3..b9c026525 100644 --- a/runtime/Java/pom.xml +++ b/runtime/Java/pom.xml @@ -3,7 +3,7 @@ org.antlr antlr4-master - 4.5.3-SNAPSHOT + 4.5.2-SNAPSHOT ../../pom.xml antlr4-runtime diff --git a/tool-testsuite/pom.xml b/tool-testsuite/pom.xml index cea14b194..dad48a51c 100644 --- a/tool-testsuite/pom.xml +++ b/tool-testsuite/pom.xml @@ -4,7 +4,7 @@ org.antlr antlr4-master - 4.5.3-SNAPSHOT + 4.5.2-SNAPSHOT antlr4-tool-testsuite ANTLR 4 Tool Tests diff --git a/tool/pom.xml b/tool/pom.xml index bda5bbd53..073f0cbd1 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -3,7 +3,7 @@ org.antlr antlr4-master - 4.5.3-SNAPSHOT + 4.5.2-SNAPSHOT antlr4 ANTLR 4 Tool From 009f3cac3a1c5d615f00f534d0b5855717a98386 Mon Sep 17 00:00:00 2001 From: parrt Date: Fri, 29 Jan 2016 15:27:19 -0800 Subject: [PATCH 27/95] [maven-release-plugin] prepare release 4.5.2 --- antlr4-maven-plugin/pom.xml | 2 +- pom.xml | 2 +- runtime-testsuite/pom.xml | 2 +- runtime/Java/pom.xml | 2 +- tool-testsuite/pom.xml | 2 +- tool/pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/antlr4-maven-plugin/pom.xml b/antlr4-maven-plugin/pom.xml index 4495a2a21..61d1b1b91 100644 --- a/antlr4-maven-plugin/pom.xml +++ b/antlr4-maven-plugin/pom.xml @@ -34,7 +34,7 @@ org.antlr antlr4-master - 4.5.2-SNAPSHOT + 4.5.2 antlr4-maven-plugin maven-plugin diff --git a/pom.xml b/pom.xml index 92ca59769..b08cc99fe 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.antlr antlr4-master - 4.5.2-SNAPSHOT + 4.5.2 pom ANTLR 4 diff --git a/runtime-testsuite/pom.xml b/runtime-testsuite/pom.xml index 40e2f2a3a..4c9da053c 100644 --- a/runtime-testsuite/pom.xml +++ b/runtime-testsuite/pom.xml @@ -4,7 +4,7 @@ org.antlr antlr4-master - 4.5.2-SNAPSHOT + 4.5.2 antlr4-runtime-testsuite ANTLR 4 Runtime Test Generator diff --git a/runtime/Java/pom.xml b/runtime/Java/pom.xml index b9c026525..bcd5c1f62 100644 --- a/runtime/Java/pom.xml +++ b/runtime/Java/pom.xml @@ -3,7 +3,7 @@ org.antlr antlr4-master - 4.5.2-SNAPSHOT + 4.5.2 ../../pom.xml antlr4-runtime diff --git a/tool-testsuite/pom.xml b/tool-testsuite/pom.xml index dad48a51c..b7a1debc0 100644 --- a/tool-testsuite/pom.xml +++ b/tool-testsuite/pom.xml @@ -4,7 +4,7 @@ org.antlr antlr4-master - 4.5.2-SNAPSHOT + 4.5.2 antlr4-tool-testsuite ANTLR 4 Tool Tests diff --git a/tool/pom.xml b/tool/pom.xml index 073f0cbd1..3067a1939 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -3,7 +3,7 @@ org.antlr antlr4-master - 4.5.2-SNAPSHOT + 4.5.2 antlr4 ANTLR 4 Tool From fa6f674e33dbeeb3b33cb3abd877bebc9e6e5d5d Mon Sep 17 00:00:00 2001 From: parrt Date: Fri, 29 Jan 2016 15:27:40 -0800 Subject: [PATCH 28/95] [maven-release-plugin] rollback the release of 4.5.2 --- antlr4-maven-plugin/pom.xml | 2 +- pom.xml | 2 +- runtime-testsuite/pom.xml | 2 +- runtime/Java/pom.xml | 2 +- tool-testsuite/pom.xml | 2 +- tool/pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/antlr4-maven-plugin/pom.xml b/antlr4-maven-plugin/pom.xml index 61d1b1b91..4495a2a21 100644 --- a/antlr4-maven-plugin/pom.xml +++ b/antlr4-maven-plugin/pom.xml @@ -34,7 +34,7 @@ org.antlr antlr4-master - 4.5.2 + 4.5.2-SNAPSHOT antlr4-maven-plugin maven-plugin diff --git a/pom.xml b/pom.xml index b08cc99fe..92ca59769 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.antlr antlr4-master - 4.5.2 + 4.5.2-SNAPSHOT pom ANTLR 4 diff --git a/runtime-testsuite/pom.xml b/runtime-testsuite/pom.xml index 4c9da053c..40e2f2a3a 100644 --- a/runtime-testsuite/pom.xml +++ b/runtime-testsuite/pom.xml @@ -4,7 +4,7 @@ org.antlr antlr4-master - 4.5.2 + 4.5.2-SNAPSHOT antlr4-runtime-testsuite ANTLR 4 Runtime Test Generator diff --git a/runtime/Java/pom.xml b/runtime/Java/pom.xml index bcd5c1f62..b9c026525 100644 --- a/runtime/Java/pom.xml +++ b/runtime/Java/pom.xml @@ -3,7 +3,7 @@ org.antlr antlr4-master - 4.5.2 + 4.5.2-SNAPSHOT ../../pom.xml antlr4-runtime diff --git a/tool-testsuite/pom.xml b/tool-testsuite/pom.xml index b7a1debc0..dad48a51c 100644 --- a/tool-testsuite/pom.xml +++ b/tool-testsuite/pom.xml @@ -4,7 +4,7 @@ org.antlr antlr4-master - 4.5.2 + 4.5.2-SNAPSHOT antlr4-tool-testsuite ANTLR 4 Tool Tests diff --git a/tool/pom.xml b/tool/pom.xml index 3067a1939..073f0cbd1 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -3,7 +3,7 @@ org.antlr antlr4-master - 4.5.2 + 4.5.2-SNAPSHOT antlr4 ANTLR 4 Tool From c5a296596d78d15c452d4cac700916a0e4208ce6 Mon Sep 17 00:00:00 2001 From: parrt Date: Fri, 29 Jan 2016 15:28:57 -0800 Subject: [PATCH 29/95] [maven-release-plugin] prepare release 4.5.2 --- antlr4-maven-plugin/pom.xml | 2 +- pom.xml | 2 +- runtime-testsuite/pom.xml | 2 +- runtime/Java/pom.xml | 2 +- tool-testsuite/pom.xml | 2 +- tool/pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/antlr4-maven-plugin/pom.xml b/antlr4-maven-plugin/pom.xml index 4495a2a21..61d1b1b91 100644 --- a/antlr4-maven-plugin/pom.xml +++ b/antlr4-maven-plugin/pom.xml @@ -34,7 +34,7 @@ org.antlr antlr4-master - 4.5.2-SNAPSHOT + 4.5.2 antlr4-maven-plugin maven-plugin diff --git a/pom.xml b/pom.xml index 92ca59769..b08cc99fe 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.antlr antlr4-master - 4.5.2-SNAPSHOT + 4.5.2 pom ANTLR 4 diff --git a/runtime-testsuite/pom.xml b/runtime-testsuite/pom.xml index 40e2f2a3a..4c9da053c 100644 --- a/runtime-testsuite/pom.xml +++ b/runtime-testsuite/pom.xml @@ -4,7 +4,7 @@ org.antlr antlr4-master - 4.5.2-SNAPSHOT + 4.5.2 antlr4-runtime-testsuite ANTLR 4 Runtime Test Generator diff --git a/runtime/Java/pom.xml b/runtime/Java/pom.xml index b9c026525..bcd5c1f62 100644 --- a/runtime/Java/pom.xml +++ b/runtime/Java/pom.xml @@ -3,7 +3,7 @@ org.antlr antlr4-master - 4.5.2-SNAPSHOT + 4.5.2 ../../pom.xml antlr4-runtime diff --git a/tool-testsuite/pom.xml b/tool-testsuite/pom.xml index dad48a51c..b7a1debc0 100644 --- a/tool-testsuite/pom.xml +++ b/tool-testsuite/pom.xml @@ -4,7 +4,7 @@ org.antlr antlr4-master - 4.5.2-SNAPSHOT + 4.5.2 antlr4-tool-testsuite ANTLR 4 Tool Tests diff --git a/tool/pom.xml b/tool/pom.xml index 073f0cbd1..3067a1939 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -3,7 +3,7 @@ org.antlr antlr4-master - 4.5.2-SNAPSHOT + 4.5.2 antlr4 ANTLR 4 Tool From 08f3ad798e245f334778a6ce232daae232e38a9b Mon Sep 17 00:00:00 2001 From: parrt Date: Fri, 29 Jan 2016 15:29:01 -0800 Subject: [PATCH 30/95] [maven-release-plugin] prepare for next development iteration --- antlr4-maven-plugin/pom.xml | 2 +- pom.xml | 2 +- runtime-testsuite/pom.xml | 2 +- runtime/Java/pom.xml | 2 +- tool-testsuite/pom.xml | 2 +- tool/pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/antlr4-maven-plugin/pom.xml b/antlr4-maven-plugin/pom.xml index 61d1b1b91..2ebf43d63 100644 --- a/antlr4-maven-plugin/pom.xml +++ b/antlr4-maven-plugin/pom.xml @@ -34,7 +34,7 @@ org.antlr antlr4-master - 4.5.2 + 4.5.3-SNAPSHOT antlr4-maven-plugin maven-plugin diff --git a/pom.xml b/pom.xml index b08cc99fe..4ff16411c 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.antlr antlr4-master - 4.5.2 + 4.5.3-SNAPSHOT pom ANTLR 4 diff --git a/runtime-testsuite/pom.xml b/runtime-testsuite/pom.xml index 4c9da053c..f9f105200 100644 --- a/runtime-testsuite/pom.xml +++ b/runtime-testsuite/pom.xml @@ -4,7 +4,7 @@ org.antlr antlr4-master - 4.5.2 + 4.5.3-SNAPSHOT antlr4-runtime-testsuite ANTLR 4 Runtime Test Generator diff --git a/runtime/Java/pom.xml b/runtime/Java/pom.xml index bcd5c1f62..c00729bf3 100644 --- a/runtime/Java/pom.xml +++ b/runtime/Java/pom.xml @@ -3,7 +3,7 @@ org.antlr antlr4-master - 4.5.2 + 4.5.3-SNAPSHOT ../../pom.xml antlr4-runtime diff --git a/tool-testsuite/pom.xml b/tool-testsuite/pom.xml index b7a1debc0..cea14b194 100644 --- a/tool-testsuite/pom.xml +++ b/tool-testsuite/pom.xml @@ -4,7 +4,7 @@ org.antlr antlr4-master - 4.5.2 + 4.5.3-SNAPSHOT antlr4-tool-testsuite ANTLR 4 Tool Tests diff --git a/tool/pom.xml b/tool/pom.xml index 3067a1939..bda5bbd53 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -3,7 +3,7 @@ org.antlr antlr4-master - 4.5.2 + 4.5.3-SNAPSHOT antlr4 ANTLR 4 Tool From 29a875dccee11f160695d511b55a3adeec3420c6 Mon Sep 17 00:00:00 2001 From: parrt Date: Fri, 29 Jan 2016 15:33:37 -0800 Subject: [PATCH 31/95] tweak release doc --- doc/releasing-antlr.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/releasing-antlr.md b/doc/releasing-antlr.md index 8131f8c9b..dc84a11a0 100644 --- a/doc/releasing-antlr.md +++ b/doc/releasing-antlr.md @@ -89,8 +89,10 @@ The maven deploy lifecycle phased deploys the artifacts and the poms for the ANT mvn deploy -DskipTests ``` +With JDK 1.7 (not 6 or 8), do this: + ```bash -mvn release:prepare +mvn release:prepare -Darguments="-DskipTests" ``` It will start out by asking you the version number: @@ -111,7 +113,7 @@ What is the new development version for "ANTLR 4"? (org.antlr:antlr4-master) 4.5 Maven will go through your pom.xml files to update versions from 4.5.2-SNAPSHOT to 4.5.2 for release and then to 4.5.3-SNAPSHOT after release, which is done with: ```bash -mvn release:perform +mvn release:perform -Darguments="-DskipTests" ``` Maven will use git to push pom.xml changes. (big smile) From c92ddcbd06fd0298084f29026996c857e80c219b Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Sat, 30 Jan 2016 09:54:47 -0800 Subject: [PATCH 32/95] update version to 4.5.2 in source / packaging stuff. --- doc/releasing-antlr.md | 15 +++++---------- .../Antlr4.Runtime/Properties/AssemblyInfo.cs | 6 +++--- .../src/org/antlr/v4/runtime/RuntimeMetaData.java | 2 +- runtime/JavaScript/src/antlr4/Recognizer.js | 2 +- runtime/JavaScript/src/antlr4/package.json | 2 +- runtime/Python2/setup.py | 4 ++-- runtime/Python2/src/antlr4/Recognizer.py | 2 +- runtime/Python3/setup.py | 4 ++-- runtime/Python3/src/antlr4/Recognizer.py | 2 +- .../org/antlr/v4/codegen/target/CSharpTarget.java | 2 +- .../antlr/v4/codegen/target/JavaScriptTarget.java | 2 +- .../antlr/v4/codegen/target/Python2Target.java | 2 +- .../antlr/v4/codegen/target/Python3Target.java | 2 +- 13 files changed, 21 insertions(+), 26 deletions(-) diff --git a/doc/releasing-antlr.md b/doc/releasing-antlr.md index 8131f8c9b..b4293c7c2 100644 --- a/doc/releasing-antlr.md +++ b/doc/releasing-antlr.md @@ -2,13 +2,6 @@ ## Github -Create a release candidate tag 4.x-rc-1 or full 4.5 tag - -```bash -git tag -a 4.5 -m 'ANTLR final release 4.5' -git push origin 4.5 -``` - Create a pre-release or full release at github; [Example 4.5-rc-1](https://github.com/antlr/antlr4/releases/tag/4.5-rc-1). ## Bump version @@ -20,7 +13,7 @@ Edit the repository looking for 4.5 or whatever and update it. Bump version in t * runtime/Python2/src/antlr4/Recognizer.py * runtime/Python3/setup.py * runtime/Python3/src/antlr4/Recognizer.py - * runtime/CSharp/Antlr4.Runtime/Properties/AssemblyInfo.cs + * runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Properties/AssemblyInfo.cs * runtime/JavaScript/src/antlr4/package.json * runtime/JavaScript/src/antlr4/Recognizer.js * tool/src/org/antlr/v4/codegen/target/CSharpTarget.java @@ -89,8 +82,10 @@ The maven deploy lifecycle phased deploys the artifacts and the poms for the ANT mvn deploy -DskipTests ``` +With JDK 1.7 (not 6 or 8), do this: + ```bash -mvn release:prepare +mvn release:prepare -Darguments="-DskipTests" ``` It will start out by asking you the version number: @@ -111,7 +106,7 @@ What is the new development version for "ANTLR 4"? (org.antlr:antlr4-master) 4.5 Maven will go through your pom.xml files to update versions from 4.5.2-SNAPSHOT to 4.5.2 for release and then to 4.5.3-SNAPSHOT after release, which is done with: ```bash -mvn release:perform +mvn release:perform -Darguments="-DskipTests" ``` Maven will use git to push pom.xml changes. (big smile) diff --git a/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Properties/AssemblyInfo.cs b/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Properties/AssemblyInfo.cs index 15edab194..e74711082 100644 --- a/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Properties/AssemblyInfo.cs +++ b/runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Properties/AssemblyInfo.cs @@ -67,8 +67,8 @@ using System.Runtime.InteropServices; // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("4.5.1.0")] +[assembly: AssemblyVersion("4.5.2.0")] #if !COMPACT -[assembly: AssemblyFileVersion("4.5.1.0")] -[assembly: AssemblyInformationalVersion("4.5.1-dev")] +[assembly: AssemblyFileVersion("4.5.2.0")] +[assembly: AssemblyInformationalVersion("4.5.2.0")] #endif diff --git a/runtime/Java/src/org/antlr/v4/runtime/RuntimeMetaData.java b/runtime/Java/src/org/antlr/v4/runtime/RuntimeMetaData.java index bf67cf029..aa1375edb 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/RuntimeMetaData.java +++ b/runtime/Java/src/org/antlr/v4/runtime/RuntimeMetaData.java @@ -91,7 +91,7 @@ public class RuntimeMetaData { * omitted. * */ - public static final String VERSION = "4.5.1"; + public static final String VERSION = "4.5.2"; /** * Gets the currently executing version of the ANTLR 4 runtime library. diff --git a/runtime/JavaScript/src/antlr4/Recognizer.js b/runtime/JavaScript/src/antlr4/Recognizer.js index fecc44140..f7a39ebc8 100644 --- a/runtime/JavaScript/src/antlr4/Recognizer.js +++ b/runtime/JavaScript/src/antlr4/Recognizer.js @@ -45,7 +45,7 @@ Recognizer.ruleIndexMapCache = {}; Recognizer.prototype.checkVersion = function(toolVersion) { - var runtimeVersion = "4.5.1"; + var runtimeVersion = "4.5.2"; if (runtimeVersion!==toolVersion) { console.log("ANTLR runtime and generated code versions disagree: "+runtimeVersion+"!="+toolVersion); } diff --git a/runtime/JavaScript/src/antlr4/package.json b/runtime/JavaScript/src/antlr4/package.json index f192da1c0..c5f4cd608 100644 --- a/runtime/JavaScript/src/antlr4/package.json +++ b/runtime/JavaScript/src/antlr4/package.json @@ -1,6 +1,6 @@ { "name": "antlr4", - "version": "4.5.1", + "version": "4.5.2", "description": "JavaScript runtime for ANTLR4", "main": "src/antlr4/index.js", "repository": "antlr/antlr4.git", diff --git a/runtime/Python2/setup.py b/runtime/Python2/setup.py index 0eeb03c5f..42dbc86b5 100644 --- a/runtime/Python2/setup.py +++ b/runtime/Python2/setup.py @@ -2,12 +2,12 @@ from distutils.core import setup setup( name='antlr4-python2-runtime', - version='4.5.2', + version='4.5.2.1', packages=['antlr4', 'antlr4.atn', 'antlr4.dfa', 'antlr4.tree', 'antlr4.error', 'antlr4.xpath'], package_dir={'': 'src'}, url='http://www.antlr.org', license='BSD', author='Eric Vergnaud, Terence Parr, Sam Harwell', author_email='eric.vergnaud@wanadoo.fr', - description='ANTLR 4.5.2 runtime for Python 2.7.6' + description='ANTLR 4.5.2.1 runtime for Python 2.7.6' ) diff --git a/runtime/Python2/src/antlr4/Recognizer.py b/runtime/Python2/src/antlr4/Recognizer.py index 35388f939..c82760e59 100644 --- a/runtime/Python2/src/antlr4/Recognizer.py +++ b/runtime/Python2/src/antlr4/Recognizer.py @@ -55,7 +55,7 @@ class Recognizer(object): return major, minor def checkVersion(self, toolVersion): - runtimeVersion = "4.5.2" + runtimeVersion = "4.5.2.1" rvmajor, rvminor = self.extractVersion(runtimeVersion) tvmajor, tvminor = self.extractVersion(toolVersion) if rvmajor!=tvmajor or rvminor!=tvminor: diff --git a/runtime/Python3/setup.py b/runtime/Python3/setup.py index d9de10715..65cdd8d27 100644 --- a/runtime/Python3/setup.py +++ b/runtime/Python3/setup.py @@ -2,12 +2,12 @@ from distutils.core import setup setup( name='antlr4-python3-runtime', - version='4.5.2', + version='4.5.2.1', packages=['antlr4', 'antlr4.atn', 'antlr4.dfa', 'antlr4.tree', 'antlr4.error', 'antlr4.xpath'], package_dir={'': 'src'}, url='http://www.antlr.org', license='BSD', author='Eric Vergnaud, Terence Parr, Sam Harwell', author_email='eric.vergnaud@wanadoo.fr', - description='ANTLR 4.5.2 runtime for Python 3.4.0' + description='ANTLR 4.5.2.1 runtime for Python 3.4.0' ) diff --git a/runtime/Python3/src/antlr4/Recognizer.py b/runtime/Python3/src/antlr4/Recognizer.py index 5212cbf9a..a887504f4 100644 --- a/runtime/Python3/src/antlr4/Recognizer.py +++ b/runtime/Python3/src/antlr4/Recognizer.py @@ -58,7 +58,7 @@ class Recognizer(object): return major, minor def checkVersion(self, toolVersion): - runtimeVersion = "4.5.2" + runtimeVersion = "4.5.2.1" rvmajor, rvminor = self.extractVersion(runtimeVersion) tvmajor, tvminor = self.extractVersion(toolVersion) if rvmajor!=tvmajor or rvminor!=tvminor: diff --git a/tool/src/org/antlr/v4/codegen/target/CSharpTarget.java b/tool/src/org/antlr/v4/codegen/target/CSharpTarget.java index 09df962cc..bfebe4b7e 100644 --- a/tool/src/org/antlr/v4/codegen/target/CSharpTarget.java +++ b/tool/src/org/antlr/v4/codegen/target/CSharpTarget.java @@ -51,7 +51,7 @@ public class CSharpTarget extends Target { @Override public String getVersion() { - return "4.5.1"; // crossing fingers that it's close enough. + return "4.5.2"; // crossing fingers that it's close enough. } @Override diff --git a/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java b/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java index 26fa4fe0d..71926acfb 100644 --- a/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java +++ b/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java @@ -74,7 +74,7 @@ public class JavaScriptTarget extends Target { @Override public String getVersion() { - return "4.5.1"; + return "4.5.2"; } public Set getBadWords() { diff --git a/tool/src/org/antlr/v4/codegen/target/Python2Target.java b/tool/src/org/antlr/v4/codegen/target/Python2Target.java index 36249d59a..efa5d2ce9 100644 --- a/tool/src/org/antlr/v4/codegen/target/Python2Target.java +++ b/tool/src/org/antlr/v4/codegen/target/Python2Target.java @@ -117,7 +117,7 @@ public class Python2Target extends Target { @Override public String getVersion() { - return "4.5.2"; + return "4.5.2.1"; } public Set getBadWords() { diff --git a/tool/src/org/antlr/v4/codegen/target/Python3Target.java b/tool/src/org/antlr/v4/codegen/target/Python3Target.java index d07247fc4..68d7a162c 100644 --- a/tool/src/org/antlr/v4/codegen/target/Python3Target.java +++ b/tool/src/org/antlr/v4/codegen/target/Python3Target.java @@ -119,7 +119,7 @@ public class Python3Target extends Target { @Override public String getVersion() { - return "4.5.2"; + return "4.5.2.1"; } /** Avoid grammar symbols in this set to prevent conflicts in gen'd code. */ From a0204f25fdaf19c8225e416463d9099cb5693e65 Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Sat, 30 Jan 2016 09:58:06 -0800 Subject: [PATCH 33/95] update doc --- doc/releasing-antlr.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/releasing-antlr.md b/doc/releasing-antlr.md index b4293c7c2..6df68607c 100644 --- a/doc/releasing-antlr.md +++ b/doc/releasing-antlr.md @@ -4,6 +4,14 @@ Create a pre-release or full release at github; [Example 4.5-rc-1](https://github.com/antlr/antlr4/releases/tag/4.5-rc-1). +Wack any existing tag as mvn will create one and it fails if already there. + +``` +$ git tag -d 4.5.2 +$ git push origin :refs/tags/4.5.2 +$ git push upstream :refs/tags/4.5.2 +``` + ## Bump version Edit the repository looking for 4.5 or whatever and update it. Bump version in the following files: From dcca95d4f0e7d6539b0dff21184e34a09829e46e Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Sat, 30 Jan 2016 10:01:48 -0800 Subject: [PATCH 34/95] [maven-release-plugin] prepare release 4.5.2 --- antlr4-maven-plugin/pom.xml | 2 +- pom.xml | 2 +- runtime-testsuite/pom.xml | 2 +- runtime/Java/pom.xml | 2 +- tool-testsuite/pom.xml | 2 +- tool/pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/antlr4-maven-plugin/pom.xml b/antlr4-maven-plugin/pom.xml index 4495a2a21..c5256288e 100644 --- a/antlr4-maven-plugin/pom.xml +++ b/antlr4-maven-plugin/pom.xml @@ -34,7 +34,7 @@ org.antlr antlr4-master - 4.5.2-SNAPSHOT + 4.5.2-1 antlr4-maven-plugin maven-plugin diff --git a/pom.xml b/pom.xml index 92ca59769..4f642840c 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.antlr antlr4-master - 4.5.2-SNAPSHOT + 4.5.2-1 pom ANTLR 4 diff --git a/runtime-testsuite/pom.xml b/runtime-testsuite/pom.xml index 40e2f2a3a..08cdde437 100644 --- a/runtime-testsuite/pom.xml +++ b/runtime-testsuite/pom.xml @@ -4,7 +4,7 @@ org.antlr antlr4-master - 4.5.2-SNAPSHOT + 4.5.2-1 antlr4-runtime-testsuite ANTLR 4 Runtime Test Generator diff --git a/runtime/Java/pom.xml b/runtime/Java/pom.xml index b9c026525..0a76da0bf 100644 --- a/runtime/Java/pom.xml +++ b/runtime/Java/pom.xml @@ -3,7 +3,7 @@ org.antlr antlr4-master - 4.5.2-SNAPSHOT + 4.5.2-1 ../../pom.xml antlr4-runtime diff --git a/tool-testsuite/pom.xml b/tool-testsuite/pom.xml index dad48a51c..81b452593 100644 --- a/tool-testsuite/pom.xml +++ b/tool-testsuite/pom.xml @@ -4,7 +4,7 @@ org.antlr antlr4-master - 4.5.2-SNAPSHOT + 4.5.2-1 antlr4-tool-testsuite ANTLR 4 Tool Tests diff --git a/tool/pom.xml b/tool/pom.xml index 073f0cbd1..d76294791 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -3,7 +3,7 @@ org.antlr antlr4-master - 4.5.2-SNAPSHOT + 4.5.2-1 antlr4 ANTLR 4 Tool From 184f71175366775ae9471dd6dea9921e1d14e4ea Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Sat, 30 Jan 2016 10:01:53 -0800 Subject: [PATCH 35/95] [maven-release-plugin] prepare for next development iteration --- antlr4-maven-plugin/pom.xml | 2 +- pom.xml | 2 +- runtime-testsuite/pom.xml | 2 +- runtime/Java/pom.xml | 2 +- tool-testsuite/pom.xml | 2 +- tool/pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/antlr4-maven-plugin/pom.xml b/antlr4-maven-plugin/pom.xml index c5256288e..2ebf43d63 100644 --- a/antlr4-maven-plugin/pom.xml +++ b/antlr4-maven-plugin/pom.xml @@ -34,7 +34,7 @@ org.antlr antlr4-master - 4.5.2-1 + 4.5.3-SNAPSHOT antlr4-maven-plugin maven-plugin diff --git a/pom.xml b/pom.xml index 4f642840c..4ff16411c 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.antlr antlr4-master - 4.5.2-1 + 4.5.3-SNAPSHOT pom ANTLR 4 diff --git a/runtime-testsuite/pom.xml b/runtime-testsuite/pom.xml index 08cdde437..f9f105200 100644 --- a/runtime-testsuite/pom.xml +++ b/runtime-testsuite/pom.xml @@ -4,7 +4,7 @@ org.antlr antlr4-master - 4.5.2-1 + 4.5.3-SNAPSHOT antlr4-runtime-testsuite ANTLR 4 Runtime Test Generator diff --git a/runtime/Java/pom.xml b/runtime/Java/pom.xml index 0a76da0bf..c00729bf3 100644 --- a/runtime/Java/pom.xml +++ b/runtime/Java/pom.xml @@ -3,7 +3,7 @@ org.antlr antlr4-master - 4.5.2-1 + 4.5.3-SNAPSHOT ../../pom.xml antlr4-runtime diff --git a/tool-testsuite/pom.xml b/tool-testsuite/pom.xml index 81b452593..cea14b194 100644 --- a/tool-testsuite/pom.xml +++ b/tool-testsuite/pom.xml @@ -4,7 +4,7 @@ org.antlr antlr4-master - 4.5.2-1 + 4.5.3-SNAPSHOT antlr4-tool-testsuite ANTLR 4 Tool Tests diff --git a/tool/pom.xml b/tool/pom.xml index d76294791..bda5bbd53 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -3,7 +3,7 @@ org.antlr antlr4-master - 4.5.2-1 + 4.5.3-SNAPSHOT antlr4 ANTLR 4 Tool From 7a59e92b8d86e3aa29a3897e03551a323c03c959 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Tue, 2 Feb 2016 21:01:28 +0800 Subject: [PATCH 36/95] latest version of honey-require --- runtime/JavaScript/src/lib/require.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/runtime/JavaScript/src/lib/require.js b/runtime/JavaScript/src/lib/require.js index 9a8069c35..d14210d24 100644 --- a/runtime/JavaScript/src/lib/require.js +++ b/runtime/JavaScript/src/lib/require.js @@ -62,7 +62,13 @@ // anchor element as parser in that case. Thes breaks web worker support, // but we don't care since these browsers also don't support web workers. - var parser = URL ? new URL(location.href) : document.createElement('A'); + try { + var parser = new URL(location.href); + } + catch (e) { + console.warn("Honey: falling back to DOM workaround for URL parser ("+e+")"); + parser = document.createElement('A'); + } // INFO Module cache // Contains getter functions for the exports objects of all the loaded @@ -81,7 +87,7 @@ delete cache.foo; } catch (e) { - console.warn("Honey: falling back to DOM workaround for defineProperty ("+e+")"); + console.warn("Honey: falling back to DOM workaround for cache object ("+e+")"); cache = document.createElement('DIV'); } From b0196bc86e1fb41e6f5064b4738dd52900653afc Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Tue, 9 Feb 2016 10:25:31 +0800 Subject: [PATCH 37/95] fixes #1108 --- runtime/JavaScript/src/antlr4/atn/ParserATNSimulator.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runtime/JavaScript/src/antlr4/atn/ParserATNSimulator.js b/runtime/JavaScript/src/antlr4/atn/ParserATNSimulator.js index 9a262949f..b740ed92a 100644 --- a/runtime/JavaScript/src/antlr4/atn/ParserATNSimulator.js +++ b/runtime/JavaScript/src/antlr4/atn/ParserATNSimulator.js @@ -1258,10 +1258,10 @@ ParserATNSimulator.prototype.closureCheckingStopState = function(config, configs } continue; } - returnState = this.atn.states[config.context.getReturnState(i)]; - newContext = config.context.getParent(i); // "pop" return state + var returnState = this.atn.states[config.context.getReturnState(i)]; + var newContext = config.context.getParent(i); // "pop" return state var parms = {state:returnState, alt:config.alt, context:newContext, semanticContext:config.semanticContext}; - c = new ATNConfig(parms, null); + var c = new ATNConfig(parms, null); // While we have context to pop back from, we may have // gotten that context AFTER having falling off a rule. // Make sure we track that we are now out of context. From ae8ea836fc8ca02a0bd6f6811f73f9f043d043b7 Mon Sep 17 00:00:00 2001 From: parrt Date: Tue, 9 Feb 2016 11:47:54 -0800 Subject: [PATCH 38/95] reopen PR #1081 --- runtime/JavaScript/src/antlr4/tree/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/JavaScript/src/antlr4/tree/index.js b/runtime/JavaScript/src/antlr4/tree/index.js index ce8844c57..e6b760ed0 100644 --- a/runtime/JavaScript/src/antlr4/tree/index.js +++ b/runtime/JavaScript/src/antlr4/tree/index.js @@ -1,6 +1,6 @@ var Tree = require('./Tree'); -exports.Trees = require('./Tree').Trees; +exports.Trees = require('./Trees').Trees; exports.RuleNode = Tree.RuleNode; exports.ParseTreeListener = Tree.ParseTreeListener; exports.ParseTreeVisitor = Tree.ParseTreeVisitor; -exports.ParseTreeWalker = Tree.ParseTreeWalker; \ No newline at end of file +exports.ParseTreeWalker = Tree.ParseTreeWalker; From 1fa0be9782eea7e780e3f0500f50a679d8cad2a0 Mon Sep 17 00:00:00 2001 From: Raphael R Date: Thu, 18 Feb 2016 11:51:59 +0100 Subject: [PATCH 39/95] Fixes Markdown table in targets.md --- doc/targets.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/targets.md b/doc/targets.md index 00058a4af..e9478aed8 100644 --- a/doc/targets.md +++ b/doc/targets.md @@ -18,7 +18,7 @@ The [ANTLR v4 book](http://pragprog.com/book/tpantlr2/the-definitive-antlr-4-ref New features generally appear in the Java target and then migrate to the other targets, but these other targets don't always get updated in the same overall tool release. This section tries to identify features added to Java that have not been added to the other targets. |Feature|Java|C♯|JavaScript|Python2|Python3|Swift|C++| -|-|-|-|-|-|-|-|-| +|---|---|---|---|---|---|---|---| |Ambiguous tree construction|4.5.1|-|-|-|-|-|-| From 99b2d6616ff03d3b0e4751880ff652dd6641c586 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Thu, 18 Feb 2016 23:39:51 +0800 Subject: [PATCH 40/95] added missing del keyword to Python targets --- tool/src/org/antlr/v4/codegen/target/Python2Target.java | 2 +- tool/src/org/antlr/v4/codegen/target/Python3Target.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tool/src/org/antlr/v4/codegen/target/Python2Target.java b/tool/src/org/antlr/v4/codegen/target/Python2Target.java index efa5d2ce9..af3f12587 100644 --- a/tool/src/org/antlr/v4/codegen/target/Python2Target.java +++ b/tool/src/org/antlr/v4/codegen/target/Python2Target.java @@ -50,7 +50,7 @@ public class Python2Target extends Target { "abs", "all", "any", "apply", "as", "bin", "bool", "buffer", "bytearray", "callable", "chr", "classmethod", "coerce", "compile", "complex", - "delattr", "dict", "dir", "divmod", + "del", "delattr", "dict", "dir", "divmod", "enumerate", "eval", "execfile", "file", "filter", "float", "format", "frozenset", "getattr", "globals", diff --git a/tool/src/org/antlr/v4/codegen/target/Python3Target.java b/tool/src/org/antlr/v4/codegen/target/Python3Target.java index 68d7a162c..32ccd6fc7 100644 --- a/tool/src/org/antlr/v4/codegen/target/Python3Target.java +++ b/tool/src/org/antlr/v4/codegen/target/Python3Target.java @@ -50,7 +50,7 @@ public class Python3Target extends Target { "abs", "all", "any", "apply", "as", "bin", "bool", "buffer", "bytearray", "callable", "chr", "classmethod", "coerce", "compile", "complex", - "delattr", "dict", "dir", "divmod", + "del", "delattr", "dict", "dir", "divmod", "enumerate", "eval", "execfile", "file", "filter", "float", "format", "frozenset", "getattr", "globals", From b1b257962147c71fc9a5b5cd336105803ffa4175 Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Mon, 22 Feb 2016 12:48:48 +0000 Subject: [PATCH 41/95] Add tests for alternative precedence in left-recursive transformation --- .../runtime/templates/LeftRecursion/Index.stg | 2 + .../LeftRecursion/PrefixAndOtherAlt.stg | 24 +++++++++ .../LeftRecursion/PrefixAndOtherAlt_1.stg | 9 ++++ .../LeftRecursion/PrefixAndOtherAlt_2.stg | 9 ++++ .../runtime/csharp/TestLeftRecursion.java | 44 +++++++++++++++ .../test/runtime/java/TestLeftRecursion.java | 52 ++++++++++++++++++ .../javascript/node/TestLeftRecursion.java | 48 +++++++++++++++++ .../runtime/python2/TestLeftRecursion.java | 54 +++++++++++++++++++ .../runtime/python3/TestLeftRecursion.java | 54 +++++++++++++++++++ 9 files changed, 296 insertions(+) create mode 100644 runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/PrefixAndOtherAlt.stg create mode 100644 runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/PrefixAndOtherAlt_1.stg create mode 100644 runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/PrefixAndOtherAlt_2.stg diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/Index.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/Index.stg index 5d2166e71..617914d9d 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/Index.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/Index.stg @@ -59,6 +59,8 @@ TestTemplates ::= [ "MultipleAlternativesWithCommonLabel_2": [], "MultipleAlternativesWithCommonLabel_3": [], "MultipleAlternativesWithCommonLabel_4": [], + "PrefixAndOtherAlt_1": [], + "PrefixAndOtherAlt_2": [], "PrefixOpWithActionAndLabel_1": [], "PrefixOpWithActionAndLabel_2": [], "PrefixOpWithActionAndLabel_3": [], diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/PrefixAndOtherAlt.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/PrefixAndOtherAlt.stg new file mode 100644 index 000000000..37d022826 --- /dev/null +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/PrefixAndOtherAlt.stg @@ -0,0 +1,24 @@ +TestType() ::= "Parser" + +Options ::= [ + "Debug": false +] + +Grammar ::= [ + "T": {} +] + +Rule() ::= "s" + +grammar(grammarName) ::= << +grammar ; +s @after {} : expr EOF ; +expr : literal + | op expr + | expr op expr + ; +literal : '-'? Integer ; +op : '+' | '-' ; +Integer : [0-9]+ ; +WS : (' '|'\n') -> skip ; +>> \ No newline at end of file diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/PrefixAndOtherAlt_1.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/PrefixAndOtherAlt_1.stg new file mode 100644 index 000000000..553b557f0 --- /dev/null +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/PrefixAndOtherAlt_1.stg @@ -0,0 +1,9 @@ +import "PrefixAndOtherAlt.stg" + +Input() ::= "-1" + +Output() ::= << +(s (expr (literal - 1)) \)<\n> +>> + +Errors() ::= "" \ No newline at end of file diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/PrefixAndOtherAlt_2.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/PrefixAndOtherAlt_2.stg new file mode 100644 index 000000000..de4cea9c5 --- /dev/null +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/PrefixAndOtherAlt_2.stg @@ -0,0 +1,9 @@ +import "PrefixAndOtherAlt.stg" + +Input() ::= "-1 + -1" + +Output() ::= << +(s (expr (expr (literal - 1)) (op +) (expr (literal - 1))) \)<\n> +>> + +Errors() ::= "" \ No newline at end of file diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/TestLeftRecursion.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/TestLeftRecursion.java index 595123e38..b3b91fdaf 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/TestLeftRecursion.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/TestLeftRecursion.java @@ -1801,6 +1801,50 @@ public class TestLeftRecursion extends BaseTest { assertEquals("(prog (statement (letterA a)) (statement (letterA a)) )\n", found); assertNull(this.stderrDuringParse); + } + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ + @Test + public void testPrefixAndOtherAlt_1() throws Exception { + mkdir(tmpdir); + StringBuilder grammarBuilder = new StringBuilder(223); + grammarBuilder.append("grammar T;\n"); + grammarBuilder.append("s @after {Console.WriteLine($ctx.ToStringTree(this));} : expr EOF ; \n"); + grammarBuilder.append("expr : literal\n"); + grammarBuilder.append(" | op expr\n"); + grammarBuilder.append(" | expr op expr\n"); + grammarBuilder.append(" ;\n"); + grammarBuilder.append("literal : '-'? Integer ;\n"); + grammarBuilder.append("op : '+' | '-' ;\n"); + grammarBuilder.append("Integer : [0-9]+ ;\n"); + grammarBuilder.append("WS : (' '|'\\n') -> skip ;"); + String grammar = grammarBuilder.toString(); + String input ="-1"; + String found = execParser("T.g4", grammar, "TParser", "TLexer", "s", input, false); + assertEquals("(s (expr (literal - 1)) )\n", found); + assertNull(this.stderrDuringParse); + + } + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ + @Test + public void testPrefixAndOtherAlt_2() throws Exception { + mkdir(tmpdir); + StringBuilder grammarBuilder = new StringBuilder(223); + grammarBuilder.append("grammar T;\n"); + grammarBuilder.append("s @after {Console.WriteLine($ctx.ToStringTree(this));} : expr EOF ; \n"); + grammarBuilder.append("expr : literal\n"); + grammarBuilder.append(" | op expr\n"); + grammarBuilder.append(" | expr op expr\n"); + grammarBuilder.append(" ;\n"); + grammarBuilder.append("literal : '-'? Integer ;\n"); + grammarBuilder.append("op : '+' | '-' ;\n"); + grammarBuilder.append("Integer : [0-9]+ ;\n"); + grammarBuilder.append("WS : (' '|'\\n') -> skip ;"); + String grammar = grammarBuilder.toString(); + String input ="-1 + -1"; + String found = execParser("T.g4", grammar, "TParser", "TLexer", "s", input, false); + assertEquals("(s (expr (expr (literal - 1)) (op +) (expr (literal - 1))) )\n", found); + assertNull(this.stderrDuringParse); + } /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ @Test diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/java/TestLeftRecursion.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/java/TestLeftRecursion.java index 6606f5ba0..2f72610f3 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/java/TestLeftRecursion.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/java/TestLeftRecursion.java @@ -2007,6 +2007,58 @@ public class TestLeftRecursion extends BaseTest { } + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ + @Test + public void testPrefixAndOtherAlt_1() throws Exception { + mkdir(tmpdir); + + StringBuilder grammarBuilder = new StringBuilder(224); + grammarBuilder.append("grammar T;\n"); + grammarBuilder.append("s @after {System.out.println($ctx.toStringTree(this));} : expr EOF ; \n"); + grammarBuilder.append("expr : literal\n"); + grammarBuilder.append(" | op expr\n"); + grammarBuilder.append(" | expr op expr\n"); + grammarBuilder.append(" ;\n"); + grammarBuilder.append("literal : '-'? Integer ;\n"); + grammarBuilder.append("op : '+' | '-' ;\n"); + grammarBuilder.append("Integer : [0-9]+ ;\n"); + grammarBuilder.append("WS : (' '|'\\n') -> skip ;"); + String grammar = grammarBuilder.toString(); + + + String input ="-1"; + String found = execParser("T.g4", grammar, "TParser", "TLexer", "s", input, false); + assertEquals("(s (expr (literal - 1)) )\n", found); + assertNull(this.stderrDuringParse); + + } + + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ + @Test + public void testPrefixAndOtherAlt_2() throws Exception { + mkdir(tmpdir); + + StringBuilder grammarBuilder = new StringBuilder(224); + grammarBuilder.append("grammar T;\n"); + grammarBuilder.append("s @after {System.out.println($ctx.toStringTree(this));} : expr EOF ; \n"); + grammarBuilder.append("expr : literal\n"); + grammarBuilder.append(" | op expr\n"); + grammarBuilder.append(" | expr op expr\n"); + grammarBuilder.append(" ;\n"); + grammarBuilder.append("literal : '-'? Integer ;\n"); + grammarBuilder.append("op : '+' | '-' ;\n"); + grammarBuilder.append("Integer : [0-9]+ ;\n"); + grammarBuilder.append("WS : (' '|'\\n') -> skip ;"); + String grammar = grammarBuilder.toString(); + + + String input ="-1 + -1"; + String found = execParser("T.g4", grammar, "TParser", "TLexer", "s", input, false); + assertEquals("(s (expr (expr (literal - 1)) (op +) (expr (literal - 1))) )\n", found); + assertNull(this.stderrDuringParse); + + } + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ @Test public void testPrefixOpWithActionAndLabel_1() throws Exception { diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/node/TestLeftRecursion.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/node/TestLeftRecursion.java index 53fb15efb..2b392fc44 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/node/TestLeftRecursion.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/node/TestLeftRecursion.java @@ -1905,6 +1905,54 @@ public class TestLeftRecursion extends BaseTest { assertEquals("(prog (statement (letterA a)) (statement (letterA a)) )\n", found); assertNull(this.stderrDuringParse); + } + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ + @Test + public void testPrefixAndOtherAlt_1() throws Exception { + mkdir(tmpdir); + StringBuilder grammarBuilder = new StringBuilder(223); + grammarBuilder.append("grammar T;\n"); + grammarBuilder.append("s @after {console.log($ctx.toStringTree(null, this));} : expr EOF ; \n"); + grammarBuilder.append("expr : literal\n"); + grammarBuilder.append(" | op expr\n"); + grammarBuilder.append(" | expr op expr\n"); + grammarBuilder.append(" ;\n"); + grammarBuilder.append("literal : '-'? Integer ;\n"); + grammarBuilder.append("op : '+' | '-' ;\n"); + grammarBuilder.append("Integer : [0-9]+ ;\n"); + grammarBuilder.append("WS : (' '|'\\n') -> skip ;"); + String grammar = grammarBuilder.toString(); + String input ="-1"; + String found = execParser("T.g4", grammar, "TParser", "TLexer", + "TListener", "TVisitor", + "s", input, false); + assertEquals("(s (expr (literal - 1)) )\n", found); + assertNull(this.stderrDuringParse); + + } + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ + @Test + public void testPrefixAndOtherAlt_2() throws Exception { + mkdir(tmpdir); + StringBuilder grammarBuilder = new StringBuilder(223); + grammarBuilder.append("grammar T;\n"); + grammarBuilder.append("s @after {console.log($ctx.toStringTree(null, this));} : expr EOF ; \n"); + grammarBuilder.append("expr : literal\n"); + grammarBuilder.append(" | op expr\n"); + grammarBuilder.append(" | expr op expr\n"); + grammarBuilder.append(" ;\n"); + grammarBuilder.append("literal : '-'? Integer ;\n"); + grammarBuilder.append("op : '+' | '-' ;\n"); + grammarBuilder.append("Integer : [0-9]+ ;\n"); + grammarBuilder.append("WS : (' '|'\\n') -> skip ;"); + String grammar = grammarBuilder.toString(); + String input ="-1 + -1"; + String found = execParser("T.g4", grammar, "TParser", "TLexer", + "TListener", "TVisitor", + "s", input, false); + assertEquals("(s (expr (expr (literal - 1)) (op +) (expr (literal - 1))) )\n", found); + assertNull(this.stderrDuringParse); + } /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ @Test diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/python2/TestLeftRecursion.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/python2/TestLeftRecursion.java index 8e07e462a..d65523920 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/python2/TestLeftRecursion.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/python2/TestLeftRecursion.java @@ -2058,6 +2058,60 @@ public class TestLeftRecursion extends BasePython2Test { } + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ + @Test + public void testPrefixAndOtherAlt_1() throws Exception { + mkdir(tmpdir); + + StringBuilder grammarBuilder = new StringBuilder(216); + grammarBuilder.append("grammar T;\n"); + grammarBuilder.append("s @after {print($ctx.toStringTree(recog=self))} : expr EOF ; \n"); + grammarBuilder.append("expr : literal\n"); + grammarBuilder.append(" | op expr\n"); + grammarBuilder.append(" | expr op expr\n"); + grammarBuilder.append(" ;\n"); + grammarBuilder.append("literal : '-'? Integer ;\n"); + grammarBuilder.append("op : '+' | '-' ;\n"); + grammarBuilder.append("Integer : [0-9]+ ;\n"); + grammarBuilder.append("WS : (' '|'\\n') -> skip ;"); + String grammar = grammarBuilder.toString(); + + + String input ="-1"; + String found = execParser("T.g4", grammar, "TParser", "TLexer", "TListener", "TVisitor", "s", input, false); + + assertEquals("(s (expr (literal - 1)) )\n", found); + assertNull(this.stderrDuringParse); + + } + + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ + @Test + public void testPrefixAndOtherAlt_2() throws Exception { + mkdir(tmpdir); + + StringBuilder grammarBuilder = new StringBuilder(216); + grammarBuilder.append("grammar T;\n"); + grammarBuilder.append("s @after {print($ctx.toStringTree(recog=self))} : expr EOF ; \n"); + grammarBuilder.append("expr : literal\n"); + grammarBuilder.append(" | op expr\n"); + grammarBuilder.append(" | expr op expr\n"); + grammarBuilder.append(" ;\n"); + grammarBuilder.append("literal : '-'? Integer ;\n"); + grammarBuilder.append("op : '+' | '-' ;\n"); + grammarBuilder.append("Integer : [0-9]+ ;\n"); + grammarBuilder.append("WS : (' '|'\\n') -> skip ;"); + String grammar = grammarBuilder.toString(); + + + String input ="-1 + -1"; + String found = execParser("T.g4", grammar, "TParser", "TLexer", "TListener", "TVisitor", "s", input, false); + + assertEquals("(s (expr (expr (literal - 1)) (op +) (expr (literal - 1))) )\n", found); + assertNull(this.stderrDuringParse); + + } + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ @Test public void testPrefixOpWithActionAndLabel_1() throws Exception { diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/python3/TestLeftRecursion.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/python3/TestLeftRecursion.java index 3b856ebe9..72e8341a7 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/python3/TestLeftRecursion.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/python3/TestLeftRecursion.java @@ -2058,6 +2058,60 @@ public class TestLeftRecursion extends BasePython3Test { } + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ + @Test + public void testPrefixAndOtherAlt_1() throws Exception { + mkdir(tmpdir); + + StringBuilder grammarBuilder = new StringBuilder(216); + grammarBuilder.append("grammar T;\n"); + grammarBuilder.append("s @after {print($ctx.toStringTree(recog=self))} : expr EOF ; \n"); + grammarBuilder.append("expr : literal\n"); + grammarBuilder.append(" | op expr\n"); + grammarBuilder.append(" | expr op expr\n"); + grammarBuilder.append(" ;\n"); + grammarBuilder.append("literal : '-'? Integer ;\n"); + grammarBuilder.append("op : '+' | '-' ;\n"); + grammarBuilder.append("Integer : [0-9]+ ;\n"); + grammarBuilder.append("WS : (' '|'\\n') -> skip ;"); + String grammar = grammarBuilder.toString(); + + + String input ="-1"; + String found = execParser("T.g4", grammar, "TParser", "TLexer", "TListener", "TVisitor", "s", input, false); + + assertEquals("(s (expr (literal - 1)) )\n", found); + assertNull(this.stderrDuringParse); + + } + + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ + @Test + public void testPrefixAndOtherAlt_2() throws Exception { + mkdir(tmpdir); + + StringBuilder grammarBuilder = new StringBuilder(216); + grammarBuilder.append("grammar T;\n"); + grammarBuilder.append("s @after {print($ctx.toStringTree(recog=self))} : expr EOF ; \n"); + grammarBuilder.append("expr : literal\n"); + grammarBuilder.append(" | op expr\n"); + grammarBuilder.append(" | expr op expr\n"); + grammarBuilder.append(" ;\n"); + grammarBuilder.append("literal : '-'? Integer ;\n"); + grammarBuilder.append("op : '+' | '-' ;\n"); + grammarBuilder.append("Integer : [0-9]+ ;\n"); + grammarBuilder.append("WS : (' '|'\\n') -> skip ;"); + String grammar = grammarBuilder.toString(); + + + String input ="-1 + -1"; + String found = execParser("T.g4", grammar, "TParser", "TLexer", "TListener", "TVisitor", "s", input, false); + + assertEquals("(s (expr (expr (literal - 1)) (op +) (expr (literal - 1))) )\n", found); + assertNull(this.stderrDuringParse); + + } + /* This file and method are generated by TestGenerator, any edits will be overwritten by the next generation. */ @Test public void testPrefixOpWithActionAndLabel_1() throws Exception { From 59676745319b864fe3b8fd61710e730e760f5afd Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Mon, 22 Feb 2016 11:55:34 +0000 Subject: [PATCH 42/95] Track 'prefix' and 'other' alternatives together These alternatives are not treated differently after they are created, and we want to retain their alternative ordering. --- .../analysis/LeftRecursiveRuleAnalyzer.java | 19 +++++++++---------- .../LeftRecursiveRuleTransformer.java | 1 - 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAnalyzer.java b/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAnalyzer.java index 5b6ba058f..0bf3708fb 100644 --- a/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAnalyzer.java +++ b/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleAnalyzer.java @@ -30,6 +30,12 @@ package org.antlr.v4.analysis; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + import org.antlr.runtime.CommonToken; import org.antlr.runtime.Token; import org.antlr.runtime.TokenStream; @@ -51,12 +57,6 @@ import org.stringtemplate.v4.ST; import org.stringtemplate.v4.STGroup; import org.stringtemplate.v4.STGroupFile; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - /** Using a tree walker on the rules, determine if a rule is directly left-recursive and if it follows * our pattern. */ @@ -69,7 +69,6 @@ public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker { public LinkedHashMap ternaryAlts = new LinkedHashMap(); public LinkedHashMap suffixAlts = new LinkedHashMap(); public List prefixAlts = new ArrayList(); - public List otherAlts = new ArrayList(); /** Pointer to ID node of ^(= ID element) */ public List> leftRecursiveRuleRefLabels = @@ -222,7 +221,9 @@ public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker { String altLabel = altTree.altLabel!=null ? altTree.altLabel.getText() : null; LeftRecursiveRuleAltInfo a = new LeftRecursiveRuleAltInfo(alt, altText, null, altLabel, false, originalAltTree); - otherAlts.add(a); + // We keep other alts with prefix alts since they are all added to the start of the generated rule, and + // we want to retain any prior ordering between them + prefixAlts.add(a); // System.out.println("otherAlt " + alt + ": " + altText); } @@ -255,7 +256,6 @@ public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker { } ruleST.add("primaryAlts", prefixAlts); - ruleST.add("primaryAlts", otherAlts); tool.log("left-recursion", ruleST.render()); @@ -440,7 +440,6 @@ public class LeftRecursiveRuleAnalyzer extends LeftRecursiveRuleWalker { ", ternaryAlts=" + ternaryAlts + ", suffixAlts=" + suffixAlts + ", prefixAlts=" + prefixAlts + - ", otherAlts=" + otherAlts + '}'; } } diff --git a/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleTransformer.java b/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleTransformer.java index ea67493f9..0df860807 100644 --- a/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleTransformer.java +++ b/tool/src/org/antlr/v4/analysis/LeftRecursiveRuleTransformer.java @@ -169,7 +169,6 @@ public class LeftRecursiveRuleTransformer { // track recursive alt info for codegen r.recPrimaryAlts = new ArrayList(); r.recPrimaryAlts.addAll(leftRecursiveRuleWalker.prefixAlts); - r.recPrimaryAlts.addAll(leftRecursiveRuleWalker.otherAlts); if (r.recPrimaryAlts.isEmpty()) { tool.errMgr.grammarError(ErrorType.NO_NON_LR_ALTS, g.fileName, ((GrammarAST)r.ast.getChild(0)).getToken(), r.name); } From eef470a707cb0ba19f9454efb1c6303a9f9836d0 Mon Sep 17 00:00:00 2001 From: Martin von Gagern Date: Mon, 7 Mar 2016 12:14:37 +0100 Subject: [PATCH 43/95] Format feature parity table as markdown table --- doc/targets.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/targets.md b/doc/targets.md index 00058a4af..170431d93 100644 --- a/doc/targets.md +++ b/doc/targets.md @@ -18,7 +18,6 @@ The [ANTLR v4 book](http://pragprog.com/book/tpantlr2/the-definitive-antlr-4-ref New features generally appear in the Java target and then migrate to the other targets, but these other targets don't always get updated in the same overall tool release. This section tries to identify features added to Java that have not been added to the other targets. |Feature|Java|C♯|JavaScript|Python2|Python3|Swift|C++| -|-|-|-|-|-|-|-|-| +|---|---|---|---|---|---|---|---| |Ambiguous tree construction|4.5.1|-|-|-|-|-|-| - - + From e3e3f2bca08b8cc8428884244f793d91351f4f45 Mon Sep 17 00:00:00 2001 From: Martin von Gagern Date: Mon, 7 Mar 2016 15:03:32 +0100 Subject: [PATCH 44/95] Set stop token in postamble This ensures that the stop token has been set before the @after action gets executed. --- .../v4/codegen/DefaultOutputModelFactory.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tool/src/org/antlr/v4/codegen/DefaultOutputModelFactory.java b/tool/src/org/antlr/v4/codegen/DefaultOutputModelFactory.java index 901e439c2..03841f6e8 100644 --- a/tool/src/org/antlr/v4/codegen/DefaultOutputModelFactory.java +++ b/tool/src/org/antlr/v4/codegen/DefaultOutputModelFactory.java @@ -30,6 +30,7 @@ package org.antlr.v4.codegen; +import org.antlr.v4.codegen.model.Action; import org.antlr.v4.codegen.model.CodeBlockForOuterMostAlt; import org.antlr.v4.codegen.model.OutputModelObject; import org.antlr.v4.codegen.model.RuleFunction; @@ -38,6 +39,9 @@ import org.antlr.v4.codegen.model.decl.CodeBlock; import org.antlr.v4.codegen.model.decl.Decl; import org.antlr.v4.tool.Alternative; import org.antlr.v4.tool.Grammar; +import org.antlr.v4.tool.Rule; +import org.stringtemplate.v4.ST; +import org.stringtemplate.v4.STGroup; import java.util.ArrayList; import java.util.Arrays; @@ -73,6 +77,19 @@ public abstract class DefaultOutputModelFactory extends BlankOutputModelFactory return controller; } + @Override + public List rulePostamble(RuleFunction function, Rule r) { + // See OutputModelController.buildLeftRecursiveRuleFunction + // and Parser.exitRule for other places which set stop. + CodeGenerator gen = getGenerator(); + STGroup codegenTemplates = gen.getTemplates(); + ST setStopTokenAST = codegenTemplates.getInstanceOf("recRuleSetStopToken"); + Action setStopTokenAction = new Action(this, function.ruleCtx, setStopTokenAST); + List ops = new ArrayList(1); + ops.add(setStopTokenAction); + return ops; + } + // Convenience methods From 0262aac8f6ef2a65efd724448c9adae76c6d4ade Mon Sep 17 00:00:00 2001 From: David Tymon Date: Tue, 15 Mar 2016 10:55:17 +1100 Subject: [PATCH 45/95] JS: LexerActionExecutor equals() should do deep comparison of Actions array - according to the corresponding Java implementation, the equals() method for LexerActionExecutor should be doing an equals() test on each of the actions rather than testing the Array reference is equal. --- .../src/antlr4/atn/LexerActionExecutor.js | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/runtime/JavaScript/src/antlr4/atn/LexerActionExecutor.js b/runtime/JavaScript/src/antlr4/atn/LexerActionExecutor.js index ff020a991..0ccea5815 100644 --- a/runtime/JavaScript/src/antlr4/atn/LexerActionExecutor.js +++ b/runtime/JavaScript/src/antlr4/atn/LexerActionExecutor.js @@ -166,9 +166,18 @@ LexerActionExecutor.prototype.equals = function(other) { return true; } else if (!(other instanceof LexerActionExecutor)) { return false; + } else if (this._hashString != other._hashString) { + return false; + } else if (this.lexerActions.length != other.lexerActions.length) { + return false; } else { - return this._hashString === other._hashString && - this.lexerActions === other.lexerActions; + var numActions = this.lexerActions.length + for (var idx = 0; idx < numActions; ++idx) { + if (!this.lexerActions[idx].equals(other.lexerActions[idx])) { + return false; + } + } + return true; } }; From 0dbb12ad1ac8e9b453121ba2796ed91df5fae001 Mon Sep 17 00:00:00 2001 From: David Tymon Date: Tue, 15 Mar 2016 11:00:36 +1100 Subject: [PATCH 46/95] JS: Fix copy-and-paste error in OR SemanticContext toString() method - the OR toString() method was incorrectly added to the AND prototype --- runtime/JavaScript/src/antlr4/atn/SemanticContext.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/JavaScript/src/antlr4/atn/SemanticContext.js b/runtime/JavaScript/src/antlr4/atn/SemanticContext.js index 2fc4b1224..4052086f9 100644 --- a/runtime/JavaScript/src/antlr4/atn/SemanticContext.js +++ b/runtime/JavaScript/src/antlr4/atn/SemanticContext.js @@ -408,7 +408,7 @@ OR.prototype.evalPrecedence = function(parser, outerContext) { return result; }; -AND.prototype.toString = function() { +OR.prototype.toString = function() { var s = ""; this.opnds.map(function(o) { s += "|| " + o.toString(); From 5e176e405f854ed29cbd5c5fa1628f8d821d89d1 Mon Sep 17 00:00:00 2001 From: David Tymon Date: Tue, 15 Mar 2016 13:32:48 +1100 Subject: [PATCH 47/95] JS: SemanticPredicate should be SemanticContext AND.evalPrecedence() method is attempting to invoke a method on a non-existent class "SemanticPredicate". It should be using "SemanticContext" instead. --- runtime/JavaScript/src/antlr4/atn/SemanticContext.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/JavaScript/src/antlr4/atn/SemanticContext.js b/runtime/JavaScript/src/antlr4/atn/SemanticContext.js index 2fc4b1224..cc5cf79ac 100644 --- a/runtime/JavaScript/src/antlr4/atn/SemanticContext.js +++ b/runtime/JavaScript/src/antlr4/atn/SemanticContext.js @@ -301,7 +301,7 @@ AND.prototype.evalPrecedence = function(parser, outerContext) { } var result = null; operands.map(function(o) { - result = result === null ? o : SemanticPredicate.andContext(result, o); + result = result === null ? o : SemanticContext.andContext(result, o); }); return result; }; From eb5af71a301fb6a388296cb2a00ceec82cf0429b Mon Sep 17 00:00:00 2001 From: David Tymon Date: Tue, 15 Mar 2016 13:38:55 +1100 Subject: [PATCH 48/95] JS: ATNConfig can incorrectly change "alt" of 0 to null When an ATNConfig instance is created with an alt of 0, it is possible for it to be incorrectly changed to null due to its "falsey" nature. --- runtime/JavaScript/src/antlr4/atn/ATNConfig.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/JavaScript/src/antlr4/atn/ATNConfig.js b/runtime/JavaScript/src/antlr4/atn/ATNConfig.js index 0b13737ed..988729dc7 100644 --- a/runtime/JavaScript/src/antlr4/atn/ATNConfig.js +++ b/runtime/JavaScript/src/antlr4/atn/ATNConfig.js @@ -50,7 +50,7 @@ function checkParams(params, isCfg) { } else { var props = {}; props.state = params.state || null; - props.alt = params.alt || null; + props.alt = (params.alt === undefined) ? null : params.alt; props.context = params.context || null; props.semanticContext = params.semanticContext || null; if(isCfg) { From 93b1fa53b450f9f55f6ef8191cc566a1037e69a5 Mon Sep 17 00:00:00 2001 From: Justin Szaday Date: Fri, 25 Mar 2016 22:16:54 -0500 Subject: [PATCH 49/95] Fixing Issue #1148, Python Target Generates Invalid Code With Rule Named 'Return' --- tool/src/org/antlr/v4/codegen/target/Python2Target.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tool/src/org/antlr/v4/codegen/target/Python2Target.java b/tool/src/org/antlr/v4/codegen/target/Python2Target.java index efa5d2ce9..51ee8689c 100644 --- a/tool/src/org/antlr/v4/codegen/target/Python2Target.java +++ b/tool/src/org/antlr/v4/codegen/target/Python2Target.java @@ -61,7 +61,7 @@ public class Python2Target extends Target { "memoryview", "object", "oct", "open", "ord", "pow", "print", "property", - "range", "raw_input", "reduce", "reload", "repr", "reversed", "round", + "range", "raw_input", "reduce", "reload", "repr", "return", "reversed", "round", "set", "setattr", "slice", "sorted", "staticmethod", "str", "sum", "super", "tuple", "type", "unichr", "unicode", From ba64a1e001096b0b37bb454e2204b477bc365b2f Mon Sep 17 00:00:00 2001 From: beardlybread Date: Sun, 27 Mar 2016 20:04:54 -0400 Subject: [PATCH 50/95] documentation typos --- contributors.txt | 1 + doc/actions.md | 2 +- doc/grammars.md | 2 +- doc/left-recursion.md | 2 +- doc/lexer-rules.md | 5 +++-- doc/options.md | 4 ++-- doc/predicates.md | 4 ++-- doc/tree-matching.md | 2 +- 8 files changed, 12 insertions(+), 10 deletions(-) diff --git a/contributors.txt b/contributors.txt index 7d3b44a41..fb6040c78 100644 --- a/contributors.txt +++ b/contributors.txt @@ -88,3 +88,4 @@ YYYY/MM/DD, github id, Full name, email 2015/12/17, sebadur, Sebastian Badur, sebadur@users.noreply.github.com 2015/12/23, pboyer, Peter Boyer, peter.b.boyer@gmail.com 2015/12/24, dtymon, David Tymon, david.tymon@gmail.com +2016/03/27, beardlybread, Bradley Steinbacher, bradley.j.steinbacher@gmail.com diff --git a/doc/actions.md b/doc/actions.md index 4ca01f9ec..91b6de1e4 100644 --- a/doc/actions.md +++ b/doc/actions.md @@ -2,7 +2,7 @@ In Chapter 10, Attributes and Actions, we learned how to embed actions within grammars and looked at the most common token and rule attributes. This section summarizes the important syntax and semantics from that chapter and provides a complete list of all available attributes. (You can learn more about actions in the grammar from the free excerpt on listeners and actions.) -Actions are blocks of text written in the target language and enclosed in curly braces. The recognizer triggers them according to their locations within the grammar. For example, the following rule emits found a decl after the parser has seen a valid declaration: +Actions are blocks of text written in the target language and enclosed in curly braces. The recognizer triggers them according to their locations within the grammar. For example, the following rule emits "found a decl" after the parser has seen a valid declaration: ``` decl: type ID ';' {System.out.println("found a decl");} ; diff --git a/doc/grammars.md b/doc/grammars.md index 3a8e77b60..c40d974b6 100644 --- a/doc/grammars.md +++ b/doc/grammars.md @@ -83,7 +83,7 @@ $ grun MyELang stat If there were any `tokens` specifications, the main grammar would merge the token sets. Any named actions such as `@members` would be merged. In general, you should avoid named actions and actions within rules in imported grammars since that limits their reuse. ANTLR also ignores any options in imported grammars. -Imported grammars can also import other grammars. ANTLR pursues all imported grammars in a depth-first fashion. If two or more imported grammars define ruler, ANTLR chooses the first version of `r` it finds. In the following diagram, ANTLR examines grammars in the following order `Nested`, `G1`, `G3`, `G2`. +Imported grammars can also import other grammars. ANTLR pursues all imported grammars in a depth-first fashion. If two or more imported grammars define rule `r`, ANTLR chooses the first version of `r` it finds. In the following diagram, ANTLR examines grammars in the following order `Nested`, `G1`, `G3`, `G2`. diff --git a/doc/left-recursion.md b/doc/left-recursion.md index a08f4357d..3430e10e9 100644 --- a/doc/left-recursion.md +++ b/doc/left-recursion.md @@ -1,6 +1,6 @@ # Left-recursive rules -The most natural expression of a some common language constructs is left recursive. For example C declarators and arithmetic expressions. Unfortunately, left recursive specifications of arithmetic expressions are typically ambiguous but much easier to write out than the multiple levels required in a typical top-down grammar. Here is a sample ANTLR 4 grammar with a left recursive expression rule: +The most natural expression of some common language constructs is left recursive. For example C declarators and arithmetic expressions. Unfortunately, left recursive specifications of arithmetic expressions are typically ambiguous but much easier to write out than the multiple levels required in a typical top-down grammar. Here is a sample ANTLR 4 grammar with a left recursive expression rule: ``` stat: expr '=' expr ';' // e.g., x=y; or x=f(x); diff --git a/doc/lexer-rules.md b/doc/lexer-rules.md index 2141b4303..adda9e8b0 100644 --- a/doc/lexer-rules.md +++ b/doc/lexer-rules.md @@ -171,7 +171,7 @@ error(126): P.g4:3:4: cannot create implicit token for string literal '&' in non ## Lexer Rule Actions -An ANTLR lexer creates a Token object after matching a lexical rule. Each request for a token starts in Lexer.nextToken, which calls emit once it has identified a token.emit collects information from the current state of the lexer to build the token. It accesses fields `_type`, `_text`, `_channel`, `_tokenStartCharIndex`, `_tokenStartLine`, and `_tokenStartCharPositionInLine`. You can set the state of these with the various setter methods such as `setType`. For example, the following rule turns `enum` into an identifier if `enumIsKeyword` is false. +An ANTLR lexer creates a Token object after matching a lexical rule. Each request for a token starts in `Lexer.nextToken`, which calls `emit` once it has identified a token. `emit` collects information from the current state of the lexer to build the token. It accesses fields `_type`, `_text`, `_channel`, `_tokenStartCharIndex`, `_tokenStartLine`, and `_tokenStartCharPositionInLine`. You can set the state of these with the various setter methods such as `setType`. For example, the following rule turns `enum` into an identifier if `enumIsKeyword` is false. ``` ENUM : 'enum' {if (!enumIsKeyword) setType(Identifier);} ; @@ -255,7 +255,8 @@ WS : [ \r\t\n]+ -> skip ; ``` For multiple 'type()' commands, only the rightmost has an effect. -channel() + +### channel() ``` BLOCK_COMMENT diff --git a/doc/options.md b/doc/options.md index 1b2c2591b..36906db20 100644 --- a/doc/options.md +++ b/doc/options.md @@ -25,7 +25,7 @@ options {...} ## Rule Element Options -Token options have the form `T` as we saw in Section 5.4, [Dealing with Precedence, Left Recursion, and Associativity](http://pragprog.com/book/tpantlr2/the-definitive-antlr-4-reference). The only token option is assocand it accepts values left and right. Here’s a sample grammar with a left-recursive expression rule that specifies a token option on the `^` exponent operator token: +Token options have the form `T` as we saw in Section 5.4, [Dealing with Precedence, Left Recursion, and Associativity](http://pragprog.com/book/tpantlr2/the-definitive-antlr-4-reference). The only token option is `assoc`, and it accepts values `left` and `right`. Here’s a sample grammar with a left-recursive expression rule that specifies a token option on the `^` exponent operator token: ``` grammar ExprLR; @@ -40,7 +40,7 @@ INT : '0'..'9'+ ; WS : [ \n]+ -> skip ; ``` -Semantic predicates also accept an option, per [Catching failed semantic predicates](http://pragprog.com/book/tpantlr2/the-definitive-antlr-4-reference). The only valid option is the fail option, which takes either a string literal in double-quotes or an action that evaluates to a string. The string literal or string result from the action should be the message to emit upon predicate failure. +Semantic predicates also accept an option, per [Catching failed semantic predicates](http://pragprog.com/book/tpantlr2/the-definitive-antlr-4-reference). The only valid option is the `fail` option, which takes either a string literal in double-quotes or an action that evaluates to a string. The string literal or string result from the action should be the message to emit upon predicate failure. ``` ints[int max] diff --git a/doc/predicates.md b/doc/predicates.md index af1d278e7..09998c425 100644 --- a/doc/predicates.md +++ b/doc/predicates.md @@ -29,7 +29,7 @@ expr: {istype()}? ID '(' expr ')' // ctor-style typecast ; ``` -The parser will only predict an expr from stat when `istype()||isfunc()` evaluates to true. This makes sense because the parser should only choose to match an expression if the upcoming `ID` is a type name or function name. It wouldn't make sense to just test one of the predicates in this case. Note that, when the parser gets to expritself, the parsing decision tests the predicates individually, one for each alternative. +The parser will only predict an expr from stat when `istype()||isfunc()` evaluates to true. This makes sense because the parser should only choose to match an expression if the upcoming `ID` is a type name or function name. It wouldn't make sense to just test one of the predicates in this case. Note that, when the parser gets to `expr` itself, the parsing decision tests the predicates individually, one for each alternative. If multiple predicates occur in a sequence, the parser joins them with the `&&` operator. For example, consider changing `stat` to include a predicate before the call `toexpr`: @@ -72,7 +72,7 @@ stat: {System.out.println("goto"); allowgoto=true;} {java5}? 'goto' ID ';' If we can't execute the action during prediction, we shouldn't evaluate the `{java5}?` predicate because it depends on that action. -The prediction process also can't see through token references. Token references have the side effect of advancing the input one symbol. A predicate that tested the current input symbol would find itself out of sync if the parser shifted it over the token reference. For example, in the following grammar, the predicates expectgetCurrentToken to return an ID token. +The prediction process also can't see through token references. Token references have the side effect of advancing the input one symbol. A predicate that tested the current input symbol would find itself out of sync if the parser shifted it over the token reference. For example, in the following grammar, the predicates expect `getCurrentToken` to return an `ID` token. ``` stat: '{' decl '}' diff --git a/doc/tree-matching.md b/doc/tree-matching.md index 5b7d94b58..f4c6d278c 100644 --- a/doc/tree-matching.md +++ b/doc/tree-matching.md @@ -29,7 +29,7 @@ ParseTreeMatch m = p.match(t); if ( m.succeeded() ) {...} ``` -We can also test for specific expressions or token values. For example, the following checks to see if t is an expression consisting of an identifier added to 0: +We can also test for specific expressions or token values. For example, the following checks to see if `t` is an expression consisting of an identifier added to 0: ```java ParseTree t = ...; // assume t is an expression From fa10ca678f357e49100fa8e7461887825e7e8470 Mon Sep 17 00:00:00 2001 From: parrt Date: Mon, 28 Mar 2016 11:35:44 -0700 Subject: [PATCH 51/95] add method to RuleContext --- .../src/org/antlr/v4/runtime/RuleContext.java | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java b/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java index 6cde47e67..7749fed7f 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java @@ -29,20 +29,17 @@ */ package org.antlr.v4.runtime; +import org.antlr.v4.runtime.atn.ATN; import org.antlr.v4.runtime.misc.Interval; import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.runtime.tree.ParseTreeVisitor; import org.antlr.v4.runtime.tree.RuleNode; import org.antlr.v4.runtime.tree.Trees; -import javax.print.PrintException; -import javax.swing.*; -import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.concurrent.Future; -/** /** A rule context is a record of a single rule invocation. +/** A rule context is a record of a single rule invocation. * * We form a stack of these context objects using the parent * pointer. A parent pointer of null indicates that the current @@ -169,6 +166,17 @@ public class RuleContext implements RuleNode { public int getRuleIndex() { return -1; } + /** For rule associated with this parse tree internal node, return + * the outer alternative number used to match the input. Default + * implementation does not compute nor store this alt num. Create + * a subclass of ParserRuleContext with backing field and set + * option treeSuperClass. Then override Parser.enterOuterAlt() + * to set it. + * + * @since 4.5.3 + */ + public int getAltNumber() { return ATN.INVALID_ALT_NUMBER; } + @Override public ParseTree getChild(int i) { return null; From 6ca812e255beb09713f621460018e8fb8f8254c8 Mon Sep 17 00:00:00 2001 From: Martin Steiger Date: Fri, 25 Mar 2016 10:34:35 +0100 Subject: [PATCH 52/95] Add Vocabulary.getMaxTokenType() --- runtime/Java/src/org/antlr/v4/runtime/Vocabulary.java | 10 +++++++++- .../Java/src/org/antlr/v4/runtime/VocabularyImpl.java | 9 +++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/Vocabulary.java b/runtime/Java/src/org/antlr/v4/runtime/Vocabulary.java index 0b6386c8e..46c4af1f7 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Vocabulary.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Vocabulary.java @@ -37,6 +37,14 @@ package org.antlr.v4.runtime; * @author Sam Harwell */ public interface Vocabulary { + + /** + * Returns the highest token type value. It can be used to iterate from + * zero to that number, thus querying all stored entries. + * @return the highest token type value + */ + int getMaxTokenType(); + /** * Gets the string literal associated with a token type. The string returned * by this method, when not {@code null}, can be used unaltered in a parser @@ -85,7 +93,7 @@ public interface Vocabulary { * *
    *
  • Tokens created by lexer rules.
    • - *
  • Tokens defined in a {@code tokens{}} block in a lexer or parser + *
  • Tokens defined in a tokens{} block in a lexer or parser * grammar.
    • *
  • The implicitly defined {@code EOF} token, which has the token type * {@link Token#EOF}.
    • diff --git a/runtime/Java/src/org/antlr/v4/runtime/VocabularyImpl.java b/runtime/Java/src/org/antlr/v4/runtime/VocabularyImpl.java index ff5beefe2..75833ec2c 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/VocabularyImpl.java +++ b/runtime/Java/src/org/antlr/v4/runtime/VocabularyImpl.java @@ -57,6 +57,8 @@ public class VocabularyImpl implements Vocabulary { private final String[] displayNames; + private final int maxTokenType; + /** * Constructs a new instance of {@link VocabularyImpl} from the specified * literal and symbolic token names. @@ -94,6 +96,8 @@ public class VocabularyImpl implements Vocabulary { this.literalNames = literalNames != null ? literalNames : EMPTY_NAMES; this.symbolicNames = symbolicNames != null ? symbolicNames : EMPTY_NAMES; this.displayNames = displayNames != null ? displayNames : EMPTY_NAMES; + this.maxTokenType = Math.max(this.displayNames.length, + Math.max(this.literalNames.length, this.symbolicNames.length)); } /** @@ -143,6 +147,11 @@ public class VocabularyImpl implements Vocabulary { return new VocabularyImpl(literalNames, symbolicNames, tokenNames); } + @Override + public int getMaxTokenType() { + return maxTokenType; + } + @Override public String getLiteralName(int tokenType) { if (tokenType >= 0 && tokenType < literalNames.length) { From 420f4e02820a93d31a5f138fdd85a0ae1061d5f2 Mon Sep 17 00:00:00 2001 From: Martin Steiger Date: Mon, 28 Mar 2016 22:10:36 +0200 Subject: [PATCH 53/95] Update contributors.txt Added msteiger --- contributors.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/contributors.txt b/contributors.txt index fb6040c78..2fab1c79d 100644 --- a/contributors.txt +++ b/contributors.txt @@ -89,3 +89,4 @@ YYYY/MM/DD, github id, Full name, email 2015/12/23, pboyer, Peter Boyer, peter.b.boyer@gmail.com 2015/12/24, dtymon, David Tymon, david.tymon@gmail.com 2016/03/27, beardlybread, Bradley Steinbacher, bradley.j.steinbacher@gmail.com +2016/03/29, msteiger, Martin Steiger, antlr@martin-steiger.de From e4a425321960f0bf4af4f9df313b4fb6ae01254f Mon Sep 17 00:00:00 2001 From: parrt Date: Mon, 28 Mar 2016 14:10:29 -0700 Subject: [PATCH 54/95] more impl of get/set alt num; update doc --- doc/options.md | 48 +++++++++++++++++++ .../Java/src/org/antlr/v4/runtime/Parser.java | 3 +- .../src/org/antlr/v4/runtime/RuleContext.java | 12 ++++- .../v4/tool/templates/codegen/Java/Java.stg | 5 +- .../antlr/v4/codegen/model/ParserFile.java | 9 +++- tool/src/org/antlr/v4/tool/Grammar.java | 1 + 6 files changed, 71 insertions(+), 7 deletions(-) diff --git a/doc/options.md b/doc/options.md index 36906db20..b96fd876b 100644 --- a/doc/options.md +++ b/doc/options.md @@ -12,6 +12,54 @@ where a value can be an identifier, a qualified identifier (for example, a.b.c), All grammars can use the following options. In combined grammars, all options except language pertain only to the generated parser. Options may be set either within the grammar file using the options syntax (described above) or when invoking ANTLR on the command line, using the `-D` option. (see Section 15.9, [ANTLR Tool Command Line Options](tool-options.md).) The following examples demonstrate both mechanisms; note that `-D` overrides options within the grammar. +* `superClass`. Set the superclass of the generated parser or lexer. For combined grammars, it sets the superclass of the parser. +``` +$ cat Hi.g4 +grammar Hi; +a : 'hi' ; +$ antlr4 -DsuperClass=XX Hi.g4 +$ grep 'public class' HiParser.java +public class HiParser extends XX { +$ grep 'public class' HiLexer.java +public class HiLexer extends Lexer { +``` +* `language` Generate code in the indicated language, if ANTLR is able to do so. Otherwise, you will see an error message like this: +``` +$ antlr4 -Dlanguage=C MyGrammar.g4 +error(31): ANTLR cannot generate C code as of version 4.0 +``` +* `tokenVocab` ANTLR assigns token type numbers to the tokens as it encounters them in a file. To use different token type values, such as with a separate lexer, use this option to have ANTLR pull in the tokens file. ANTLR generates a tokens file from each grammar. +``` +$ cat SomeLexer.g4 +lexer grammar SomeLexer; +ID : [a-z]+ ; +$ cat R.g4 +parser grammar R; +options {tokenVocab=SomeLexer;} +tokens {A,B,C} // normally, these would be token types 1, 2, 3 +a : ID ; +$ antlr4 SomeLexer.g4 +$ cat SomeLexer.tokens +ID=1 +$ antlr4 R.g4 +$ cat R.tokens +A=2 +B=3 +C=4 +ID=1 +``` +* `TokenLabelType` ANTLR normally uses type Token when it generates variables referencing tokens. If you have passed a TokenFactory to your parser and lexer so that they create custom tokens, you should set this option to your specific type. This ensures that the context objects know your type for fields and method return values. +``` +$ cat T2.g4 +grammar T2; +options {TokenLabelType=MyToken;} +a : x=ID ; +$ antlr4 T2.g4 +$ grep MyToken T2Parser.java + public MyToken x; +``` +* `contextSuperClass`. Specify the super class of parse tree internal nodes. Default is `ParserRuleContext`. Should derived from ultimately `RuleContext` at minimum. + ## Rule Options There are currently no valid rule-level options, but the tool still supports the following syntax for future use: diff --git a/runtime/Java/src/org/antlr/v4/runtime/Parser.java b/runtime/Java/src/org/antlr/v4/runtime/Parser.java index eb299ecd1..eda34a13b 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/Parser.java +++ b/runtime/Java/src/org/antlr/v4/runtime/Parser.java @@ -34,7 +34,6 @@ import org.antlr.v4.runtime.atn.ATNDeserializationOptions; import org.antlr.v4.runtime.atn.ATNDeserializer; import org.antlr.v4.runtime.atn.ATNSimulator; import org.antlr.v4.runtime.atn.ATNState; -import org.antlr.v4.runtime.atn.AmbiguityInfo; import org.antlr.v4.runtime.atn.ParseInfo; import org.antlr.v4.runtime.atn.ParserATNSimulator; import org.antlr.v4.runtime.atn.PredictionMode; @@ -51,7 +50,6 @@ import org.antlr.v4.runtime.tree.pattern.ParseTreePattern; import org.antlr.v4.runtime.tree.pattern.ParseTreePatternMatcher; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -649,6 +647,7 @@ public abstract class Parser extends Recognizer { } public void enterOuterAlt(ParserRuleContext localctx, int altNum) { + localctx.setAltNumber(altNum); // if we have new localctx, make sure we replace existing ctx // that is previous child of parse tree if ( _buildParseTrees && _ctx != localctx ) { diff --git a/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java b/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java index 7749fed7f..010396625 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java +++ b/runtime/Java/src/org/antlr/v4/runtime/RuleContext.java @@ -170,13 +170,23 @@ public class RuleContext implements RuleNode { * the outer alternative number used to match the input. Default * implementation does not compute nor store this alt num. Create * a subclass of ParserRuleContext with backing field and set - * option treeSuperClass. Then override Parser.enterOuterAlt() + * option contextSuperClass. * to set it. * * @since 4.5.3 */ public int getAltNumber() { return ATN.INVALID_ALT_NUMBER; } + /** Set the outer alternative number for this context node. Default + * implementation does nothing to avoid backing field overhead for + * trees that don't need it. Create + * a subclass of ParserRuleContext with backing field and set + * option contextSuperClass. + * + * @since 4.5.3 + */ + public void setAltNumber(int altNumber) { } + @Override public ParseTree getChild(int i) { return null; diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg index 8bd429a0b..bfc604c41 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg @@ -765,9 +765,8 @@ ListLabelName(label) ::= "
*/ - public static final String VERSION = "4.5.2"; + public static final String VERSION = "4.5.3"; /** * Gets the currently executing version of the ANTLR 4 runtime library. diff --git a/runtime/JavaScript/src/antlr4/Recognizer.js b/runtime/JavaScript/src/antlr4/Recognizer.js index f7a39ebc8..96a8e0729 100644 --- a/runtime/JavaScript/src/antlr4/Recognizer.js +++ b/runtime/JavaScript/src/antlr4/Recognizer.js @@ -45,7 +45,7 @@ Recognizer.ruleIndexMapCache = {}; Recognizer.prototype.checkVersion = function(toolVersion) { - var runtimeVersion = "4.5.2"; + var runtimeVersion = "4.5.3"; if (runtimeVersion!==toolVersion) { console.log("ANTLR runtime and generated code versions disagree: "+runtimeVersion+"!="+toolVersion); } diff --git a/runtime/JavaScript/src/antlr4/package.json b/runtime/JavaScript/src/antlr4/package.json index c5f4cd608..dfa05e29e 100644 --- a/runtime/JavaScript/src/antlr4/package.json +++ b/runtime/JavaScript/src/antlr4/package.json @@ -1,6 +1,6 @@ { "name": "antlr4", - "version": "4.5.2", + "version": "4.5.3", "description": "JavaScript runtime for ANTLR4", "main": "src/antlr4/index.js", "repository": "antlr/antlr4.git", diff --git a/runtime/Python2/src/antlr4/Recognizer.py b/runtime/Python2/src/antlr4/Recognizer.py index c82760e59..438e80cb7 100644 --- a/runtime/Python2/src/antlr4/Recognizer.py +++ b/runtime/Python2/src/antlr4/Recognizer.py @@ -55,7 +55,7 @@ class Recognizer(object): return major, minor def checkVersion(self, toolVersion): - runtimeVersion = "4.5.2.1" + runtimeVersion = "4.5.3" rvmajor, rvminor = self.extractVersion(runtimeVersion) tvmajor, tvminor = self.extractVersion(toolVersion) if rvmajor!=tvmajor or rvminor!=tvminor: diff --git a/runtime/Python3/src/antlr4/Recognizer.py b/runtime/Python3/src/antlr4/Recognizer.py index a887504f4..8b31f106e 100644 --- a/runtime/Python3/src/antlr4/Recognizer.py +++ b/runtime/Python3/src/antlr4/Recognizer.py @@ -58,7 +58,7 @@ class Recognizer(object): return major, minor def checkVersion(self, toolVersion): - runtimeVersion = "4.5.2.1" + runtimeVersion = "4.5.3" rvmajor, rvminor = self.extractVersion(runtimeVersion) tvmajor, tvminor = self.extractVersion(toolVersion) if rvmajor!=tvmajor or rvminor!=tvminor: diff --git a/tool/src/org/antlr/v4/codegen/target/CSharpTarget.java b/tool/src/org/antlr/v4/codegen/target/CSharpTarget.java index bfebe4b7e..b4aa5843a 100644 --- a/tool/src/org/antlr/v4/codegen/target/CSharpTarget.java +++ b/tool/src/org/antlr/v4/codegen/target/CSharpTarget.java @@ -51,7 +51,7 @@ public class CSharpTarget extends Target { @Override public String getVersion() { - return "4.5.2"; // crossing fingers that it's close enough. + return "4.5.3"; } @Override diff --git a/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java b/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java index 31220b3dc..964837626 100644 --- a/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java +++ b/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java @@ -74,7 +74,7 @@ public class JavaScriptTarget extends Target { @Override public String getVersion() { - return "4.5.2"; + return "4.5.3"; } public Set getBadWords() { diff --git a/tool/src/org/antlr/v4/codegen/target/Python2Target.java b/tool/src/org/antlr/v4/codegen/target/Python2Target.java index 9b88a205d..c80949af2 100644 --- a/tool/src/org/antlr/v4/codegen/target/Python2Target.java +++ b/tool/src/org/antlr/v4/codegen/target/Python2Target.java @@ -117,7 +117,7 @@ public class Python2Target extends Target { @Override public String getVersion() { - return "4.5.2.1"; + return "4.5.3"; } public Set getBadWords() { diff --git a/tool/src/org/antlr/v4/codegen/target/Python3Target.java b/tool/src/org/antlr/v4/codegen/target/Python3Target.java index 97d56d0fa..f70d2a27e 100644 --- a/tool/src/org/antlr/v4/codegen/target/Python3Target.java +++ b/tool/src/org/antlr/v4/codegen/target/Python3Target.java @@ -119,7 +119,7 @@ public class Python3Target extends Target { @Override public String getVersion() { - return "4.5.2.1"; + return "4.5.3"; } /** Avoid grammar symbols in this set to prevent conflicts in gen'd code. */ From 567fcc631129603e28a265adb9814d9dfb46d236 Mon Sep 17 00:00:00 2001 From: parrt Date: Wed, 30 Mar 2016 16:25:45 -0700 Subject: [PATCH 76/95] [maven-release-plugin] prepare release antlr4-master-4.5.3 --- antlr4-maven-plugin/pom.xml | 2 +- pom.xml | 2 +- runtime-testsuite/pom.xml | 2 +- runtime/Java/pom.xml | 2 +- tool-testsuite/pom.xml | 2 +- tool/pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/antlr4-maven-plugin/pom.xml b/antlr4-maven-plugin/pom.xml index 2ebf43d63..54f5edd92 100644 --- a/antlr4-maven-plugin/pom.xml +++ b/antlr4-maven-plugin/pom.xml @@ -34,7 +34,7 @@ org.antlr antlr4-master - 4.5.3-SNAPSHOT + 4.5.3 antlr4-maven-plugin maven-plugin diff --git a/pom.xml b/pom.xml index 4ff16411c..2fcbe8bfe 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.antlr antlr4-master - 4.5.3-SNAPSHOT + 4.5.3 pom ANTLR 4 diff --git a/runtime-testsuite/pom.xml b/runtime-testsuite/pom.xml index f9f105200..d4daa3846 100644 --- a/runtime-testsuite/pom.xml +++ b/runtime-testsuite/pom.xml @@ -4,7 +4,7 @@ org.antlr antlr4-master - 4.5.3-SNAPSHOT + 4.5.3 antlr4-runtime-testsuite ANTLR 4 Runtime Test Generator diff --git a/runtime/Java/pom.xml b/runtime/Java/pom.xml index c00729bf3..7d0434ed6 100644 --- a/runtime/Java/pom.xml +++ b/runtime/Java/pom.xml @@ -3,7 +3,7 @@ org.antlr antlr4-master - 4.5.3-SNAPSHOT + 4.5.3 ../../pom.xml antlr4-runtime diff --git a/tool-testsuite/pom.xml b/tool-testsuite/pom.xml index cea14b194..967c5df16 100644 --- a/tool-testsuite/pom.xml +++ b/tool-testsuite/pom.xml @@ -4,7 +4,7 @@ org.antlr antlr4-master - 4.5.3-SNAPSHOT + 4.5.3 antlr4-tool-testsuite ANTLR 4 Tool Tests diff --git a/tool/pom.xml b/tool/pom.xml index bda5bbd53..bfd74c869 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -3,7 +3,7 @@ org.antlr antlr4-master - 4.5.3-SNAPSHOT + 4.5.3 antlr4 ANTLR 4 Tool From b92f22769ef06e7d009fb13c20363c052ec29462 Mon Sep 17 00:00:00 2001 From: parrt Date: Wed, 30 Mar 2016 16:25:49 -0700 Subject: [PATCH 77/95] [maven-release-plugin] prepare for next development iteration --- antlr4-maven-plugin/pom.xml | 2 +- pom.xml | 2 +- runtime-testsuite/pom.xml | 2 +- runtime/Java/pom.xml | 2 +- tool-testsuite/pom.xml | 2 +- tool/pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/antlr4-maven-plugin/pom.xml b/antlr4-maven-plugin/pom.xml index 54f5edd92..b9e144d33 100644 --- a/antlr4-maven-plugin/pom.xml +++ b/antlr4-maven-plugin/pom.xml @@ -34,7 +34,7 @@ org.antlr antlr4-master - 4.5.3 + 4.5.4-SNAPSHOT antlr4-maven-plugin maven-plugin diff --git a/pom.xml b/pom.xml index 2fcbe8bfe..e8cb3f6b6 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.antlr antlr4-master - 4.5.3 + 4.5.4-SNAPSHOT pom ANTLR 4 diff --git a/runtime-testsuite/pom.xml b/runtime-testsuite/pom.xml index d4daa3846..e09140dea 100644 --- a/runtime-testsuite/pom.xml +++ b/runtime-testsuite/pom.xml @@ -4,7 +4,7 @@ org.antlr antlr4-master - 4.5.3 + 4.5.4-SNAPSHOT antlr4-runtime-testsuite ANTLR 4 Runtime Test Generator diff --git a/runtime/Java/pom.xml b/runtime/Java/pom.xml index 7d0434ed6..9824ffdd4 100644 --- a/runtime/Java/pom.xml +++ b/runtime/Java/pom.xml @@ -3,7 +3,7 @@ org.antlr antlr4-master - 4.5.3 + 4.5.4-SNAPSHOT ../../pom.xml antlr4-runtime diff --git a/tool-testsuite/pom.xml b/tool-testsuite/pom.xml index 967c5df16..c7b1564d4 100644 --- a/tool-testsuite/pom.xml +++ b/tool-testsuite/pom.xml @@ -4,7 +4,7 @@ org.antlr antlr4-master - 4.5.3 + 4.5.4-SNAPSHOT antlr4-tool-testsuite ANTLR 4 Tool Tests diff --git a/tool/pom.xml b/tool/pom.xml index bfd74c869..42e5f825f 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -3,7 +3,7 @@ org.antlr antlr4-master - 4.5.3 + 4.5.4-SNAPSHOT antlr4 ANTLR 4 Tool From b9ed52ecd9dbbb17fa706cd400e9bc8cb06ab2f2 Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Wed, 30 Mar 2016 16:51:40 -0700 Subject: [PATCH 78/95] Update releasing-antlr.md --- doc/releasing-antlr.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/releasing-antlr.md b/doc/releasing-antlr.md index 6df68607c..6a7450c45 100644 --- a/doc/releasing-antlr.md +++ b/doc/releasing-antlr.md @@ -135,16 +135,20 @@ cp ~/.m2/repository/org/antlr/antlr4/4.5.2/antlr4-4.5.2.jar ~/antlr/sites/websit cd ~/antlr/sites/website-antlr4/download git add antlr-4.5.2-complete.jar git add antlr-runtime-4.5.2.jar -git commit -a -m 'add 4.5.2 jars' -git push origin gh-pages ``` Update on site: * download.html * index.html +* download/index.html * scripts/topnav.js +``` +git commit -a -m 'add 4.5.2 jars' +git push origin gh-pages +``` + ## Deploying Targets ### JavaScript From 14f05bbcc6573dfe8c367df2cf051b79cc1332f6 Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Wed, 30 Mar 2016 16:59:48 -0700 Subject: [PATCH 79/95] Update releasing-antlr.md --- doc/releasing-antlr.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/releasing-antlr.md b/doc/releasing-antlr.md index 6a7450c45..05815086e 100644 --- a/doc/releasing-antlr.md +++ b/doc/releasing-antlr.md @@ -141,6 +141,7 @@ Update on site: * download.html * index.html +* api/index.html * download/index.html * scripts/topnav.js From 7562bb3b45ebe781b236d48a8bd8782ffc9ef7e8 Mon Sep 17 00:00:00 2001 From: Renata Hodovan Date: Wed, 13 Apr 2016 17:52:17 +0200 Subject: [PATCH 80/95] Add missing default pass statements to Python targets. The default pass statements were missing from the LL1AltBlock's template in case of Python targets, which generated conditional statements without body in some cases and ended in syntax error. The patch fixes this. --- .../org/antlr/v4/tool/templates/codegen/Python2/Python2.stg | 2 +- .../org/antlr/v4/tool/templates/codegen/Python3/Python3.stg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg index c4ab7cccd..8be3403f4 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg @@ -319,7 +319,7 @@ self.state = token = self._input.LA(1) - }; separator="\nel"> + pass}; separator="\nel"> else: diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg index c3410795b..b498a736d 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg @@ -327,7 +327,7 @@ self.state = token = self._input.LA(1) - }; separator="\nel"> + pass}; separator="\nel"> else: From d430f495eafa7b0c727ac7d8bfe4e5421660311b Mon Sep 17 00:00:00 2001 From: parrt Date: Mon, 25 Apr 2016 09:49:29 -0700 Subject: [PATCH 81/95] add comments to "dead" code in serializer. text from Sam Harwell. --- .../antlr/v4/runtime/atn/ATNSerializer.java | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSerializer.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSerializer.java index 4c31ac92f..9183eef61 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSerializer.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ATNSerializer.java @@ -424,6 +424,12 @@ public class ATNSerializer { .append(ATNState.serializationNames.get(stype)).append(" ") .append(ruleIndex).append(arg).append("\n"); } + // this code is meant to model the form of ATNDeserializer.deserialize, + // since both need to be updated together whenever a change is made to + // the serialization format. The "dead" code is only used in debugging + // and testing scenarios, so the form you see here was kept for + // improved maintainability. + // start int numNonGreedyStates = ATNDeserializer.toInt(data[p++]); for (int i = 0; i < numNonGreedyStates; i++) { int stateNumber = ATNDeserializer.toInt(data[p++]); @@ -432,6 +438,7 @@ public class ATNSerializer { for (int i = 0; i < numPrecedenceStates; i++) { int stateNumber = ATNDeserializer.toInt(data[p++]); } + // finish int nrules = ATNDeserializer.toInt(data[p++]); for (int i=0; i Date: Tue, 26 Apr 2016 13:27:26 +0800 Subject: [PATCH 82/95] links update --- runtime/JavaScript/README.md | 2 +- runtime/JavaScript/src/antlr4/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/JavaScript/README.md b/runtime/JavaScript/README.md index f35326d6d..5205fd423 100644 --- a/runtime/JavaScript/README.md +++ b/runtime/JavaScript/README.md @@ -8,6 +8,6 @@ This runtime has been tested in Node.js, Safari, Firefox, Chrome and IE. See www.antlr.org for more information on ANTLR -See https://raw.githubusercontent.com/antlr/antlr4/master/doc/javascript-target.md for more information on using ANTLR in JavaScript +See https://github.com/antlr/antlr4/blob/master/doc/javascript-target.md for more information on using ANTLR in JavaScript diff --git a/runtime/JavaScript/src/antlr4/README.md b/runtime/JavaScript/src/antlr4/README.md index 9ff7e5cfc..5205fd423 100644 --- a/runtime/JavaScript/src/antlr4/README.md +++ b/runtime/JavaScript/src/antlr4/README.md @@ -8,6 +8,6 @@ This runtime has been tested in Node.js, Safari, Firefox, Chrome and IE. See www.antlr.org for more information on ANTLR -See https://theantlrguy.atlassian.net/wiki/display/ANTLR4/JavaScript+Target for more information on using ANTLR in JavaScript +See https://github.com/antlr/antlr4/blob/master/doc/javascript-target.md for more information on using ANTLR in JavaScript From c260ce75d13a20f016ea57fda0e1f347b79082ae Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Thu, 23 Jun 2016 15:06:14 +0200 Subject: [PATCH 83/95] make it possible to add breakpoint --- .../Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java index 15f71ec58..4f217a73c 100755 --- a/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java +++ b/runtime/Java/src/org/antlr/v4/runtime/atn/ParserATNSimulator.java @@ -788,7 +788,8 @@ public class ParserATNSimulator extends ATNSimulator { protected ATNConfigSet computeReachSet(ATNConfigSet closure, int t, boolean fullCtx) { - if ( debug ) System.out.println("in computeReachSet, starting closure: " + closure); + if ( debug ) + System.out.println("in computeReachSet, starting closure: " + closure); if (mergeCache == null) { mergeCache = new DoubleKeyMap(); From 15430d4bf92f4aa187e5263467a2fa922f4589e4 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Thu, 23 Jun 2016 15:06:49 +0200 Subject: [PATCH 84/95] remove unwanted semicolon in generated Python code --- .../org/antlr/v4/tool/templates/codegen/Python2/Python2.stg | 4 ++-- .../org/antlr/v4/tool/templates/codegen/Python3/Python3.stg | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg index c4ab7cccd..84dc10cdf 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg @@ -375,7 +375,7 @@ while True: AltBlock(choice, preamble, alts, error) ::= << self.state = -self._errHandler.sync(self); +self._errHandler.sync(self) = _input.LT(1) la_ = self._interp.adaptivePredict(self._input,,self._ctx) @@ -389,7 +389,7 @@ if la_ == : OptionalBlock(choice, alts, error) ::= << self.state = -self._errHandler.sync(self); +self._errHandler.sync(self) la_ = self._interp.adaptivePredict(self._input,,self._ctx) +1: diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg index c3410795b..80f776e6d 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg @@ -383,7 +383,7 @@ while True: AltBlock(choice, preamble, alts, error) ::= << self.state = -self._errHandler.sync(self); +self._errHandler.sync(self) = _input.LT(1) la_ = self._interp.adaptivePredict(self._input,,self._ctx) @@ -397,7 +397,7 @@ if la_ == : OptionalBlock(choice, alts, error) ::= << self.state = -self._errHandler.sync(self); +self._errHandler.sync(self) la_ = self._interp.adaptivePredict(self._input,,self._ctx) +1: From 26c409103d3b21c2d400b571fb8f948fcdad34de Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Thu, 23 Jun 2016 15:51:39 +0200 Subject: [PATCH 85/95] Fix #1217 The root cause was that ATNConfigSet was not using he required custom hashing strategy for ParserATNSimulator. The commit includes a number of additional fixes, related to code that was never executed before due to the root cause. A similar issue is also likely to exist in the JavaScript runtime, I'll fix it later. --- .../Python2/src/antlr4/PredictionContext.py | 73 ++++++++-------- runtime/Python2/src/antlr4/atn/ATNConfig.py | 25 ++++++ .../Python2/src/antlr4/atn/ATNConfigSet.py | 8 +- .../src/antlr4/atn/LexerATNSimulator.py | 36 ++++---- .../src/antlr4/atn/ParserATNSimulator.py | 87 +++++++++---------- .../Python2/src/antlr4/atn/PredictionMode.py | 3 +- .../Python3/src/antlr4/PredictionContext.py | 72 +++++++-------- runtime/Python3/src/antlr4/atn/ATNConfig.py | 26 +++++- .../Python3/src/antlr4/atn/ATNConfigSet.py | 8 +- .../src/antlr4/atn/LexerATNSimulator.py | 36 ++++---- .../src/antlr4/atn/ParserATNSimulator.py | 87 +++++++++---------- .../Python3/src/antlr4/atn/PredictionMode.py | 3 +- 12 files changed, 259 insertions(+), 205 deletions(-) diff --git a/runtime/Python2/src/antlr4/PredictionContext.py b/runtime/Python2/src/antlr4/PredictionContext.py index ba28eca7b..dae19e9e0 100644 --- a/runtime/Python2/src/antlr4/PredictionContext.py +++ b/runtime/Python2/src/antlr4/PredictionContext.py @@ -30,6 +30,7 @@ #/ from io import StringIO from antlr4.RuleContext import RuleContext +from antlr4.atn.ATN import ATN from antlr4.atn.ATNState import ATNState @@ -98,7 +99,7 @@ def calculateHashCode(parent, returnState): def calculateListsHashCode(parents, returnStates ): h = 0 - for parent, returnState in parents, returnStates: + for parent, returnState in zip(parents, returnStates): h = hash((h, calculateHashCode(parent, returnState))) return h @@ -254,6 +255,10 @@ class ArrayPredictionContext(PredictionContext): buf.write(u"]") return buf.getvalue() + def __hash__(self): + return self.cachedHashCode + + # Convert a {@link RuleContext} tree to a {@link PredictionContext} graph. # Return {@link #EMPTY} if {@code outerContext} is empty or null. @@ -328,18 +333,18 @@ def merge(a, b, rootIsWildcard, mergeCache): #/ def mergeSingletons(a, b, rootIsWildcard, mergeCache): if mergeCache is not None: - previous = mergeCache.get(a,b) + previous = mergeCache.get((a,b), None) if previous is not None: return previous - previous = mergeCache.get(b,a) + previous = mergeCache.get((b,a), None) if previous is not None: return previous - rootMerge = mergeRoot(a, b, rootIsWildcard) - if rootMerge is not None: + merged = mergeRoot(a, b, rootIsWildcard) + if merged is not None: if mergeCache is not None: - mergeCache.put(a, b, rootMerge) - return rootMerge + mergeCache[(a, b)] = merged + return merged if a.returnState==b.returnState: parent = merge(a.parentCtx, b.parentCtx, rootIsWildcard, mergeCache) @@ -352,10 +357,10 @@ def mergeSingletons(a, b, rootIsWildcard, mergeCache): # merge parents x and y, giving array node with x,y then remainders # of those graphs. dup a, a' points at merged array # new joined parent so create new singleton pointing to it, a' - a_ = SingletonPredictionContext.create(parent, a.returnState) + merged = SingletonPredictionContext.create(parent, a.returnState) if mergeCache is not None: - mergeCache.put(a, b, a_) - return a_ + mergeCache[(a, b)] = merged + return merged else: # a != b payloads differ # see if we can collapse parents due to $+x parents if local ctx singleParent = None @@ -365,26 +370,24 @@ def mergeSingletons(a, b, rootIsWildcard, mergeCache): # sort payloads and use same parent payloads = [ a.returnState, b.returnState ] if a.returnState > b.returnState: - payloads[0] = b.returnState - payloads[1] = a.returnState + payloads = [ b.returnState, a.returnState ] parents = [singleParent, singleParent] - a_ = ArrayPredictionContext(parents, payloads) + merged = ArrayPredictionContext(parents, payloads) if mergeCache is not None: - mergeCache.put(a, b, a_) - return a_ + mergeCache[(a, b)] = merged + return merged # parents differ and can't merge them. Just pack together # into array; can't merge. # ax + by = [ax,by] payloads = [ a.returnState, b.returnState ] parents = [ a.parentCtx, b.parentCtx ] if a.returnState > b.returnState: # sort by payload - payloads[0] = b.returnState - payloads[1] = a.returnState + payloads = [ b.returnState, a.returnState ] parents = [ b.parentCtx, a.parentCtx ] - a_ = ArrayPredictionContext(parents, payloads) + merged = ArrayPredictionContext(parents, payloads) if mergeCache is not None: - mergeCache.put(a, b, a_) - return a_ + mergeCache[(a, b)] = merged + return merged # @@ -466,10 +469,10 @@ def mergeRoot(a, b, rootIsWildcard): #/ def mergeArrays(a, b, rootIsWildcard, mergeCache): if mergeCache is not None: - previous = mergeCache.get(a,b) + previous = mergeCache.get((a,b), None) if previous is not None: return previous - previous = mergeCache.get(b,a) + previous = mergeCache.get((b,a), None) if previous is not None: return previous @@ -478,8 +481,8 @@ def mergeArrays(a, b, rootIsWildcard, mergeCache): j = 0 # walks b k = 0 # walks target M array - mergedReturnStates = [] * (len(a.returnState) + len( b.returnStates)) - mergedParents = [] * len(mergedReturnStates) + mergedReturnStates = [None] * (len(a.returnStates) + len( b.returnStates)) + mergedParents = [None] * len(mergedReturnStates) # walk and merge to yield mergedParents, mergedReturnStates while itrg, src->trg, we keep track of the previous trg to # avoid looking up the DFA state again, which is expensive. @@ -223,8 +223,8 @@ class LexerATNSimulator(ATNSimulator): return None target = s.edges[t - self.MIN_DFA_EDGE] - if self.debug and target is not None: - print("reuse state "+s.stateNumber+ " edge to "+target.stateNumber) + if LexerATNSimulator.debug and target is not None: + print("reuse state", str(s.stateNumber), "edge to", str(target.stateNumber)) return target @@ -280,8 +280,8 @@ class LexerATNSimulator(ATNSimulator): if currentAltReachedAcceptState and cfg.passedThroughNonGreedyDecision: continue - if self.debug: - print("testing %s at %s\n", self.getTokenName(t), cfg.toString(self.recog, True)) + if LexerATNSimulator.debug: + print("testing", self.getTokenName(t), "at", str(cfg)) for trans in cfg.state.transitions: # for each transition target = self.getReachableTarget(trans, t) @@ -298,8 +298,8 @@ class LexerATNSimulator(ATNSimulator): skipAlt = cfg.alt def accept(self, input, lexerActionExecutor, startIndex, index, line, charPos): - if self.debug: - print("ACTION %s\n", lexerActionExecutor) + if LexerATNSimulator.debug: + print("ACTION", lexerActionExecutor) # seek to after last char in token input.seek(index) @@ -334,15 +334,15 @@ class LexerATNSimulator(ATNSimulator): # {@code false}. def closure(self, input, config, configs, currentAltReachedAcceptState, speculative, treatEofAsEpsilon): - if self.debug: - print("closure("+config.toString(self.recog, True)+")") + if LexerATNSimulator.debug: + print("closure(" + str(config) + ")") if isinstance( config.state, RuleStopState ): - if self.debug: + if LexerATNSimulator.debug: if self.recog is not None: - print("closure at %s rule stop %s\n", self.recog.getRuleNames()[config.state.ruleIndex], config) + print("closure at", self.recog.symbolicNames[config.state.ruleIndex], "rule stop", str(config)) else: - print("closure at rule stop %s\n", config) + print("closure at rule stop", str(config)) if config.context is None or config.context.hasEmptyPath(): if config.context is None or config.context.isEmpty(): @@ -404,7 +404,7 @@ class LexerATNSimulator(ATNSimulator): # states reached by traversing predicates. Since this is when we # test them, we cannot cash the DFA state target of ID. - if self.debug: + if LexerATNSimulator.debug: print("EVAL rule "+ str(t.ruleIndex) + ":" + str(t.predIndex)) configs.hasSemanticContext = True if self.evaluatePredicate(input, t.ruleIndex, t.predIndex, speculative): @@ -516,7 +516,7 @@ class LexerATNSimulator(ATNSimulator): # Only track edges within the DFA bounds return to - if self.debug: + if LexerATNSimulator.debug: print("EDGE " + str(from_) + " -> " + str(to) + " upon "+ chr(tk)) if from_.edges is None: diff --git a/runtime/Python2/src/antlr4/atn/ParserATNSimulator.py b/runtime/Python2/src/antlr4/atn/ParserATNSimulator.py index dea6b7e14..a979b40c2 100755 --- a/runtime/Python2/src/antlr4/atn/ParserATNSimulator.py +++ b/runtime/Python2/src/antlr4/atn/ParserATNSimulator.py @@ -308,7 +308,7 @@ class ParserATNSimulator(ATNSimulator): pass def adaptivePredict(self, input, decision, outerContext): - if self.debug or self.debug_list_atn_decisions: + if ParserATNSimulator.debug or ParserATNSimulator.debug_list_atn_decisions: print("adaptivePredict decision " + str(decision) + " exec LA(1)==" + self.getLookaheadName(input) + " line " + str(input.LT(1).line) + ":" + @@ -336,10 +336,10 @@ class ParserATNSimulator(ATNSimulator): if s0 is None: if outerContext is None: outerContext = ParserRuleContext.EMPTY - if self.debug or self.debug_list_atn_decisions: + if ParserATNSimulator.debug or ParserATNSimulator.debug_list_atn_decisions: print("predictATN decision " + str(dfa.decision) + " exec LA(1)==" + self.getLookaheadName(input) + - ", outerContext=" + outerContext.toString(self.parser)) + ", outerContext=" + outerContext.toString(self.parser.literalNames, None)) # If this is not a precedence DFA, we check the ATN start state # to determine if this ATN start state is the decision for the @@ -368,8 +368,8 @@ class ParserATNSimulator(ATNSimulator): dfa.s0 = s0 alt = self.execATN(dfa, s0, input, index, outerContext) - if self.debug: - print("DFA after predictATN: " + dfa.toString(self.parser.tokenNames)) + if ParserATNSimulator.debug: + print("DFA after predictATN: " + dfa.toString(self.parser.literalNames)) return alt finally: self._dfa = None @@ -408,14 +408,14 @@ class ParserATNSimulator(ATNSimulator): # conflict + preds # def execATN(self, dfa, s0, input, startIndex, outerContext ): - if self.debug or self.debug_list_atn_decisions: + if ParserATNSimulator.debug or ParserATNSimulator.debug_list_atn_decisions: print("execATN decision " + str(dfa.decision) + " exec LA(1)==" + self.getLookaheadName(input) + " line " + str(input.LT(1).line) + ":" + str(input.LT(1).column)) previousD = s0 - if self.debug: + if ParserATNSimulator.debug: print("s0 = " + str(s0)) t = input.LA(1) @@ -445,7 +445,7 @@ class ParserATNSimulator(ATNSimulator): # IF PREDS, MIGHT RESOLVE TO SINGLE ALT => SLL (or syntax error) conflictingAlts = None if D.predicates is not None: - if self.debug: + if ParserATNSimulator.debug: print("DFA state has preds in DFA sim LL failover") conflictIndex = input.index if conflictIndex != startIndex: @@ -453,7 +453,7 @@ class ParserATNSimulator(ATNSimulator): conflictingAlts = self.evalSemanticContext(D.predicates, outerContext, True) if len(conflictingAlts)==1: - if self.debug: + if ParserATNSimulator.debug: print("Full LL avoided") return min(conflictingAlts) @@ -462,7 +462,7 @@ class ParserATNSimulator(ATNSimulator): # context occurs with the index at the correct spot input.seek(conflictIndex) - if self.dfa_debug: + if ParserATNSimulator.dfa_debug: print("ctx sensitive state " + str(outerContext) +" in " + str(D)) fullCtx = True s0_closure = self.computeStartState(dfa.atnStartState, outerContext, fullCtx) @@ -534,7 +534,7 @@ class ParserATNSimulator(ATNSimulator): predictedAlt = self.getUniqueAlt(reach) - if self.debug: + if ParserATNSimulator.debug: altSubSets = PredictionMode.getConflictingAltSubsets(reach) print("SLL altSubSets=" + str(altSubSets) + ", configs=" + str(reach) + ", predict=" + str(predictedAlt) + ", allSubsetsConflict=" + @@ -586,8 +586,8 @@ class ParserATNSimulator(ATNSimulator): input, startIndex, outerContext): - if self.debug or self.debug_list_atn_decisions: - print("execATNWithFullContext "+s0) + if ParserATNSimulator.debug or ParserATNSimulator.debug_list_atn_decisions: + print("execATNWithFullContext", str(s0)) fullCtx = True foundExactAmbig = False reach = None @@ -616,7 +616,7 @@ class ParserATNSimulator(ATNSimulator): raise e altSubSets = PredictionMode.getConflictingAltSubsets(reach) - if self.debug: + if ParserATNSimulator.debug: print("LL altSubSets=" + str(altSubSets) + ", predict=" + str(PredictionMode.getUniqueAlt(altSubSets)) + ", resolvesToJustOneViableAlt=" + str(PredictionMode.resolvesToJustOneViableAlt(altSubSets))) @@ -685,7 +685,7 @@ class ParserATNSimulator(ATNSimulator): return predictedAlt def computeReachSet(self, closure, t, fullCtx): - if self.debug: + if ParserATNSimulator.debug: print("in computeReachSet, starting closure: " + str(closure)) if self.mergeCache is None: @@ -707,7 +707,7 @@ class ParserATNSimulator(ATNSimulator): # First figure out where we can reach on input t for c in closure: - if self.debug: + if ParserATNSimulator.debug: print("testing " + self.getTokenName(t) + " at " + str(c)) if isinstance(c.state, RuleStopState): @@ -967,7 +967,7 @@ class ParserATNSimulator(ATNSimulator): # nonambig alts are null in altToPred if nPredAlts==0: altToPred = None - if self.debug: + if ParserATNSimulator.debug: print("getPredsForAmbigAlts result " + str_list(altToPred)) return altToPred @@ -1093,11 +1093,11 @@ class ParserATNSimulator(ATNSimulator): break continue predicateEvaluationResult = pair.pred.eval(self.parser, outerContext) - if self.debug or self.dfa_debug: + if ParserATNSimulator.debug or ParserATNSimulator.dfa_debug: print("eval pred " + str(pair) + "=" + str(predicateEvaluationResult)) if predicateEvaluationResult: - if self.debug or self.dfa_debug: + if ParserATNSimulator.debug or ParserATNSimulator.dfa_debug: print("PREDICT " + str(pair.alt)) predictions.add(pair.alt) if not complete: @@ -1119,8 +1119,8 @@ class ParserATNSimulator(ATNSimulator): def closureCheckingStopState(self, config, configs, closureBusy, collectPredicates, fullCtx, depth, treatEofAsEpsilon): - if self.debug: - print("closure(" + config.toString(self.parser,True) + ")") + if ParserATNSimulator.debug: + print("closure(" + str(config) + ")") if isinstance(config.state, RuleStopState): # We hit rule end. If we have context info, use it @@ -1134,7 +1134,7 @@ class ParserATNSimulator(ATNSimulator): continue else: # we have no context info, just chase follow links (if greedy) - if self.debug: + if ParserATNSimulator.debug: print("FALLING off rule " + self.getRuleName(config.state.ruleIndex)) self.closure_(config, configs, closureBusy, collectPredicates, fullCtx, depth, treatEofAsEpsilon) @@ -1154,7 +1154,7 @@ class ParserATNSimulator(ATNSimulator): return else: # else if we have no context info, just chase follow links (if greedy) - if self.debug: + if ParserATNSimulator.debug: print("FALLING off rule " + self.getRuleName(config.state.ruleIndex)) self.closure_(config, configs, closureBusy, collectPredicates, fullCtx, depth, treatEofAsEpsilon) @@ -1196,7 +1196,7 @@ class ParserATNSimulator(ATNSimulator): c.reachesIntoOuterContext += 1 configs.dipsIntoOuterContext = True # TODO: can remove? only care when we add to set per middle of this method newDepth -= 1 - if self.debug: + if ParserATNSimulator.debug: print("dips into outer ctx: " + str(c)) elif isinstance(t, RuleTransition): # latch when newDepth goes negative - once we step out of the entry context we can't return @@ -1237,12 +1237,12 @@ class ParserATNSimulator(ATNSimulator): return m(self, config, t, collectPredicates, inContext, fullCtx, treatEofAsEpsilon) def actionTransition(self, config, t): - if self.debug: + if ParserATNSimulator.debug: print("ACTION edge " + str(t.ruleIndex) + ":" + str(t.actionIndex)) return ATNConfig(state=t.target, config=config) def precedenceTransition(self, config, pt, collectPredicates, inContext, fullCtx): - if self.debug: + if ParserATNSimulator.debug: print("PRED (collectPredicates=" + str(collectPredicates) + ") " + str(pt.precedence) + ">=_p, ctx dependent=true") if self.parser is not None: @@ -1267,12 +1267,12 @@ class ParserATNSimulator(ATNSimulator): else: c = ATNConfig(state=pt.target, config=config) - if self.debug: + if ParserATNSimulator.debug: print("config from pred transition=" + str(c)) return c def predTransition(self, config, pt, collectPredicates, inContext, fullCtx): - if self.debug: + if ParserATNSimulator.debug: print("PRED (collectPredicates=" + str(collectPredicates) + ") " + str(pt.ruleIndex) + ":" + str(pt.predIndex) + ", ctx dependent=" + str(pt.isCtxDependent)) if self.parser is not None: @@ -1297,12 +1297,12 @@ class ParserATNSimulator(ATNSimulator): else: c = ATNConfig(state=pt.target, config=config) - if self.debug: + if ParserATNSimulator.debug: print("config from pred transition=" + str(c)) return c def ruleTransition(self, config, t): - if self.debug: + if ParserATNSimulator.debug: print("CALL rule " + self.getRuleName(t.target.ruleIndex) + ", ctx=" + str(config.context)) returnState = t.followState newContext = SingletonPredictionContext.create(config.context, returnState.stateNumber) @@ -1360,13 +1360,12 @@ class ParserATNSimulator(ATNSimulator): def getTokenName(self, t): if t==Token.EOF: return u"EOF" - if self.parser is not None and self.parser.tokenNames is not None: - if t >= len(self.parser.tokenNames): - print(str(t) + " ttype out of range: " + str_list(self.parser.tokenNames)) - print(str_list(self.parser.getInputStream().getTokens())) - else: - return self.parser.tokensNames[t] + u"<" + unicode(t) + ">" - return unicode(t) + if self.parser is not None and \ + self.parser.literalNames is not None and \ + t < len(self.parser.literalNames): + return self.parser.literalNames[t] + u"<" + unicode(t) + ">" + else: + return unicode(t) def getLookaheadName(self, input): return self.getTokenName(input.LA(1)) @@ -1421,7 +1420,7 @@ class ParserATNSimulator(ATNSimulator): # on {@code to} # def addDFAEdge(self, dfa, from_, t, to): - if self.debug: + if ParserATNSimulator.debug: print("EDGE " + str(from_) + " -> " + str(to) + " upon " + self.getTokenName(t)) if to is None: @@ -1435,8 +1434,8 @@ class ParserATNSimulator(ATNSimulator): from_.edges = [None] * (self.atn.maxTokenType + 2) from_.edges[t+1] = to # connect - if self.debug: - names = None if self.parser is None else self.parser.tokenNames + if ParserATNSimulator.debug: + names = None if self.parser is None else self.parser.literalNames print("DFA=\n" + dfa.toString(names)) return to @@ -1470,12 +1469,12 @@ class ParserATNSimulator(ATNSimulator): D.configs.optimizeConfigs(self) D.configs.setReadonly(True) dfa.states[D] = D - if self.debug: + if ParserATNSimulator.debug: print("adding new DFA state: " + str(D)) return D def reportAttemptingFullContext(self, dfa, conflictingAlts, configs, startIndex, stopIndex): - if self.debug or self.retry_debug: + if ParserATNSimulator.debug or ParserATNSimulator.retry_debug: interval = range(startIndex, stopIndex + 1) print("reportAttemptingFullContext decision=" + str(dfa.decision) + ":" + str(configs) + ", input=" + self.parser.getTokenStream().getText(interval)) @@ -1483,7 +1482,7 @@ class ParserATNSimulator(ATNSimulator): self.parser.getErrorListenerDispatch().reportAttemptingFullContext(self.parser, dfa, startIndex, stopIndex, conflictingAlts, configs) def reportContextSensitivity(self, dfa, prediction, configs, startIndex, stopIndex): - if self.debug or self.retry_debug: + if ParserATNSimulator.debug or ParserATNSimulator.retry_debug: interval = range(startIndex, stopIndex + 1) print("reportContextSensitivity decision=" + str(dfa.decision) + ":" + str(configs) + ", input=" + self.parser.getTokenStream().getText(interval)) @@ -1493,7 +1492,7 @@ class ParserATNSimulator(ATNSimulator): # If context sensitive parsing, we know it's ambiguity not conflict# def reportAmbiguity(self, dfa, D, startIndex, stopIndex, exact, ambigAlts, configs ): - if self.debug or self.retry_debug: + if ParserATNSimulator.debug or ParserATNSimulator.retry_debug: # ParserATNPathFinder finder = new ParserATNPathFinder(parser, atn); # int i = 1; # for (Transition t : dfa.atnStartState.transitions) { diff --git a/runtime/Python2/src/antlr4/atn/PredictionMode.py b/runtime/Python2/src/antlr4/atn/PredictionMode.py index c01641972..14280a5b7 100644 --- a/runtime/Python2/src/antlr4/atn/PredictionMode.py +++ b/runtime/Python2/src/antlr4/atn/PredictionMode.py @@ -467,7 +467,8 @@ class PredictionMode(object): def getUniqueAlt(cls, altsets): all = cls.getAlts(altsets) if len(all)==1: - return all[0] + for one in all: + return one else: return ATN.INVALID_ALT_NUMBER diff --git a/runtime/Python3/src/antlr4/PredictionContext.py b/runtime/Python3/src/antlr4/PredictionContext.py index a6e643ea4..6cfc08052 100644 --- a/runtime/Python3/src/antlr4/PredictionContext.py +++ b/runtime/Python3/src/antlr4/PredictionContext.py @@ -95,7 +95,7 @@ def calculateHashCode(parent:PredictionContext, returnState:int): def calculateListsHashCode(parents:[], returnStates:[] ): h = 0 - for parent, returnState in parents, returnStates: + for parent, returnState in zip(parents, returnStates): h = hash((h, calculateHashCode(parent, returnState))) return h @@ -242,7 +242,7 @@ class ArrayPredictionContext(PredictionContext): if self.returnStates[i]==PredictionContext.EMPTY_RETURN_STATE: buf.write("$") continue - buf.write(self.returnStates[i]) + buf.write(str(self.returnStates[i])) if self.parents[i] is not None: buf.write(' ') buf.write(str(self.parents[i])) @@ -251,6 +251,10 @@ class ArrayPredictionContext(PredictionContext): buf.write("]") return buf.getvalue() + def __hash__(self): + return self.cachedHashCode + + # Convert a {@link RuleContext} tree to a {@link PredictionContext} graph. # Return {@link #EMPTY} if {@code outerContext} is empty or null. @@ -325,18 +329,18 @@ def merge(a:PredictionContext, b:PredictionContext, rootIsWildcard:bool, mergeCa #/ def mergeSingletons(a:SingletonPredictionContext, b:SingletonPredictionContext, rootIsWildcard:bool, mergeCache:dict): if mergeCache is not None: - previous = mergeCache.get(a,b) + previous = mergeCache.get((a,b), None) if previous is not None: return previous - previous = mergeCache.get(b,a) + previous = mergeCache.get((b,a), None) if previous is not None: return previous - rootMerge = mergeRoot(a, b, rootIsWildcard) - if rootMerge is not None: + merged = mergeRoot(a, b, rootIsWildcard) + if merged is not None: if mergeCache is not None: - mergeCache.put(a, b, rootMerge) - return rootMerge + mergeCache[(a, b)] = merged + return merged if a.returnState==b.returnState: parent = merge(a.parentCtx, b.parentCtx, rootIsWildcard, mergeCache) @@ -349,10 +353,10 @@ def mergeSingletons(a:SingletonPredictionContext, b:SingletonPredictionContext, # merge parents x and y, giving array node with x,y then remainders # of those graphs. dup a, a' points at merged array # new joined parent so create new singleton pointing to it, a' - a_ = SingletonPredictionContext.create(parent, a.returnState) + merged = SingletonPredictionContext.create(parent, a.returnState) if mergeCache is not None: - mergeCache.put(a, b, a_) - return a_ + mergeCache[(a, b)] = merged + return merged else: # a != b payloads differ # see if we can collapse parents due to $+x parents if local ctx singleParent = None @@ -362,26 +366,24 @@ def mergeSingletons(a:SingletonPredictionContext, b:SingletonPredictionContext, # sort payloads and use same parent payloads = [ a.returnState, b.returnState ] if a.returnState > b.returnState: - payloads[0] = b.returnState - payloads[1] = a.returnState + payloads = [ b.returnState, a.returnState ] parents = [singleParent, singleParent] - a_ = ArrayPredictionContext(parents, payloads) + merged = ArrayPredictionContext(parents, payloads) if mergeCache is not None: - mergeCache.put(a, b, a_) - return a_ + mergeCache[(a, b)] = merged + return merged # parents differ and can't merge them. Just pack together # into array; can't merge. # ax + by = [ax,by] payloads = [ a.returnState, b.returnState ] parents = [ a.parentCtx, b.parentCtx ] if a.returnState > b.returnState: # sort by payload - payloads[0] = b.returnState - payloads[1] = a.returnState + payloads = [ b.returnState, a.returnState ] parents = [ b.parentCtx, a.parentCtx ] - a_ = ArrayPredictionContext(parents, payloads) + merged = ArrayPredictionContext(parents, payloads) if mergeCache is not None: - mergeCache.put(a, b, a_) - return a_ + mergeCache[(a, b)] = merged + return merged # @@ -463,10 +465,10 @@ def mergeRoot(a:SingletonPredictionContext, b:SingletonPredictionContext, rootIs #/ def mergeArrays(a:ArrayPredictionContext, b:ArrayPredictionContext, rootIsWildcard:bool, mergeCache:dict): if mergeCache is not None: - previous = mergeCache.get(a,b) + previous = mergeCache.get((a,b), None) if previous is not None: return previous - previous = mergeCache.get(b,a) + previous = mergeCache.get((b,a), None) if previous is not None: return previous @@ -475,8 +477,8 @@ def mergeArrays(a:ArrayPredictionContext, b:ArrayPredictionContext, rootIsWildca j = 0 # walks b k = 0 # walks target M array - mergedReturnStates = [] * (len(a.returnState) + len( b.returnStates)) - mergedParents = [] * len(mergedReturnStates) + mergedReturnStates = [None] * (len(a.returnStates) + len( b.returnStates)) + mergedParents = [None] * len(mergedReturnStates) # walk and merge to yield mergedParents, mergedReturnStates while itrg, src->trg, we keep track of the previous trg to # avoid looking up the DFA state again, which is expensive. @@ -229,8 +229,8 @@ class LexerATNSimulator(ATNSimulator): return None target = s.edges[t - self.MIN_DFA_EDGE] - if self.debug and target is not None: - print("reuse state "+s.stateNumber+ " edge to "+target.stateNumber) + if LexerATNSimulator.debug and target is not None: + print("reuse state", str(s.stateNumber), "edge to", str(target.stateNumber)) return target @@ -286,8 +286,8 @@ class LexerATNSimulator(ATNSimulator): if currentAltReachedAcceptState and cfg.passedThroughNonGreedyDecision: continue - if self.debug: - print("testing %s at %s\n", self.getTokenName(t), cfg.toString(self.recog, True)) + if LexerATNSimulator.debug: + print("testing", self.getTokenName(t), "at", str(cfg)) for trans in cfg.state.transitions: # for each transition target = self.getReachableTarget(trans, t) @@ -304,8 +304,8 @@ class LexerATNSimulator(ATNSimulator): skipAlt = cfg.alt def accept(self, input:InputStream, lexerActionExecutor:LexerActionExecutor, startIndex:int, index:int, line:int, charPos:int): - if self.debug: - print("ACTION %s\n", lexerActionExecutor) + if LexerATNSimulator.debug: + print("ACTION", lexerActionExecutor) # seek to after last char in token input.seek(index) @@ -340,15 +340,15 @@ class LexerATNSimulator(ATNSimulator): # {@code false}. def closure(self, input:InputStream, config:LexerATNConfig, configs:ATNConfigSet, currentAltReachedAcceptState:bool, speculative:bool, treatEofAsEpsilon:bool): - if self.debug: - print("closure("+config.toString(self.recog, True)+")") + if LexerATNSimulator.debug: + print("closure(" + str(config) + ")") if isinstance( config.state, RuleStopState ): - if self.debug: + if LexerATNSimulator.debug: if self.recog is not None: - print("closure at %s rule stop %s\n", self.recog.getRuleNames()[config.state.ruleIndex], config) + print("closure at", self.recog.symbolicNames[config.state.ruleIndex], "rule stop", str(config)) else: - print("closure at rule stop %s\n", config) + print("closure at rule stop", str(config)) if config.context is None or config.context.hasEmptyPath(): if config.context is None or config.context.isEmpty(): @@ -411,7 +411,7 @@ class LexerATNSimulator(ATNSimulator): # states reached by traversing predicates. Since this is when we # test them, we cannot cash the DFA state target of ID. - if self.debug: + if LexerATNSimulator.debug: print("EVAL rule "+ str(t.ruleIndex) + ":" + str(t.predIndex)) configs.hasSemanticContext = True if self.evaluatePredicate(input, t.ruleIndex, t.predIndex, speculative): @@ -523,7 +523,7 @@ class LexerATNSimulator(ATNSimulator): # Only track edges within the DFA bounds return to - if self.debug: + if LexerATNSimulator.debug: print("EDGE " + str(from_) + " -> " + str(to) + " upon "+ chr(tk)) if from_.edges is None: diff --git a/runtime/Python3/src/antlr4/atn/ParserATNSimulator.py b/runtime/Python3/src/antlr4/atn/ParserATNSimulator.py index fee18087f..7bcaa3912 100755 --- a/runtime/Python3/src/antlr4/atn/ParserATNSimulator.py +++ b/runtime/Python3/src/antlr4/atn/ParserATNSimulator.py @@ -313,7 +313,7 @@ class ParserATNSimulator(ATNSimulator): pass def adaptivePredict(self, input:TokenStream, decision:int, outerContext:ParserRuleContext): - if self.debug or self.debug_list_atn_decisions: + if ParserATNSimulator.debug or ParserATNSimulator.debug_list_atn_decisions: print("adaptivePredict decision " + str(decision) + " exec LA(1)==" + self.getLookaheadName(input) + " line " + str(input.LT(1).line) + ":" + @@ -341,10 +341,10 @@ class ParserATNSimulator(ATNSimulator): if s0 is None: if outerContext is None: outerContext = ParserRuleContext.EMPTY - if self.debug or self.debug_list_atn_decisions: + if ParserATNSimulator.debug or ParserATNSimulator.debug_list_atn_decisions: print("predictATN decision " + str(dfa.decision) + " exec LA(1)==" + self.getLookaheadName(input) + - ", outerContext=" + outerContext.toString(self.parser)) + ", outerContext=" + outerContext.toString(self.parser.literalNames, None)) # If this is not a precedence DFA, we check the ATN start state # to determine if this ATN start state is the decision for the @@ -373,8 +373,8 @@ class ParserATNSimulator(ATNSimulator): dfa.s0 = s0 alt = self.execATN(dfa, s0, input, index, outerContext) - if self.debug: - print("DFA after predictATN: " + dfa.toString(self.parser.tokenNames)) + if ParserATNSimulator.debug: + print("DFA after predictATN: " + dfa.toString(self.parser.literalNames)) return alt finally: self._dfa = None @@ -413,14 +413,14 @@ class ParserATNSimulator(ATNSimulator): # conflict + preds # def execATN(self, dfa:DFA, s0:DFAState, input:TokenStream, startIndex:int, outerContext:ParserRuleContext ): - if self.debug or self.debug_list_atn_decisions: + if ParserATNSimulator.debug or ParserATNSimulator.debug_list_atn_decisions: print("execATN decision " + str(dfa.decision) + " exec LA(1)==" + self.getLookaheadName(input) + " line " + str(input.LT(1).line) + ":" + str(input.LT(1).column)) previousD = s0 - if self.debug: + if ParserATNSimulator.debug: print("s0 = " + str(s0)) t = input.LA(1) @@ -450,7 +450,7 @@ class ParserATNSimulator(ATNSimulator): # IF PREDS, MIGHT RESOLVE TO SINGLE ALT => SLL (or syntax error) conflictingAlts = None if D.predicates is not None: - if self.debug: + if ParserATNSimulator.debug: print("DFA state has preds in DFA sim LL failover") conflictIndex = input.index if conflictIndex != startIndex: @@ -458,7 +458,7 @@ class ParserATNSimulator(ATNSimulator): conflictingAlts = self.evalSemanticContext(D.predicates, outerContext, True) if len(conflictingAlts)==1: - if self.debug: + if ParserATNSimulator.debug: print("Full LL avoided") return min(conflictingAlts) @@ -467,7 +467,7 @@ class ParserATNSimulator(ATNSimulator): # context occurs with the index at the correct spot input.seek(conflictIndex) - if self.dfa_debug: + if ParserATNSimulator.dfa_debug: print("ctx sensitive state " + str(outerContext) +" in " + str(D)) fullCtx = True s0_closure = self.computeStartState(dfa.atnStartState, outerContext, fullCtx) @@ -539,7 +539,7 @@ class ParserATNSimulator(ATNSimulator): predictedAlt = self.getUniqueAlt(reach) - if self.debug: + if ParserATNSimulator.debug: altSubSets = PredictionMode.getConflictingAltSubsets(reach) print("SLL altSubSets=" + str(altSubSets) + ", configs=" + str(reach) + ", predict=" + str(predictedAlt) + ", allSubsetsConflict=" + @@ -591,8 +591,8 @@ class ParserATNSimulator(ATNSimulator): input:TokenStream, startIndex:int, outerContext:ParserRuleContext): - if self.debug or self.debug_list_atn_decisions: - print("execATNWithFullContext "+s0) + if ParserATNSimulator.debug or ParserATNSimulator.debug_list_atn_decisions: + print("execATNWithFullContext", str(s0)) fullCtx = True foundExactAmbig = False reach = None @@ -621,7 +621,7 @@ class ParserATNSimulator(ATNSimulator): raise e altSubSets = PredictionMode.getConflictingAltSubsets(reach) - if self.debug: + if ParserATNSimulator.debug: print("LL altSubSets=" + str(altSubSets) + ", predict=" + str(PredictionMode.getUniqueAlt(altSubSets)) + ", resolvesToJustOneViableAlt=" + str(PredictionMode.resolvesToJustOneViableAlt(altSubSets))) @@ -690,7 +690,7 @@ class ParserATNSimulator(ATNSimulator): return predictedAlt def computeReachSet(self, closure:ATNConfigSet, t:int, fullCtx:bool): - if self.debug: + if ParserATNSimulator.debug: print("in computeReachSet, starting closure: " + str(closure)) if self.mergeCache is None: @@ -712,7 +712,7 @@ class ParserATNSimulator(ATNSimulator): # First figure out where we can reach on input t for c in closure: - if self.debug: + if ParserATNSimulator.debug: print("testing " + self.getTokenName(t) + " at " + str(c)) if isinstance(c.state, RuleStopState): @@ -972,7 +972,7 @@ class ParserATNSimulator(ATNSimulator): # nonambig alts are null in altToPred if nPredAlts==0: altToPred = None - if self.debug: + if ParserATNSimulator.debug: print("getPredsForAmbigAlts result " + str_list(altToPred)) return altToPred @@ -1098,11 +1098,11 @@ class ParserATNSimulator(ATNSimulator): break continue predicateEvaluationResult = pair.pred.eval(self.parser, outerContext) - if self.debug or self.dfa_debug: + if ParserATNSimulator.debug or ParserATNSimulator.dfa_debug: print("eval pred " + str(pair) + "=" + str(predicateEvaluationResult)) if predicateEvaluationResult: - if self.debug or self.dfa_debug: + if ParserATNSimulator.debug or ParserATNSimulator.dfa_debug: print("PREDICT " + str(pair.alt)) predictions.add(pair.alt) if not complete: @@ -1124,8 +1124,8 @@ class ParserATNSimulator(ATNSimulator): def closureCheckingStopState(self, config:ATNConfig, configs:ATNConfigSet, closureBusy:set, collectPredicates:bool, fullCtx:bool, depth:int, treatEofAsEpsilon:bool): - if self.debug: - print("closure(" + config.toString(self.parser,True) + ")") + if ParserATNSimulator.debug: + print("closure(" + str(config) + ")") if isinstance(config.state, RuleStopState): # We hit rule end. If we have context info, use it @@ -1139,7 +1139,7 @@ class ParserATNSimulator(ATNSimulator): continue else: # we have no context info, just chase follow links (if greedy) - if self.debug: + if ParserATNSimulator.debug: print("FALLING off rule " + self.getRuleName(config.state.ruleIndex)) self.closure_(config, configs, closureBusy, collectPredicates, fullCtx, depth, treatEofAsEpsilon) @@ -1159,7 +1159,7 @@ class ParserATNSimulator(ATNSimulator): return else: # else if we have no context info, just chase follow links (if greedy) - if self.debug: + if ParserATNSimulator.debug: print("FALLING off rule " + self.getRuleName(config.state.ruleIndex)) self.closure_(config, configs, closureBusy, collectPredicates, fullCtx, depth, treatEofAsEpsilon) @@ -1201,7 +1201,7 @@ class ParserATNSimulator(ATNSimulator): c.reachesIntoOuterContext += 1 configs.dipsIntoOuterContext = True # TODO: can remove? only care when we add to set per middle of this method newDepth -= 1 - if self.debug: + if ParserATNSimulator.debug: print("dips into outer ctx: " + str(c)) elif isinstance(t, RuleTransition): # latch when newDepth goes negative - once we step out of the entry context we can't return @@ -1242,12 +1242,12 @@ class ParserATNSimulator(ATNSimulator): return m(self, config, t, collectPredicates, inContext, fullCtx, treatEofAsEpsilon) def actionTransition(self, config:ATNConfig, t:ActionTransition): - if self.debug: + if ParserATNSimulator.debug: print("ACTION edge " + str(t.ruleIndex) + ":" + str(t.actionIndex)) return ATNConfig(state=t.target, config=config) def precedenceTransition(self, config:ATNConfig, pt:PrecedencePredicateTransition, collectPredicates:bool, inContext:bool, fullCtx:bool): - if self.debug: + if ParserATNSimulator.debug: print("PRED (collectPredicates=" + str(collectPredicates) + ") " + str(pt.precedence) + ">=_p, ctx dependent=true") if self.parser is not None: @@ -1272,12 +1272,12 @@ class ParserATNSimulator(ATNSimulator): else: c = ATNConfig(state=pt.target, config=config) - if self.debug: + if ParserATNSimulator.debug: print("config from pred transition=" + str(c)) return c def predTransition(self, config:ATNConfig, pt:PredicateTransition, collectPredicates:bool, inContext:bool, fullCtx:bool): - if self.debug: + if ParserATNSimulator.debug: print("PRED (collectPredicates=" + str(collectPredicates) + ") " + str(pt.ruleIndex) + ":" + str(pt.predIndex) + ", ctx dependent=" + str(pt.isCtxDependent)) if self.parser is not None: @@ -1302,12 +1302,12 @@ class ParserATNSimulator(ATNSimulator): else: c = ATNConfig(state=pt.target, config=config) - if self.debug: + if ParserATNSimulator.debug: print("config from pred transition=" + str(c)) return c def ruleTransition(self, config:ATNConfig, t:RuleTransition): - if self.debug: + if ParserATNSimulator.debug: print("CALL rule " + self.getRuleName(t.target.ruleIndex) + ", ctx=" + str(config.context)) returnState = t.followState newContext = SingletonPredictionContext.create(config.context, returnState.stateNumber) @@ -1365,13 +1365,12 @@ class ParserATNSimulator(ATNSimulator): def getTokenName(self, t:int): if t==Token.EOF: return "EOF" - if self.parser is not None and self.parser.tokenNames is not None: - if t >= len(self.parser.tokenNames): - print(str(t) + " ttype out of range: " + str_list(self.parser.tokenNames)) - print(str_list(self.parser.getInputStream().getTokens())) - else: - return self.parser.tokensNames[t] + "<" + str(t) + ">" - return str(t) + if self.parser is not None and \ + self.parser.literalNames is not None and \ + t < len(self.parser.literalNames): + return self.parser.literalNames[t] + "<" + str(t) + ">" + else: + return str(t) def getLookaheadName(self, input:TokenStream): return self.getTokenName(input.LA(1)) @@ -1426,7 +1425,7 @@ class ParserATNSimulator(ATNSimulator): # on {@code to} # def addDFAEdge(self, dfa:DFA, from_:DFAState, t:int, to:DFAState): - if self.debug: + if ParserATNSimulator.debug: print("EDGE " + str(from_) + " -> " + str(to) + " upon " + self.getTokenName(t)) if to is None: @@ -1440,8 +1439,8 @@ class ParserATNSimulator(ATNSimulator): from_.edges = [None] * (self.atn.maxTokenType + 2) from_.edges[t+1] = to # connect - if self.debug: - names = None if self.parser is None else self.parser.tokenNames + if ParserATNSimulator.debug: + names = None if self.parser is None else self.parser.literalNames print("DFA=\n" + dfa.toString(names)) return to @@ -1475,12 +1474,12 @@ class ParserATNSimulator(ATNSimulator): D.configs.optimizeConfigs(self) D.configs.setReadonly(True) dfa.states[D] = D - if self.debug: + if ParserATNSimulator.debug: print("adding new DFA state: " + str(D)) return D def reportAttemptingFullContext(self, dfa:DFA, conflictingAlts:set, configs:ATNConfigSet, startIndex:int, stopIndex:int): - if self.debug or self.retry_debug: + if ParserATNSimulator.debug or ParserATNSimulator.retry_debug: interval = range(startIndex, stopIndex + 1) print("reportAttemptingFullContext decision=" + str(dfa.decision) + ":" + str(configs) + ", input=" + self.parser.getTokenStream().getText(interval)) @@ -1488,7 +1487,7 @@ class ParserATNSimulator(ATNSimulator): self.parser.getErrorListenerDispatch().reportAttemptingFullContext(self.parser, dfa, startIndex, stopIndex, conflictingAlts, configs) def reportContextSensitivity(self, dfa:DFA, prediction:int, configs:ATNConfigSet, startIndex:int, stopIndex:int): - if self.debug or self.retry_debug: + if ParserATNSimulator.debug or ParserATNSimulator.retry_debug: interval = range(startIndex, stopIndex + 1) print("reportContextSensitivity decision=" + str(dfa.decision) + ":" + str(configs) + ", input=" + self.parser.getTokenStream().getText(interval)) @@ -1498,7 +1497,7 @@ class ParserATNSimulator(ATNSimulator): # If context sensitive parsing, we know it's ambiguity not conflict# def reportAmbiguity(self, dfa:DFA, D:DFAState, startIndex:int, stopIndex:int, exact:bool, ambigAlts:set, configs:ATNConfigSet ): - if self.debug or self.retry_debug: + if ParserATNSimulator.debug or ParserATNSimulator.retry_debug: # ParserATNPathFinder finder = new ParserATNPathFinder(parser, atn); # int i = 1; # for (Transition t : dfa.atnStartState.transitions) { diff --git a/runtime/Python3/src/antlr4/atn/PredictionMode.py b/runtime/Python3/src/antlr4/atn/PredictionMode.py index 0fba6351f..03ca896c7 100644 --- a/runtime/Python3/src/antlr4/atn/PredictionMode.py +++ b/runtime/Python3/src/antlr4/atn/PredictionMode.py @@ -470,7 +470,8 @@ class PredictionMode(Enum): def getUniqueAlt(cls, altsets:list): all = cls.getAlts(altsets) if len(all)==1: - return all[0] + for one in all: + return one else: return ATN.INVALID_ALT_NUMBER From a51b70995403cbb6ee26fc67759e56f2fa8c005a Mon Sep 17 00:00:00 2001 From: Burt Harris Date: Thu, 11 Aug 2016 12:56:34 -0700 Subject: [PATCH 86/95] Fix .gitignore for target language directory Also added .gitignore entries for Node.js & typescript dependency cache directories. --- .gitignore | 5 +++++ contributors.txt | 1 + 2 files changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index 2684429fc..89d66784e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,10 @@ # Maven build folders target/ +!tool/src/org/antlr/v4/codegen/target/ + +# Node.js (npm and typings) cached dependencies +node_modules/ +typings/ # Ant build folders build/ diff --git a/contributors.txt b/contributors.txt index 687eeaf20..64409232b 100644 --- a/contributors.txt +++ b/contributors.txt @@ -92,3 +92,4 @@ YYYY/MM/DD, github id, Full name, email 2016/03/27, beardlybread, Bradley Steinbacher, bradley.j.steinbacher@gmail.com 2016/03/29, msteiger, Martin Steiger, antlr@martin-steiger.de 2016/03/28, gagern, Martin von Gagern, gagern@ma.tum.de +2016/08/11, BurtHarris, Ralph "Burt" Harris, Burt_Harris_antlr4@azxs.33mail.com From 89987e35792809d2f5e76f9b60d28a64644a8049 Mon Sep 17 00:00:00 2001 From: Burt Harris Date: Thu, 11 Aug 2016 13:08:02 -0700 Subject: [PATCH 87/95] + comment in .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 89d66784e..951adb468 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Maven build folders target/ +# ... but not code generation targets !tool/src/org/antlr/v4/codegen/target/ # Node.js (npm and typings) cached dependencies From 71a2a1295fb0b047dc1d23b50b15e09c38410e67 Mon Sep 17 00:00:00 2001 From: Andreas Jonson Date: Fri, 19 Aug 2016 22:22:05 +0200 Subject: [PATCH 88/95] update the runtime paths to work in windows the old paths contained a / at the beginning of the path e.g. "/C:/" but the expected path is "C:/" and due to this all test for the targets javascript, python and c# failed on windows. --- .../test/org/antlr/v4/test/runtime/csharp/BaseTest.java | 3 +++ .../antlr/v4/test/runtime/javascript/node/BaseTest.java | 7 +++++++ .../org/antlr/v4/test/runtime/python/BasePythonTest.java | 7 +++++++ 3 files changed, 17 insertions(+) diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseTest.java index 23eb6eca4..14848a7aa 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseTest.java @@ -482,6 +482,9 @@ public abstract class BaseTest { throw new RuntimeException("C# runtime project file not found!"); } String runtimeProjPath = runtimeProj.getPath(); + if(isWindows()){ + runtimeProjPath = runtimeProjPath.replaceFirst("/", ""); + } XPathExpression exp = XPathFactory.newInstance().newXPath() .compile("/Project/ItemGroup/ProjectReference[@Include='" + runtimeName + "']"); Element node = (Element)exp.evaluate(prjXml, XPathConstants.NODE); diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/node/BaseTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/node/BaseTest.java index 59b68d14c..8db0f0716 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/node/BaseTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/node/BaseTest.java @@ -445,9 +445,16 @@ public abstract class BaseTest { if ( runtimeSrc==null ) { throw new RuntimeException("Cannot find JavaScript runtime"); } + if(isWindows()){ + return runtimeSrc.getPath().replaceFirst("/", ""); + } return runtimeSrc.getPath(); } + private boolean isWindows() { + return System.getProperty("os.name").toLowerCase().contains("windows"); + } + public void testErrors(String[] pairs, boolean printTree) { for (int i = 0; i < pairs.length; i += 2) { String input = pairs[i]; diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/python/BasePythonTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/python/BasePythonTest.java index ab91d9b02..f221f3008 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/python/BasePythonTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/python/BasePythonTest.java @@ -583,9 +583,16 @@ public abstract class BasePythonTest { if ( runtimeSrc==null ) { throw new RuntimeException("Cannot find "+targetName+" runtime"); } + if(isWindows()){ + return runtimeSrc.getPath().replaceFirst("/", ""); + } return runtimeSrc.getPath(); } + private boolean isWindows() { + return System.getProperty("os.name").toLowerCase().contains("windows"); + } + public void testErrors(String[] pairs, boolean printTree) { for (int i = 0; i < pairs.length; i+=2) { String input = pairs[i]; From 6e780c90bb7d453d76b87bb5ab1540d4cb4e4718 Mon Sep 17 00:00:00 2001 From: Andreas Jonson Date: Fri, 19 Aug 2016 22:22:05 +0200 Subject: [PATCH 89/95] update the runtime paths to work in windows the old paths contained a / at the beginning of the path e.g. "/C:/" but the expected path is "C:/" and due to this all test for the targets javascript, python and c# failed on windows. --- contributors.txt | 1 + .../test/org/antlr/v4/test/runtime/csharp/BaseTest.java | 3 +++ .../antlr/v4/test/runtime/javascript/node/BaseTest.java | 7 +++++++ .../org/antlr/v4/test/runtime/python/BasePythonTest.java | 7 +++++++ 4 files changed, 18 insertions(+) diff --git a/contributors.txt b/contributors.txt index 687eeaf20..7d7311c82 100644 --- a/contributors.txt +++ b/contributors.txt @@ -92,3 +92,4 @@ YYYY/MM/DD, github id, Full name, email 2016/03/27, beardlybread, Bradley Steinbacher, bradley.j.steinbacher@gmail.com 2016/03/29, msteiger, Martin Steiger, antlr@martin-steiger.de 2016/03/28, gagern, Martin von Gagern, gagern@ma.tum.de +2016/08/19, andjo403, Andreas Jonson, andjo403@hotmail.com diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseTest.java index 23eb6eca4..14848a7aa 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseTest.java @@ -482,6 +482,9 @@ public abstract class BaseTest { throw new RuntimeException("C# runtime project file not found!"); } String runtimeProjPath = runtimeProj.getPath(); + if(isWindows()){ + runtimeProjPath = runtimeProjPath.replaceFirst("/", ""); + } XPathExpression exp = XPathFactory.newInstance().newXPath() .compile("/Project/ItemGroup/ProjectReference[@Include='" + runtimeName + "']"); Element node = (Element)exp.evaluate(prjXml, XPathConstants.NODE); diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/node/BaseTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/node/BaseTest.java index 59b68d14c..8db0f0716 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/node/BaseTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/javascript/node/BaseTest.java @@ -445,9 +445,16 @@ public abstract class BaseTest { if ( runtimeSrc==null ) { throw new RuntimeException("Cannot find JavaScript runtime"); } + if(isWindows()){ + return runtimeSrc.getPath().replaceFirst("/", ""); + } return runtimeSrc.getPath(); } + private boolean isWindows() { + return System.getProperty("os.name").toLowerCase().contains("windows"); + } + public void testErrors(String[] pairs, boolean printTree) { for (int i = 0; i < pairs.length; i += 2) { String input = pairs[i]; diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/python/BasePythonTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/python/BasePythonTest.java index ab91d9b02..f221f3008 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/python/BasePythonTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/python/BasePythonTest.java @@ -583,9 +583,16 @@ public abstract class BasePythonTest { if ( runtimeSrc==null ) { throw new RuntimeException("Cannot find "+targetName+" runtime"); } + if(isWindows()){ + return runtimeSrc.getPath().replaceFirst("/", ""); + } return runtimeSrc.getPath(); } + private boolean isWindows() { + return System.getProperty("os.name").toLowerCase().contains("windows"); + } + public void testErrors(String[] pairs, boolean printTree) { for (int i = 0; i < pairs.length; i+=2) { String input = pairs[i]; From d3ceee50430f77e2c3672f2ae925e29b28400186 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Sat, 24 Sep 2016 00:02:18 +0800 Subject: [PATCH 90/95] Enhancing the doc --- doc/adding-tests.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/doc/adding-tests.md b/doc/adding-tests.md index 769383d31..4da6d4093 100644 --- a/doc/adding-tests.md +++ b/doc/adding-tests.md @@ -63,7 +63,22 @@ TestTemplates ::= [ ... ``` -For every name mentioned, you will find a `.stg` file with the actual test. E.g., `Sets/StarSet.stg`: +For every name mentioned, you will find a `.stg` file with the actual test template. E.g., `Sets/StarSet.stg`. + +Each `.stg` file descripes the following mandatory elements for the test: + - the test type: "Parser" or "Lexer" + - some ANTLR options, such as "Debug" + - the grammar + - the start rule + - the input i.e. the text to parse + - the expected output + - the expected errors + +The grammar can itself contain template expressions such as . +The test generator replaces these with the corresponding values from the target language template (see below). +It then generates a unit test in which the grammar, the input and the expected output and errors are inlined. + +Here is an example test template: ``` TestType() ::= "Parser" @@ -92,6 +107,7 @@ a : ('a'|'b')* 'c' {} ; >> ``` + ### Cross-language actions embedded within grammars To get: From 2f92b475003e1dd4254a519a88bf10c3667de1c6 Mon Sep 17 00:00:00 2001 From: parrt Date: Fri, 23 Sep 2016 13:19:44 -0700 Subject: [PATCH 91/95] tweak links in doc --- README.md | 12 ++++++------ doc/building-antlr.md | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index c1d6902de..e743269a7 100644 --- a/README.md +++ b/README.md @@ -15,18 +15,18 @@ ANTLR project lead and supreme dictator for life ## Useful information * [Release notes](https://github.com/antlr/antlr4/releases) -* [Getting started with v4](https://raw.githubusercontent.com/antlr/antlr4/master/doc/getting-started.md) +* [Getting started with v4](https://github.com/antlr/antlr4/blob/master/doc/getting-started.md) * [Official site](http://www.antlr.org/) -* [Documentation](https://raw.githubusercontent.com/antlr/antlr4/master/doc/index.md) -* [FAQ](https://raw.githubusercontent.com/antlr/antlr4/master/doc/faq/index.md) +* [Documentation](https://github.com/antlr/antlr4/blob/master/doc/index.md) +* [FAQ](https://github.com/antlr/antlr4/blob/master/doc/faq/index.md) * [API](http://www.antlr.org/api/Java/index.html) * [ANTLR v3](http://www.antlr3.org/) -* [v3 to v4 Migration, differences](https://raw.githubusercontent.com/antlr/antlr4/master/doc/faq/general.md) +* [v3 to v4 Migration, differences](https://github.com/antlr/antlr4/blob/master/doc/faq/general.md) You might also find the following pages useful, particularly if you want to mess around with the various target languages. -* [How to build ANTLR itself](https://raw.githubusercontent.com/antlr/antlr4/master/doc/building-antlr.md) -* [How we create and deploy an ANTLR release](https://raw.githubusercontent.com/antlr/antlr4/master/doc/releasing-antlr.md) +* [How to build ANTLR itself](https://github.com/antlr/antlr4/blob/master/doc/building-antlr.md) +* [How we create and deploy an ANTLR release](https://github.com/antlr/antlr4/blob/master/doc/releasing-antlr.md) ## The Definitive ANTLR 4 Reference diff --git a/doc/building-antlr.md b/doc/building-antlr.md index f6780d970..b054438d1 100644 --- a/doc/building-antlr.md +++ b/doc/building-antlr.md @@ -13,14 +13,14 @@ The first step is to get the Java source code from the ANTLR 4 repository at git ```bash $ cd /tmp -/tmp $ git clone git@github.com:antlr/antlr4.git +/tmp $ git clone https://github.com/antlr/antlr4.git Cloning into 'antlr4'... -remote: Counting objects: 43273, done. -remote: Compressing objects: 100% (57/57), done. -remote: Total 43273 (delta 26), reused 0 (delta 0) -Receiving objects: 100% (43273/43273), 18.76 MiB | 1.60 MiB/s, done. -Resolving deltas: 100% (22419/22419), done. +remote: Counting objects: 61480, done. +remote: Total 61480 (delta 0), reused 0 (delta 0), pack-reused 61480 +Receiving objects: 100% (61480/61480), 31.24 MiB | 7.18 MiB/s, done. +Resolving deltas: 100% (32970/32970), done. Checking connectivity... done. +Checking out files: 100% (1427/1427), done. ``` # Compile From 8e44b19ec92dc4bfc47a7acae14bc326a6ebb970 Mon Sep 17 00:00:00 2001 From: Burt Harris Date: Fri, 23 Sep 2016 14:58:21 -0700 Subject: [PATCH 92/95] Update .gitignroe fules for VisualStudio etc. Avoids temp and user-preference files from getting commited --- .gitignore | 39 +++++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 951adb468..d8996ae04 100644 --- a/.gitignore +++ b/.gitignore @@ -16,15 +16,38 @@ user.build.properties # MacOSX files .DS_Store -# Python -*.pyc +## Python, selected lines from https://raw.githubusercontent.com/github/gitignore/master/Python.gitignore +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class -# CSharp -bin/ -obj/ +## CSharp and VisualStudio, selected lines from https://raw.githubusercontent.com/github/gitignore/master/VisualStudio.gitignore +# User-specific files +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) *.userprefs -# NetBeans user configuration files +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ + +# Visual Studio 2015 cache/options directory +.vs/ + +## NetBeans user configuration files nbactions*.xml /nbproject/private/ */nbproject/private/ @@ -63,3 +86,7 @@ bilder.pyc bild.log bild_output.txt + +# VSCode Java plugin temporary files +javac-services.0.log +javac-services.0.log.lck From 10f5ebe22a51cfdcc69b48e1a9a7de37a83afde5 Mon Sep 17 00:00:00 2001 From: Burt Harris Date: Fri, 23 Sep 2016 15:02:38 -0700 Subject: [PATCH 93/95] Repair comment --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d8996ae04..cece5e29c 100644 --- a/.gitignore +++ b/.gitignore @@ -47,7 +47,7 @@ bld/ # Visual Studio 2015 cache/options directory .vs/ -## NetBeans user configuration files +# NetBeans user configuration files nbactions*.xml /nbproject/private/ */nbproject/private/ From 0b2d5304a5176531edd249d143dca14b099b6d0d Mon Sep 17 00:00:00 2001 From: parrt Date: Sat, 24 Sep 2016 11:20:13 -0700 Subject: [PATCH 94/95] tweak links in doc --- doc/faq/general.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/faq/general.md b/doc/faq/general.md index fb9c386f5..9189ac9ab 100644 --- a/doc/faq/general.md +++ b/doc/faq/general.md @@ -68,9 +68,9 @@ expr : expr '*' expr ; ``` -ANTLR 4 automatically constructs parse trees for you and abstract syntax tree (AST) construction is no longer an option. See also What if I need ASTs not parse trees for a compiler, for example? +ANTLR 4 automatically constructs parse trees for you and abstract syntax tree (AST) construction is no longer an option. See also [What if I need ASTs not parse trees for a compiler, for example?](https://github.com/antlr/antlr4/blob/master/doc/faq/parse-trees.md#what-if-i-need-asts-not-parse-trees-for-a-compiler-for-example). -Another big difference is that we discourage the use of actions directly within the grammar because ANTLR 4 automatically generates [listeners and visitors](https://raw.githubusercontent.com/antlr/antlr4/master/doc/listeners.md) for you to use that trigger method calls when some phrases of interest are recognized during a tree walk after parsing. See also [Parse Tree Matching and XPath](https://raw.githubusercontent.com/antlr/antlr4/master/doc/tree-matching.md). +Another big difference is that we discourage the use of actions directly within the grammar because ANTLR 4 automatically generates [listeners and visitors](https://github.com/antlr/antlr4/blob/master/doc/listeners.md) for you to use that trigger method calls when some phrases of interest are recognized during a tree walk after parsing. See also [Parse Tree Matching and XPath](https://github.com/antlr/antlr4/blob/master/doc/tree-matching.md). Semantic predicates are still allowed in both the parser and lexer rules as our actions. For efficiency sake keep semantic predicates to the right edge of lexical rules. From 03d5fb1244268fff521df5c2e7d89c6d1361d5a7 Mon Sep 17 00:00:00 2001 From: Renata Hodovan Date: Thu, 11 Aug 2016 00:49:45 +0200 Subject: [PATCH 95/95] Make Python targets more pythonic. The Python implementations are completely synchronous with the Java version even if some of the constructs can be expressed with simpler Python solutions. These are typically the all, any, count, next builtins or the list comprehensions, etc. Beside using them makes the code clearer, they are also prefered by the standard and can result in performance speedup. The patch contains such equivalent transformations in the Python targets. --- runtime/Python2/src/antlr4/IntervalSet.py | 12 ++--- runtime/Python2/src/antlr4/ListTokenSource.py | 4 +- .../Python2/src/antlr4/PredictionContext.py | 5 +- .../Python2/src/antlr4/atn/ATNConfigSet.py | 23 +++------ runtime/Python2/src/antlr4/atn/ATNState.py | 5 +- .../src/antlr4/atn/LexerATNSimulator.py | 6 +-- .../Python2/src/antlr4/atn/PredictionMode.py | 47 +++++------------- .../Python2/src/antlr4/atn/SemanticContext.py | 37 ++++---------- runtime/Python2/src/antlr4/dfa/DFAState.py | 9 +--- runtime/Python2/src/antlr4/tree/Trees.py | 3 +- runtime/Python2/src/antlr4/xpath/XPath.py | 17 ++----- runtime/Python3/src/antlr4/IntervalSet.py | 12 ++--- runtime/Python3/src/antlr4/ListTokenSource.py | 4 +- .../Python3/src/antlr4/PredictionContext.py | 5 +- .../Python3/src/antlr4/atn/ATNConfigSet.py | 23 +++------ runtime/Python3/src/antlr4/atn/ATNState.py | 5 +- .../src/antlr4/atn/LexerATNSimulator.py | 6 +-- .../Python3/src/antlr4/atn/PredictionMode.py | 48 +++++-------------- .../Python3/src/antlr4/atn/SemanticContext.py | 38 +++++---------- runtime/Python3/src/antlr4/dfa/DFAState.py | 9 +--- runtime/Python3/src/antlr4/tree/Trees.py | 3 +- runtime/Python3/src/antlr4/xpath/XPath.py | 16 ++----- 22 files changed, 88 insertions(+), 249 deletions(-) diff --git a/runtime/Python2/src/antlr4/IntervalSet.py b/runtime/Python2/src/antlr4/IntervalSet.py index e28a556d4..3a55a2631 100644 --- a/runtime/Python2/src/antlr4/IntervalSet.py +++ b/runtime/Python2/src/antlr4/IntervalSet.py @@ -97,16 +97,10 @@ class IntervalSet(object): if self.intervals is None: return False else: - for i in self.intervals: - if item in i: - return True - return False + return any(item in i for i in self.intervals) def __len__(self): - xlen = 0 - for i in self.intervals: - xlen += len(i) - return xlen + return sum(len(i) for i in self.intervals) def removeRange(self, v): if v.start==v.stop-1: @@ -126,7 +120,7 @@ class IntervalSet(object): # check for included range, remove it elif v.start<=i.start and v.stop>=i.stop: self.intervals.pop(k) - k = k - 1 # need another pass + k -= 1 # need another pass # check for lower boundary elif v.start1: - return True - return False + return any(len(alts) > 1 for alts in altsets) # # Determines if every alternative subset in {@code altsets} is equivalent. @@ -448,13 +436,9 @@ class PredictionMode(object): # @classmethod def allSubsetsEqual(cls, altsets): - first = None - for alts in altsets: - if first is None: - first = alts - elif not alts==first: - return False - return True + if not altsets: + return True + return all(alts == altsets[0] for alts in altsets[1:]) # # Returns the unique alternative predicted by all alternative subsets in @@ -467,10 +451,8 @@ class PredictionMode(object): def getUniqueAlt(cls, altsets): all = cls.getAlts(altsets) if len(all)==1: - for one in all: - return one - else: - return ATN.INVALID_ALT_NUMBER + return all.pop() + return ATN.INVALID_ALT_NUMBER # Gets the complete set of represented alternatives for a collection of # alternative subsets. This method returns the union of each {@link BitSet} @@ -481,10 +463,7 @@ class PredictionMode(object): # @classmethod def getAlts(cls, altsets): - all = set() - for alts in altsets: - all = all | alts - return all + return set.union(*altsets) # # This function gets the conflicting alt subsets from a configuration set. @@ -528,11 +507,7 @@ class PredictionMode(object): @classmethod def hasStateAssociatedWithOneAlt(cls, configs): - x = cls.getStateToAltMap(configs) - for alts in x.values(): - if len(alts)==1: - return True - return False + return any(len(alts) == 1 for alts in cls.getStateToAltMap(configs).values()) @classmethod def getSingleViableAlt(cls, altsets): diff --git a/runtime/Python2/src/antlr4/atn/SemanticContext.py b/runtime/Python2/src/antlr4/atn/SemanticContext.py index 021d4770c..b8fe7b1a0 100644 --- a/runtime/Python2/src/antlr4/atn/SemanticContext.py +++ b/runtime/Python2/src/antlr4/atn/SemanticContext.py @@ -115,14 +115,7 @@ def orContext(a, b): return result def filterPrecedencePredicates(collection): - result = [] - for context in collection: - if isinstance(context, PrecedencePredicate): - if result is None: - result = [] - result.append(context) - return result - + return [context for context in collection if isinstance(context, PrecedencePredicate)] class Predicate(SemanticContext): @@ -187,13 +180,11 @@ class AND(SemanticContext): def __init__(self, a, b): operands = set() if isinstance( a, AND): - for o in a.opnds: - operands.add(o) + operands.update(a.opnds) else: operands.add(a) if isinstance( b, AND): - for o in b.opnds: - operands.add(o) + operands.update(b.opnds) else: operands.add(b) @@ -203,7 +194,7 @@ class AND(SemanticContext): reduced = min(precedencePredicates) operands.add(reduced) - self.opnds = [ o for o in operands ] + self.opnds = list(operands) def __eq__(self, other): if self is other: @@ -227,10 +218,7 @@ class AND(SemanticContext): # unordered.

# def eval(self, parser, outerContext): - for opnd in self.opnds: - if not opnd.eval(parser, outerContext): - return False - return True + return all(opnd.eval(parser, outerContext) for opnd in self.opnds) def evalPrecedence(self, parser, outerContext): differs = False @@ -277,13 +265,11 @@ class OR (SemanticContext): def __init__(self, a, b): operands = set() if isinstance( a, OR): - for o in a.opnds: - operands.add(o) + operands.update(a.opnds) else: operands.add(a) if isinstance( b, OR): - for o in b.opnds: - operands.add(o) + operands.update(b.opnds) else: operands.add(b) @@ -291,10 +277,10 @@ class OR (SemanticContext): if len(precedencePredicates)>0: # interested in the transition with the highest precedence s = sorted(precedencePredicates) - reduced = s[len(s)-1] + reduced = s[-1] operands.add(reduced) - self.opnds = [ o for o in operands ] + self.opnds = list(operands) def __eq__(self, other): if self is other: @@ -315,10 +301,7 @@ class OR (SemanticContext): # unordered.

# def eval(self, parser, outerContext): - for opnd in self.opnds: - if opnd.eval(parser, outerContext): - return True - return False + return any(opnd.eval(parser, outerContext) for opnd in self.opnds) def evalPrecedence(self, parser, outerContext): differs = False diff --git a/runtime/Python2/src/antlr4/dfa/DFAState.py b/runtime/Python2/src/antlr4/dfa/DFAState.py index 85ccac106..7045f7c22 100644 --- a/runtime/Python2/src/antlr4/dfa/DFAState.py +++ b/runtime/Python2/src/antlr4/dfa/DFAState.py @@ -105,14 +105,9 @@ class DFAState(object): # Get the set of all alts mentioned by all ATN configurations in this # DFA state. def getAltSet(self): - alts = set() if self.configs is not None: - for c in self.configs: - alts.add(c.alt) - if len(alts)==0: - return None - else: - return alts + return set(cfg.alt for cfg in self.configs) or None + return None def __hash__(self): return hash(self.configs) diff --git a/runtime/Python2/src/antlr4/tree/Trees.py b/runtime/Python2/src/antlr4/tree/Trees.py index d48f4a904..00a322b45 100644 --- a/runtime/Python2/src/antlr4/tree/Trees.py +++ b/runtime/Python2/src/antlr4/tree/Trees.py @@ -129,8 +129,7 @@ class Trees(object): @classmethod def descendants(cls, t): - nodes = [] - nodes.append(t) + nodes = [t] for i in range(0, t.getChildCount()): nodes.extend(cls.descendants(t.getChild(i))) return nodes diff --git a/runtime/Python2/src/antlr4/xpath/XPath.py b/runtime/Python2/src/antlr4/xpath/XPath.py index eee87da6f..e138d3cdf 100644 --- a/runtime/Python2/src/antlr4/xpath/XPath.py +++ b/runtime/Python2/src/antlr4/xpath/XPath.py @@ -289,12 +289,7 @@ class XPathRuleElement(XPathElement): def evaluate(self, t): # return all children of t that match nodeName - nodes = [] - for c in Trees.getChildren(t): - if isinstance(c, ParserRuleContext ): - if (c.ruleIndex == self.ruleIndex ) == (not self.invert): - nodes.append(c) - return nodes + return [c for c in Trees.getChildren(t) if isinstance(c, ParserRuleContext) and (c.ruleIndex == self.ruleIndex) == (not self.invert)] class XPathTokenAnywhereElement(XPathElement): @@ -314,12 +309,8 @@ class XPathTokenElement(XPathElement): def evaluate(self, t): # return all children of t that match nodeName - nodes = [] - for c in Trees.getChildren(t): - if isinstance(c, TerminalNode): - if (c.symbol.type == self.tokenType ) == (not self.invert): - nodes.append(c) - return nodes + return [c for c in Trees.getChildren(t) if isinstance(c, TerminalNode) and (c.symbol.type == self.tokenType) == (not self.invert)] + class XPathWildcardAnywhereElement(XPathElement): @@ -343,4 +334,4 @@ class XPathWildcardElement(XPathElement): if self.invert: return list() # !* is weird but valid (empty) else: - return Trees.getChildren(t) \ No newline at end of file + return Trees.getChildren(t) diff --git a/runtime/Python3/src/antlr4/IntervalSet.py b/runtime/Python3/src/antlr4/IntervalSet.py index e5f535f45..980509821 100644 --- a/runtime/Python3/src/antlr4/IntervalSet.py +++ b/runtime/Python3/src/antlr4/IntervalSet.py @@ -84,16 +84,10 @@ class IntervalSet(object): if self.intervals is None: return False else: - for i in self.intervals: - if item in i: - return True - return False + return any(item in i for i in self.intervals) def __len__(self): - xlen = 0 - for i in self.intervals: - xlen += len(i) - return xlen + return sum(len(i) for i in self.intervals) def removeRange(self, v): if v.start==v.stop-1: @@ -113,7 +107,7 @@ class IntervalSet(object): # check for included range, remove it elif v.start<=i.start and v.stop>=i.stop: self.intervals.pop(k) - k = k - 1 # need another pass + k -= 1 # need another pass # check for lower boundary elif v.start DFAState: proposed = DFAState(configs=configs) - firstConfigWithRuleStopState = None - for c in configs: - if isinstance(c.state, RuleStopState): - firstConfigWithRuleStopState = c - break + firstConfigWithRuleStopState = next((cfg for cfg in configs if isinstance(cfg.state, RuleStopState)), None) if firstConfigWithRuleStopState is not None: proposed.isAcceptState = True diff --git a/runtime/Python3/src/antlr4/atn/PredictionMode.py b/runtime/Python3/src/antlr4/atn/PredictionMode.py index 03ca896c7..67039c728 100644 --- a/runtime/Python3/src/antlr4/atn/PredictionMode.py +++ b/runtime/Python3/src/antlr4/atn/PredictionMode.py @@ -235,10 +235,7 @@ class PredictionMode(Enum): # {@link RuleStopState}, otherwise {@code false} @classmethod def hasConfigInRuleStopState(cls, configs:ATNConfigSet): - for c in configs: - if isinstance(c.state, RuleStopState): - return True - return False + return any(isinstance(cfg.state, RuleStopState) for cfg in configs) # Checks if all configurations in {@code configs} are in a # {@link RuleStopState}. Configurations meeting this condition have reached @@ -250,10 +247,7 @@ class PredictionMode(Enum): # {@link RuleStopState}, otherwise {@code false} @classmethod def allConfigsInRuleStopStates(cls, configs:ATNConfigSet): - for config in configs: - if not isinstance(config.state, RuleStopState): - return False - return True + return all(isinstance(cfg.state, RuleStopState) for cfg in configs) # # Full LL prediction termination. @@ -422,10 +416,7 @@ class PredictionMode(Enum): # @classmethod def hasNonConflictingAltSet(cls, altsets:list): - for alts in altsets: - if len(alts)==1: - return True - return False + return any(len(alts) == 1 for alts in altsets) # # Determines if any single alternative subset in {@code altsets} contains @@ -437,10 +428,7 @@ class PredictionMode(Enum): # @classmethod def hasConflictingAltSet(cls, altsets:list): - for alts in altsets: - if len(alts)>1: - return True - return False + return any(len(alts) > 1 for alts in altsets) # # Determines if every alternative subset in {@code altsets} is equivalent. @@ -451,13 +439,10 @@ class PredictionMode(Enum): # @classmethod def allSubsetsEqual(cls, altsets:list): - first = None - for alts in altsets: - if first is None: - first = alts - elif not alts==first: - return False - return True + if not altsets: + return True + first = next(iter(altsets)) + return all(alts == first for alts in iter(altsets)) # # Returns the unique alternative predicted by all alternative subsets in @@ -470,10 +455,8 @@ class PredictionMode(Enum): def getUniqueAlt(cls, altsets:list): all = cls.getAlts(altsets) if len(all)==1: - for one in all: - return one - else: - return ATN.INVALID_ALT_NUMBER + return next(iter(all)) + return ATN.INVALID_ALT_NUMBER # Gets the complete set of represented alternatives for a collection of # alternative subsets. This method returns the union of each {@link BitSet} @@ -484,10 +467,7 @@ class PredictionMode(Enum): # @classmethod def getAlts(cls, altsets:list): - all = set() - for alts in altsets: - all = all | alts - return all + return set.union(*altsets) # # This function gets the conflicting alt subsets from a configuration set. @@ -531,11 +511,7 @@ class PredictionMode(Enum): @classmethod def hasStateAssociatedWithOneAlt(cls, configs:ATNConfigSet): - x = cls.getStateToAltMap(configs) - for alts in x.values(): - if len(alts)==1: - return True - return False + return any(len(alts) == 1 for alts in cls.getStateToAltMap(configs).values()) @classmethod def getSingleViableAlt(cls, altsets:list): diff --git a/runtime/Python3/src/antlr4/atn/SemanticContext.py b/runtime/Python3/src/antlr4/atn/SemanticContext.py index c012e35c2..d69165e4d 100644 --- a/runtime/Python3/src/antlr4/atn/SemanticContext.py +++ b/runtime/Python3/src/antlr4/atn/SemanticContext.py @@ -116,13 +116,7 @@ def orContext(a:SemanticContext, b:SemanticContext): return result def filterPrecedencePredicates(collection:list): - result = [] - for context in collection: - if isinstance(context, PrecedencePredicate): - if result is None: - result = [] - result.append(context) - return result + return [context for context in collection if isinstance(context, PrecedencePredicate)] class Predicate(SemanticContext): @@ -188,13 +182,11 @@ class AND(SemanticContext): def __init__(self, a:SemanticContext, b:SemanticContext): operands = set() if isinstance( a, AND ): - for o in a.opnds: - operands.add(o) + operands.update(a.opnds) else: operands.add(a) if isinstance( b, AND ): - for o in b.opnds: - operands.add(o) + operands.update(b.opnds) else: operands.add(b) @@ -204,7 +196,7 @@ class AND(SemanticContext): reduced = min(precedencePredicates) operands.add(reduced) - self.opnds = [ o for o in operands ] + self.opnds = list(operands) def __eq__(self, other): if self is other: @@ -227,11 +219,8 @@ class AND(SemanticContext): # The evaluation of predicates by this context is short-circuiting, but # unordered.

# - def eval(self, parser:Recognizer , outerContext:RuleContext ): - for opnd in self.opnds: - if not opnd.eval(parser, outerContext): - return False - return True + def eval(self, parser:Recognizer, outerContext:RuleContext): + return all(opnd.eval(parser, outerContext) for opnd in self.opnds) def evalPrecedence(self, parser:Recognizer, outerContext:RuleContext): differs = False @@ -278,13 +267,11 @@ class OR (SemanticContext): def __init__(self, a:SemanticContext, b:SemanticContext): operands = set() if isinstance( a, OR ): - for o in a.opnds: - operands.add(o) + operands.update(a.opnds) else: operands.add(a) if isinstance( b, OR ): - for o in b.opnds: - operands.add(o) + operands.update(b.opnds) else: operands.add(b) @@ -292,10 +279,10 @@ class OR (SemanticContext): if len(precedencePredicates)>0: # interested in the transition with the highest precedence s = sorted(precedencePredicates) - reduced = s[len(s)-1] + reduced = s[-1] operands.add(reduced) - self.opnds = [ o for o in operands ] + self.opnds = list(operands) def __eq__(self, other): if self is other: @@ -316,10 +303,7 @@ class OR (SemanticContext): # unordered.

# def eval(self, parser:Recognizer, outerContext:RuleContext): - for opnd in self.opnds: - if opnd.eval(parser, outerContext): - return True - return False + return any(opnd.eval(parser, outerContext) for opnd in self.opnds) def evalPrecedence(self, parser:Recognizer, outerContext:RuleContext): differs = False diff --git a/runtime/Python3/src/antlr4/dfa/DFAState.py b/runtime/Python3/src/antlr4/dfa/DFAState.py index 1ab5a0e77..9ad996384 100644 --- a/runtime/Python3/src/antlr4/dfa/DFAState.py +++ b/runtime/Python3/src/antlr4/dfa/DFAState.py @@ -104,14 +104,9 @@ class DFAState(object): # Get the set of all alts mentioned by all ATN configurations in this # DFA state. def getAltSet(self): - alts = set() if self.configs is not None: - for c in self.configs: - alts.add(c.alt) - if len(alts)==0: - return None - else: - return alts + return set(cfg.alt for cfg in self.configs) or None + return None def __hash__(self): return hash(self.configs) diff --git a/runtime/Python3/src/antlr4/tree/Trees.py b/runtime/Python3/src/antlr4/tree/Trees.py index 101df6d9a..42fde282b 100644 --- a/runtime/Python3/src/antlr4/tree/Trees.py +++ b/runtime/Python3/src/antlr4/tree/Trees.py @@ -130,8 +130,7 @@ class Trees(object): @classmethod def descendants(cls, t:ParseTree): - nodes = [] - nodes.append(t) + nodes = [t] for i in range(0, t.getChildCount()): nodes.extend(cls.descendants(t.getChild(i))) return nodes diff --git a/runtime/Python3/src/antlr4/xpath/XPath.py b/runtime/Python3/src/antlr4/xpath/XPath.py index 6b41a943a..6d3d825a5 100644 --- a/runtime/Python3/src/antlr4/xpath/XPath.py +++ b/runtime/Python3/src/antlr4/xpath/XPath.py @@ -290,12 +290,8 @@ class XPathRuleElement(XPathElement): def evaluate(self, t:ParseTree): # return all children of t that match nodeName - nodes = [] - for c in Trees.getChildren(t): - if isinstance(c, ParserRuleContext ): - if (c.ruleIndex == self.ruleIndex ) == (not self.invert): - nodes.append(c) - return nodes + return [c for c in Trees.getChildren(t) if isinstance(c, ParserRuleContext) and (c.ruleIndex == self.ruleIndex) == (not self.invert)] + class XPathTokenAnywhereElement(XPathElement): @@ -315,12 +311,8 @@ class XPathTokenElement(XPathElement): def evaluate(self, t:ParseTree): # return all children of t that match nodeName - nodes = [] - for c in Trees.getChildren(t): - if isinstance(c, TerminalNode): - if (c.symbol.type == self.tokenType ) == (not self.invert): - nodes.append(c) - return nodes + return [c for c in Trees.getChildren(t) if isinstance(c, TerminalNode) and (c.symbol.type == self.tokenType) == (not self.invert)] + class XPathWildcardAnywhereElement(XPathElement):