diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index b22dc023c..52b36a92f 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -3,8 +3,4 @@ - [ ] I am not submitting a question on how to use ANTLR; instead, go to [antlr4-discussion google group](https://groups.google.com/forum/#!forum/antlr-discussion) or ask at [stackoverflow](http://stackoverflow.com/questions/tagged/antlr4) - [ ] I have done a search of the existing issues to make sure I'm not sending in a duplicate -### Expected behavior - -### Actual behavior - -### Steps to reproduce the behavior +Please include information about the expected behavior, actual behavior, and the smallest grammar or code that reproduces the behavior. If appropriate, please indicate the code generation targets such as Java, C#, ... Pointers into offending code regions are also very welcome. diff --git a/README.md b/README.md index 5e367a416..d2d29d3e0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ANTLR v4 -[![Build Status](https://travis-ci.org/antlr/antlr4.png?branch=master)](https://travis-ci.org/antlr/antlr4) [![Java 7+](https://img.shields.io/badge/java-7+-4c7e9f.svg)](http://java.oracle.com) [![License](https://img.shields.io/badge/license-BSD-blue.svg)](https://raw.githubusercontent.com/antlr/antlr4/master/LICENSE.txt) +[![Build Status](https://travis-ci.org/antlr/antlr4.png?branch=master)](https://travis-ci.org/antlr/antlr4) [![Java 7+](https://img.shields.io/badge/java-7+-4c7e9f.svg)](http://java.oracle.com) [![License](https://img.shields.io/badge/license-BSD-blue.svg)](https://raw.githubusercontent.com/antlr/antlr4/master/LICENSE.txt) **ANTLR** (ANother Tool for Language Recognition) is a powerful parser generator for reading, processing, executing, or translating structured text or binary files. It's widely used to build languages, tools, and frameworks. From a grammar, ANTLR generates a parser that can build parse trees and also generates a listener interface (or visitor) that makes it easy to respond to the recognition of phrases of interest. diff --git a/contributors.txt b/contributors.txt index 17a172a30..cc5d5f48b 100644 --- a/contributors.txt +++ b/contributors.txt @@ -94,6 +94,7 @@ YYYY/MM/DD, github id, Full name, email 2016/03/27, beardlybread, Bradley Steinbacher, bradley.j.steinbacher@gmail.com 2016/03/29, msteiger, Martin Steiger, antlr@martin-steiger.de 2016/03/28, gagern, Martin von Gagern, gagern@ma.tum.de +2016/07/10, twz123, Tom Wieczorek, tom.wieczorek@zalando.de 2016/07/20, chrisheller, Chris Heller, chris.heller.greyheller@gmail.com 2016/07/20, nburles, Nathan Burles, nburles@gmail.com 2016/07/20, kosl90, Li Liqiang, kos1990l@gmail.com @@ -118,3 +119,5 @@ YYYY/MM/DD, github id, Full name, email 2016/11/06, janyou, Janyou, janyou.antlr@outlook.com 2016/11/20, marcohu, Marco Hunsicker, antlr@hunsicker.de 2016/09/02, lygav, Vladimir (Vladi) Lyga, lyvladi@gmail.com +2016/09/23, ghosthope, Dmitry Shakhtanov, sudstrike@gmail.com +2016/11/25, MrSampson, Oliver Sampson, olsam@quickaudio.com diff --git a/doc/getting-started.md b/doc/getting-started.md index a1332ff5e..056f0fa9d 100644 --- a/doc/getting-started.md +++ b/doc/getting-started.md @@ -6,7 +6,7 @@ Hi and welcome to the version 4 release of ANTLR! It's named after the fearless ANTLR is really two things: a tool that translates your grammar to a parser/lexer in Java (or other target language) and the runtime needed by the generated parsers/lexers. Even if you are using the ANTLR Intellij plug-in or ANTLRWorks to run the ANTLR tool, the generated code will still need the runtime library. -The first thing you should do is probably download and install a development tool plug-in. Even if you only use such tools for editing, they are great. Then, follow the instructions below to get the runtime environment available to your system to run generated parsers/lexers. In what follows, I talk about antlr-4.5-complete.jar, which has the tool and the runtime and any other support libraries (e.g., ANTLR v4 is written in v3). +The first thing you should do is probably download and install a development tool plug-in. Even if you only use such tools for editing, they are great. Then, follow the instructions below to get the runtime environment available to your system to run generated parsers/lexers. In what follows, I talk about antlr-4.5.3-complete.jar, which has the tool and the runtime and any other support libraries (e.g., ANTLR v4 is written in v3). If you are going to integrate ANTLR into your existing build system using mvn, ant, or want to get ANTLR into your IDE such as eclipse or intellij, see Integrating ANTLR into Development Systems. @@ -16,19 +16,19 @@ If you are going to integrate ANTLR into your existing build system using mvn, a 1. Download ``` $ cd /usr/local/lib -$ curl -O http://www.antlr.org/download/antlr-4.5-complete.jar +$ curl -O http://www.antlr.org/download/antlr-4.5.3-complete.jar ``` Or just download in browser from website: [http://www.antlr.org/download.html](http://www.antlr.org/download.html) and put it somewhere rational like `/usr/local/lib`. -2. Add `antlr-4.5-complete.jar` to your `CLASSPATH`: +2. Add `antlr-4.5.3-complete.jar` to your `CLASSPATH`: ``` -$ export CLASSPATH=".:/usr/local/lib/antlr-4.5-complete.jar:$CLASSPATH" +$ export CLASSPATH=".:/usr/local/lib/antlr-4.5.3-complete.jar:$CLASSPATH" ``` It's also a good idea to put this in your `.bash_profile` or whatever your startup script is. 3. Create aliases for the ANTLR Tool, and `TestRig`. ``` -$ alias antlr4='java -Xmx500M -cp "/usr/local/lib/antlr-4.5-complete.jar:$CLASSPATH" org.antlr.v4.Tool' +$ alias antlr4='java -Xmx500M -cp "/usr/local/lib/antlr-4.5.3-complete.jar:$CLASSPATH" org.antlr.v4.Tool' $ alias grun='java org.antlr.v4.gui.TestRig' ``` @@ -37,13 +37,13 @@ $ alias grun='java org.antlr.v4.gui.TestRig' (*Thanks to Graham Wideman*) 0. Install Java (version 1.6 or higher) -1. Download antlr-4.5-complete.jar (or whatever version) from [http://www.antlr.org/download/](http://www.antlr.org/download/) +1. Download antlr-4.5.3-complete.jar (or whatever version) from [http://www.antlr.org/download/](http://www.antlr.org/download/) Save to your directory for 3rd party Java libraries, say `C:\Javalib` 2. Add `antlr-4.5-complete.jar` to CLASSPATH, either: * Permanently: Using System Properties dialog > Environment variables > Create or append to `CLASSPATH` variable * Temporarily, at command line: ``` -SET CLASSPATH=.;C:\Javalib\antlr-4.5-complete.jar;%CLASSPATH% +SET CLASSPATH=.;C:\Javalib\antlr-4.5.3-complete.jar;%CLASSPATH% ``` 3. Create short convenient commands for the ANTLR Tool, and TestRig, using batch files or doskey commands: * Batch files (in directory in system PATH) antlr4.bat and grun.bat @@ -65,7 +65,7 @@ Either launch org.antlr.v4.Tool directly: ``` $ java org.antlr.v4.Tool -ANTLR Parser Generator Version 4.5 +ANTLR Parser Generator Version 4.5.3 -o ___ specify output directory where all output is generated -lib ___ specify location of .tokens files ... @@ -74,8 +74,8 @@ ANTLR Parser Generator Version 4.5 or use -jar option on java: ``` -$ java -jar /usr/local/lib/antlr-4.5-complete.jar -ANTLR Parser Generator Version 4.5 +$ java -jar /usr/local/lib/antlr-4.5.3-complete.jar +ANTLR Parser Generator Version 4.5.3 -o ___ specify output directory where all output is generated -lib ___ specify location of .tokens files ... diff --git a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java index ba1417c68..068c86aa0 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/BufferedTokenStream.java @@ -456,18 +456,17 @@ public class BufferedTokenStream implements TokenStream { @Override public String getText() { - lazyInit(); - fill(); return getText(Interval.of(0,size()-1)); } - - @Override - public String getText(Interval interval) { + @Override + public String getText(Interval interval) { + lazyInit(); + fill(); int start = interval.a; int stop = interval.b; - if ( start<0 || stop<0 ) return ""; - lazyInit(); + if ( start<0 || stop<0 ) return ""; + lazyInit(); if ( stop>=tokens.size() ) stop = tokens.size()-1; StringBuilder buf = new StringBuilder(); diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/IterativeParseTreeWalker.java b/runtime/Java/src/org/antlr/v4/runtime/tree/IterativeParseTreeWalker.java new file mode 100644 index 000000000..ebdf70998 --- /dev/null +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/IterativeParseTreeWalker.java @@ -0,0 +1,103 @@ +/* + * [The "BSD license"] + * Copyright (c) 2012 Terence Parr + * Copyright (c) 2012 Sam Harwell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.antlr.v4.runtime.tree; + +import java.util.ArrayDeque; +import java.util.Deque; + +import org.antlr.v4.runtime.misc.IntegerStack; + +/** + * An iterative (read: non-recursive) pre-order and post-order tree walker that + * doesn't use the thread stack but heap-based stacks. Makes it possible to + * process deeply nested parse trees. + */ +public class IterativeParseTreeWalker extends ParseTreeWalker { + + @Override + public void walk(ParseTreeListener listener, ParseTree t) { + + final Deque nodeStack = new ArrayDeque(); + final IntegerStack indexStack = new IntegerStack(); + + ParseTree currentNode = t; + int currentIndex = 0; + + while (currentNode != null) { + + // pre-order visit + if (currentNode instanceof ErrorNode) { + listener.visitErrorNode((ErrorNode) currentNode); + } else if (currentNode instanceof TerminalNode) { + listener.visitTerminal((TerminalNode) currentNode); + } else { + final RuleNode r = (RuleNode) currentNode; + enterRule(listener, r); + } + + // Move down to first child, if exists + if (currentNode.getChildCount() > 0) { + nodeStack.push(currentNode); + indexStack.push(currentIndex); + currentIndex = 0; + currentNode = currentNode.getChild(0); + continue; + } + + // No child nodes, so walk tree + do { + + // post-order visit + if (currentNode instanceof RuleNode) { + exitRule(listener, (RuleNode) currentNode); + } + + // No parent, so no siblings + if (nodeStack.isEmpty()) { + currentNode = null; + currentIndex = 0; + break; + } + + // Move to next sibling if possible + currentNode = nodeStack.peek().getChild(++currentIndex); + if (currentNode != null) { + break; + } + + // No next, sibling, so move up + currentNode = nodeStack.pop(); + currentIndex = indexStack.pop(); + + } while (currentNode != null); + } + } +} diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeWalker.java b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeWalker.java index 1065204b4..e860327f8 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeWalker.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/ParseTreeWalker.java @@ -34,7 +34,7 @@ import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.RuleContext; public class ParseTreeWalker { - public static final ParseTreeWalker DEFAULT = new ParseTreeWalker(); + public static final ParseTreeWalker DEFAULT = new IterativeParseTreeWalker(); public void walk(ParseTreeListener listener, ParseTree t) { if ( t instanceof ErrorNode) { diff --git a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPath.java b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPath.java index 7bd249f6b..8d721d61d 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPath.java +++ b/runtime/Java/src/org/antlr/v4/runtime/tree/xpath/XPath.java @@ -85,6 +85,7 @@ public class XPath { throw new IllegalArgumentException("Could not read path: "+path, ioe); } XPathLexer lexer = new XPathLexer(in) { + @Override public void recover(LexerNoViableAltException e) { throw e; } }; lexer.removeErrorListeners(); diff --git a/runtime/JavaScript/src/antlr4/Utils.js b/runtime/JavaScript/src/antlr4/Utils.js index b47e75872..6b54da305 100644 --- a/runtime/JavaScript/src/antlr4/Utils.js +++ b/runtime/JavaScript/src/antlr4/Utils.js @@ -2,18 +2,63 @@ function arrayToString(a) { return "[" + a.join(", ") + "]"; } -String.prototype.hashCode = function(s) { - var hash = 0; - if (this.length === 0) { - return hash; - } - for (var i = 0; i < this.length; i++) { - var character = this.charCodeAt(i); - hash = ((hash << 5) - hash) + character; - hash = hash & hash; // Convert to 32bit integer - } - return hash; -}; + String.prototype.seed = String.prototype.seed || Math.round(Math.random() * Math.pow(2, 32));; + + String.prototype.hashCode = function () { + var remainder, bytes, h1, h1b, c1, c1b, c2, c2b, k1, i, + key = this.toString(); + + remainder = key.length & 3; // key.length % 4 + bytes = key.length - remainder; + h1 = String.prototype.seed; + c1 = 0xcc9e2d51; + c2 = 0x1b873593; + i = 0; + + while (i < bytes) { + k1 = + ((key.charCodeAt(i) & 0xff)) | + ((key.charCodeAt(++i) & 0xff) << 8) | + ((key.charCodeAt(++i) & 0xff) << 16) | + ((key.charCodeAt(++i) & 0xff) << 24); + ++i; + + k1 = ((((k1 & 0xffff) * c1) + ((((k1 >>> 16) * c1) & 0xffff) << 16))) & 0xffffffff; + k1 = (k1 << 15) | (k1 >>> 17); + k1 = ((((k1 & 0xffff) * c2) + ((((k1 >>> 16) * c2) & 0xffff) << 16))) & 0xffffffff; + + h1 ^= k1; + h1 = (h1 << 13) | (h1 >>> 19); + h1b = ((((h1 & 0xffff) * 5) + ((((h1 >>> 16) * 5) & 0xffff) << 16))) & 0xffffffff; + h1 = (((h1b & 0xffff) + 0x6b64) + ((((h1b >>> 16) + 0xe654) & 0xffff) << 16)); + } + + k1 = 0; + + switch (remainder) { + case 3: + k1 ^= (key.charCodeAt(i + 2) & 0xff) << 16; + case 2: + k1 ^= (key.charCodeAt(i + 1) & 0xff) << 8; + case 1: + k1 ^= (key.charCodeAt(i) & 0xff); + + k1 = (((k1 & 0xffff) * c1) + ((((k1 >>> 16) * c1) & 0xffff) << 16)) & 0xffffffff; + k1 = (k1 << 15) | (k1 >>> 17); + k1 = (((k1 & 0xffff) * c2) + ((((k1 >>> 16) * c2) & 0xffff) << 16)) & 0xffffffff; + h1 ^= k1; + } + + h1 ^= key.length; + + h1 ^= h1 >>> 16; + h1 = (((h1 & 0xffff) * 0x85ebca6b) + ((((h1 >>> 16) * 0x85ebca6b) & 0xffff) << 16)) & 0xffffffff; + h1 ^= h1 >>> 13; + h1 = ((((h1 & 0xffff) * 0xc2b2ae35) + ((((h1 >>> 16) * 0xc2b2ae35) & 0xffff) << 16))) & 0xffffffff; + h1 ^= h1 >>> 16; + + return h1 >>> 0; + }; function standardEqualsFunction(a,b) { return a.equals(b); diff --git a/runtime/Python3/src/antlr4/atn/ATNConfigSet.py b/runtime/Python3/src/antlr4/atn/ATNConfigSet.py old mode 100755 new mode 100644 diff --git a/runtime/Python3/src/antlr4/atn/ParserATNSimulator.py b/runtime/Python3/src/antlr4/atn/ParserATNSimulator.py old mode 100755 new mode 100644 diff --git a/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java b/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java index a229942ff..1e4540bdc 100644 --- a/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java +++ b/tool/src/org/antlr/v4/codegen/target/JavaScriptTarget.java @@ -220,14 +220,17 @@ public class JavaScriptTarget extends Target { } + @Override public boolean wantsBaseListener() { return false; } + @Override public boolean wantsBaseVisitor() { return false; } + @Override public boolean supportsOverloadedMethods() { return false; } diff --git a/tool/src/org/antlr/v4/gui/TreeViewer.java b/tool/src/org/antlr/v4/gui/TreeViewer.java index 96a0ca213..e9ec00134 100644 --- a/tool/src/org/antlr/v4/gui/TreeViewer.java +++ b/tool/src/org/antlr/v4/gui/TreeViewer.java @@ -393,6 +393,7 @@ public class TreeViewer extends JComponent { // make viz WindowListener exitListener = new WindowAdapter() { + @Override public void windowClosing(WindowEvent e) { prefs.putInt(DIALOG_WIDTH_PREFS_KEY, (int) dialog.getSize().getWidth()); prefs.putInt(DIALOG_HEIGHT_PREFS_KEY, (int) dialog.getSize().getHeight());