Merge branch 'master' into master

.travis.yml

@@ -151,32 +151,32 @@ matrix:
     - os: linux
       jdk: openjdk7
       env: TARGET=csharp
-      stage: extended-test
+      stage: main-test
     - os: linux
       jdk: oraclejdk8
       dist: trusty
       env:
         - TARGET=dotnet
         - GROUP=LEXER
-      stage: main-test
+      stage: extended-test
     - os: linux
       jdk: openjdk8
       dist: trusty
       env:
         - TARGET=dotnet
         - GROUP=PARSER
-      stage: main-test
+      stage: extended-test
     - os: linux
       jdk: oraclejdk8
       dist: trusty
       env:
         - TARGET=dotnet
         - GROUP=RECURSION
-      stage: main-test
+      stage: extended-test
     - os: linux
       jdk: openjdk7
       env: TARGET=python2
-      stage: extended-test
+      stage: main-test
     - os: linux
       jdk: openjdk7
       env: TARGET=python3

contributors.txt

@@ -180,5 +180,17 @@ YYYY/MM/DD, github id, Full name, email
 2017/12/03, oranoran, Oran Epelbaum, oran / epelbaum me
 2017/12/20, kbsletten, Kyle Sletten, kbsletten@gmail.com
 2017/12/27, jkmar, Jakub Marciniszyn, marciniszyn.jk@gmail.com
+2018/01/06, kasbah, Kaspar Emanuel, kaspar@monostable.co.uk
+2018/02/08, razfriman, Raz Friman, raz@razfriman.com
 2018/02/11, io7m, Mark Raynsford, code@io7m.com
 2018/04/24, solussd, Joe Smith, joe@uwcreations.com
+2018/05/15, johnvanderholt, jan dillingh johnvanderholte@gmail.com
+2018/06/16, EternalPhane, Zongyuan Zuo, eternalphane@gmail.com
+2018/05/15, johnvanderholt, jan dillingh johnvanderholte@gmail.com
+2018/05/17, sinopsysHK, Eric Bardes, sinofwd@gmail.com
+2018/05/23, srvance, Stephen Vance, steve@vance.com
+2018/06/14, alecont, Alessandro Contenti, alecontenti@hotmail.com
+2018/06/16, EternalPhane, Zongyuan Zuo, eternalphane@gmail.com
+2018/07/03, jgoppert, James Goppert, james.goppert@gmail.com
+2018/07/27, Maksim Novikov, mnovikov.work@gmail.com
+2018/08/03, ENDOH takanao, djmchl@gmail.com

doc/csharp-target.md

@@ -21,7 +21,7 @@ You will find full instructions on the [Git repo page for ANTLR C# runtime](http
 
 Let's suppose that your grammar is named `MyGrammar`. The tool will generate for you the following files:
 
-*  MyGrammarLexer.cs
+*   MyGrammarLexer.cs
 *   MyGrammarParser.cs
 *   MyGrammarListener.cs (if you have not activated the -no-listener option)
 *   MyGrammarBaseListener.cs (if you have not activated the -no-listener option)
@@ -32,6 +32,7 @@ Now a fully functioning code might look like the following for start rule `Start
 
 ```
 using Antlr4.Runtime;
+using Antlr4.Runtime.Tree;
      
 public void MyParseMethod() {
       String input = "your text to parse here";
@@ -39,7 +40,7 @@ public void MyParseMethod() {
       ITokenSource lexer = new MyGrammarLexer(stream);
       ITokenStream tokens = new CommonTokenStream(lexer);
       MyGrammarParser parser = new MyGrammarParser(tokens);
-      parser.buildParseTrees = true;
+      parser.BuildParseTree = true;
       IParseTree tree = parser.StartRule();
 }
 ```

doc/javascript-target.md

@@ -62,7 +62,7 @@ The steps to create your parsing code are the following:
 You are now ready to bundle your parsing code as follows:
  - following webpack specs, create a webpack.config file
  - in the webpack.config file, exclude node.js only modules using: node: { module: "empty", net: "empty", fs: "empty" }
- - from the cmd line, nag-vigate to the directory containing webpack.config and type: webpack
+ - from the cmd line, navigate to the directory containing webpack.config and type: webpack
  
 This will produce a single js file containing all your parsing code. Easy to include in your web pages!
 
@@ -95,11 +95,16 @@ Let's suppose that your grammar is named, as above, "MyGrammar". Let's suppose t
 Now a fully functioning script might look like the following:
 
 ```javascript
+   var antlr4 = require('antlr4');
+   var MyGrammarLexer = require('./MyGrammarLexer').MyGrammarLexer;
+   var MyGrammarParser = require('./MyGrammarParser').MyGrammarParser;
+   var MyGrammarListener = require('./MyGrammarListener').MyGrammarListener;
+
    var input = "your text to parse here"
    var chars = new antlr4.InputStream(input);
-   var lexer = new MyGrammarLexer.MyGrammarLexer(chars);
+   var lexer = new MyGrammarLexer(chars);
    var tokens  = new antlr4.CommonTokenStream(lexer);
-   var parser = new MyGrammarParser.MyGrammarParser(tokens);
+   var parser = new MyGrammarParser(tokens);
    parser.buildParseTrees = true;
    var tree = parser.MyStartRule();
 ```
@@ -128,30 +133,30 @@ Let's suppose your MyGrammar grammar comprises 2 rules: "key" and "value". The a
 ```
 
 In order to provide custom behavior, you might want to create the following class:
-  
+
 ```javascript
-    KeyPrinter = function() {
-         MyGrammarListener.call(this); // inherit default listener
-         return this;
-    };
- 
-// inherit default listener
+var KeyPrinter = function() {
+    MyGrammarListener.call(this); // inherit default listener
+    return this;
+};
+
+// continue inheriting default listener
 KeyPrinter.prototype = Object.create(MyGrammarListener.prototype);
 KeyPrinter.prototype.constructor = KeyPrinter;
- 
+
 // override default listener behavior
-       KeyPrinter.prototype.exitKey = function(ctx) {      
-       console.log("Oh, a key!");
-   }; 
+KeyPrinter.prototype.exitKey = function(ctx) {
+    console.log("Oh, a key!");
+};
 ```
 
 In order to execute this listener, you would simply add the following lines to the above code:
- 
+
 ```javascript
-        ...
-       tree = parser.StartRule() - only repeated here for reference
-   var printer = new KeyPrinter();
- antlr4.tree.ParseTreeWalker.DEFAULT.walk(printer, tree);
+    ...
+    tree = parser.StartRule() // only repeated here for reference
+var printer = new KeyPrinter();
+antlr4.tree.ParseTreeWalker.DEFAULT.walk(printer, tree);
 ```
 
 ## What about TypeScript?

doc/releasing-antlr.md

@@ -172,6 +172,7 @@ alias java='/Library/Java/JavaVirtualMachines/jdk1.7.0_21.jdk/Contents/Home/bin/
 alias javac='/Library/Java/JavaVirtualMachines/jdk1.7.0_21.jdk/Contents/Home/bin/javac'
 alias javadoc='/Library/Java/JavaVirtualMachines/jdk1.7.0_21.jdk/Contents/Home/bin/javadoc'
 alias jar='/Library/Java/JavaVirtualMachines/jdk1.7.0_21.jdk/Contents/Home/bin/jar'
+export JAVA_HOME=`/usr/libexec/java_home -v 1.7`
 ```
 
 You should see 0x33 in generated .class files after 0xCAFEBABE; see [Java SE 7 = 51 (0x33 hex)](https://en.wikipedia.org/wiki/Java_class_file):

runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/ParserExecDescriptors.java

@@ -854,4 +854,39 @@ public class ParserExecDescriptors {
 		@CommentHasStringValue
 		public String grammar;
 	}
+
+	/**
+	 * This is a regression test for antlr/antlr4#2301.
+	 */
+	public static class OrderingPredicates extends BaseParserTestDescriptor {
+		public String input = "POINT AT X";
+		public String output = null;
+		public String errors = null;
+		public String startRule = "expr";
+		public String grammarName = "Issue2301";
+
+		/**
+		 grammar Issue2301;
+
+		 SPACES: [ \t\r\n]+ -> skip;
+
+		 AT: 'AT';
+		 X : 'X';
+		 Y : 'Y';
+
+		 ID: [A-Z]+;
+
+		 constant
+		 : 'DUMMY'
+		 ;
+
+		 expr
+		 : ID constant?
+		 | expr AT X
+		 | expr AT Y
+		 ;
+		 */
+		@CommentHasStringValue
+		public String grammar;
+	}
 }

runtime/CSharp/README.md

@@ -51,7 +51,13 @@ This is just a quick start. The tool has many useful options to control generati
 
 The Antlr 4 standard runtime for C# is now available from NuGet.
 We trust that you know how to do add NuGet references to your project :-).
-The package id is Antlr4.Runtime.Standard. We do not support other packages.
+The package id is [Antlr4.Runtime.Standard](https://www.nuget.org/packages/Antlr4.Runtime.Standard/). We do not support other packages.
+
+Use the GUI or the following command in the Package Manager Console:
+
+```
+Install-Package Antlr4.Runtime.Standard
+```
 
 
 ### Step 6: You're done!

runtime/Cpp/CMakeLists.txt

@@ -136,3 +136,7 @@ endif()
 
 install(FILES README.md VERSION 
     DESTINATION "share/doc/libantlr4")
+
+set(CPACK_PACKAGE_CONTACT "antlr-discussion@googlegroups.com")
+set(CPACK_PACKAGE_VERSION ${ANTLR_VERSION})
+include(CPack)

runtime/Cpp/runtime/src/support/Any.cpp

@@ -11,6 +11,3 @@ Any::~Any()
 {
     delete _ptr;
 }
-
-Any::Base::~Base() {
-}

runtime/Cpp/runtime/src/support/Any.h

@@ -100,7 +100,7 @@ struct ANTLR4CPP_PUBLIC Any
 
 private:
   struct Base {
-    virtual ~Base();
+    virtual ~Base() {};
     virtual Base* clone() const = 0;
   };
 
@@ -112,10 +112,21 @@ private:
 
     T value;
 
+    Base* clone() const {
+      return clone<>();
+    }
+
+  private:
+    template<int N = 0, typename std::enable_if<N == N && std::is_nothrow_copy_constructible<T>::value, int>::type = 0>
     Base* clone() const {
       return new Derived<T>(value);
     }
 
+    template<int N = 0, typename std::enable_if<N == N && !std::is_nothrow_copy_constructible<T>::value, int>::type = 0>
+    Base* clone() const {
+      return nullptr;
+    }
+
   };
 
   Base* clone() const

runtime/JavaScript/src/antlr4/tree/Tree.js

@@ -73,7 +73,11 @@ ParseTreeVisitor.prototype.visit = function(ctx) {
 };
 
 ParseTreeVisitor.prototype.visitChildren = function(ctx) {
-  return this.visit(ctx.children);
+	if (ctx.children) {
+		return this.visit(ctx.children);
+	} else {
+		return null;
+	}
 }
 
 ParseTreeVisitor.prototype.visitTerminal = function(node) {

runtime/Python3/src/antlr4/StdinStream.py

@@ -0,0 +1,11 @@
+import codecs
+import sys
+
+from antlr4.InputStream import InputStream
+
+
+class StdinStream(InputStream):
+    def __init__(self, encoding:str='ascii', errors:str='strict') -> None:
+        bytes = sys.stdin.buffer.read()
+        data = codecs.decode(bytes, encoding, errors)
+        super().__init__(data)

runtime/Python3/src/antlr4/atn/ParserATNSimulator.py

@@ -1608,7 +1608,7 @@ class ParserATNSimulator(ATNSimulator):
 
     def reportAttemptingFullContext(self, dfa:DFA, conflictingAlts:set, configs:ATNConfigSet, startIndex:int, stopIndex:int):
         if ParserATNSimulator.debug or ParserATNSimulator.retry_debug:
-            interval = range(startIndex, stopIndex + 1)
+            interval = (startIndex, stopIndex + 1)
             print("reportAttemptingFullContext decision=" + str(dfa.decision) + ":" + str(configs) +
                                ", input=" + self.parser.getTokenStream().getText(interval))
         if self.parser is not None:
@@ -1616,7 +1616,7 @@ class ParserATNSimulator(ATNSimulator):
 
     def reportContextSensitivity(self, dfa:DFA, prediction:int, configs:ATNConfigSet, startIndex:int, stopIndex:int):
         if ParserATNSimulator.debug or ParserATNSimulator.retry_debug:
-            interval = range(startIndex, stopIndex + 1)
+            interval = (startIndex, stopIndex + 1)
             print("reportContextSensitivity decision=" + str(dfa.decision) + ":" + str(configs) +
                                ", input=" + self.parser.getTokenStream().getText(interval))
         if self.parser is not None:
@@ -1642,7 +1642,7 @@ class ParserATNSimulator(ATNSimulator):
 #				}
 #				i++;
 #			}
-            interval = range(startIndex, stopIndex + 1)
+            interval = (startIndex, stopIndex + 1)
             print("reportAmbiguity " + str(ambigAlts) + ":" + str(configs) +
                                ", input=" + self.parser.getTokenStream().getText(interval))
         if self.parser is not None:

runtime/Python3/src/antlr4/atn/SemanticContext.py

@@ -135,8 +135,8 @@ class PrecedencePredicate(SemanticContext):
         else:
             return None
 
-    def __cmp__(self, other):
-        return self.precedence - other.precedence
+    def __lt__(self, other):
+        return self.precedence < other.precedence
 
     def __hash__(self):
         return 31

tool/pom.xml

@@ -44,7 +44,7 @@
 		<dependency>
 			<groupId>com.ibm.icu</groupId>
 			<artifactId>icu4j</artifactId>
-			<version>58.2</version>
+			<version>61.1</version>
 		</dependency>
 	</dependencies>
 	<build>

tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg

@@ -236,9 +236,12 @@ public class <parser.name> extends <superClass; null="Parser"> {
 	<endif>
 	public static final int
 		<parser.rules:{r | RULE_<r.name> = <r.index>}; separator=", ", wrap, anchor>;
-	public static final String[] ruleNames = {
-		<parser.ruleNames:{r | "<r>"}; separator=", ", wrap, anchor>
-	};
+	private static String[] makeRuleNames() {
+		return new String[] {
+			<parser.ruleNames:{r | "<r>"}; separator=", ", wrap, anchor>
+		};
+	}
+	public static final String[] ruleNames = makeRuleNames();
 
 	<vocabulary(parser.literalNames, parser.symbolicNames)>
 
@@ -275,12 +278,18 @@ case <f.ruleIndex>:
 >>
 
 vocabulary(literalNames, symbolicNames) ::= <<
-private static final String[] _LITERAL_NAMES = {
-	<literalNames:{t | <t>}; null="null", separator=", ", wrap, anchor>
-};
-private static final String[] _SYMBOLIC_NAMES = {
-	<symbolicNames:{t | <t>}; null="null", separator=", ", wrap, anchor>
-};
+private static String[] makeLiteralNames() {
+	return new String[] {
+		<literalNames:{t | <t>}; null="null", separator=", ", wrap, anchor>
+	};
+}
+private static final String[] _LITERAL_NAMES = makeLiteralNames();
+private static String[] makeSymbolicNames() {
+	return new String[] {
+		<symbolicNames:{t | <t>}; null="null", separator=", ", wrap, anchor>
+	};
+}
+private static final String[] _SYMBOLIC_NAMES = makeSymbolicNames();
 public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);
 
 /**
@@ -914,9 +923,12 @@ public class <lexer.name> extends <superClass; null="Lexer"> {
 		<lexer.modes:{m| "<m>"}; separator=", ", wrap, anchor>
 	};
 
-	public static final String[] ruleNames = {
-		<lexer.ruleNames:{r | "<r>"}; separator=", ", wrap, anchor>
-	};
+	private static String[] makeRuleNames() {
+		return new String[] {
+			<lexer.ruleNames:{r | "<r>"}; separator=", ", wrap, anchor>
+		};
+	}
+	public static final String[] ruleNames = makeRuleNames();
 
 	<vocabulary(lexer.literalNames, lexer.symbolicNames)>
 

tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg

@@ -749,6 +749,10 @@ import sys
 >>
 
 Lexer(lexer, atn, actionFuncs, sempredFuncs, superClass) ::= <<
+<if(superClass)>
+from .<superClass> import <superClass>
+
+<endif>
 
 <atn>
 

tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg

@@ -119,7 +119,10 @@ Parser(parser, funcs, atn, sempredFuncs, superClass) ::= <<
 
 Parser_(parser, funcs, atn, sempredFuncs, ctor, superClass) ::= <<
 <if(superClass)>
-from .<superClass> import <superClass>
+if __name__ is not None and "." in __name__:
+    from .<superClass> import <superClass>
+else:
+    from <superClass> import <superClass>
 
 <endif>
 <atn>
@@ -756,7 +759,13 @@ import sys
 >>
 
 Lexer(lexer, atn, actionFuncs, sempredFuncs, superClass) ::= <<
+<if(superClass)>
+if __name__ is not None and "." in __name__:
+    from .<superClass> import <superClass>
+else:
+    from <superClass> import <superClass>
 
+<endif>
 <atn>
 
 class <lexer.name>(<if(superClass)><superClass><else>Lexer<endif>):

tool/src/org/antlr/v4/codegen/target/Python2Target.java

@@ -24,27 +24,28 @@ import java.util.Set;
  */
 public class Python2Target extends Target {
 	protected static final String[] python2Keywords = {
-		"abs", "all", "any", "apply", "as",
-		"bin", "bool", "buffer", "bytearray",
-		"callable", "chr", "classmethod", "coerce", "compile", "complex",
-		"del", "delattr", "dict", "dir", "divmod",
-		"enumerate", "eval", "execfile",
-		"file", "filter", "float", "format", "frozenset",
-		"getattr", "globals",
+		"abs", "all", "and", "any", "apply", "as", "assert",
+		"bin", "bool", "break", "buffer", "bytearray",
+		"callable", "chr", "class", "classmethod", "coerce", "compile", "complex", "continue",
+		"def", "del", "delattr", "dict", "dir", "divmod",
+		"elif", "else", "enumerate", "eval", "except", "exec", "execfile",
+		"file", "filter", "finally", "float", "for", "format", "from", "frozenset",
+		"getattr", "global", "globals",
 		"hasattr", "hash", "help", "hex",
-		"id", "input", "int", "intern", "isinstance", "issubclass", "iter",
-		"len", "list", "locals",
-		"map", "max", "min", "next",
+		"id", "if", "import", "in", "input", "int", "intern", "is", "isinstance", "issubclass", "iter",
+		"lambda", "len", "list", "locals",
+		"map", "max", "min", "next", "not",
 		"memoryview",
-		"object", "oct", "open", "ord",
-		"pow", "print", "property",
-		"range", "raw_input", "reduce", "reload", "repr", "return", "reversed", "round",
+		"object", "oct", "open", "or", "ord",
+		"pass", "pow", "print", "property",
+		"raise", "range", "raw_input", "reduce", "reload", "repr", "return", "reversed", "round",
 		"set", "setattr", "slice", "sorted", "staticmethod", "str", "sum", "super",
-		"tuple", "type",
+		"try", "tuple", "type",
 		"unichr", "unicode",
 		"vars",
-		"with",
+		"while", "with",
 		"xrange",
+		"yield",
 		"zip",
 		"__import__",
 		"True", "False", "None"

tool/src/org/antlr/v4/codegen/target/Python3Target.java

@@ -24,26 +24,27 @@ import java.util.Set;
  */
 public class Python3Target extends Target {
 	protected static final String[] python3Keywords = {
-		"abs", "all", "any", "apply", "as",
-		"bin", "bool", "buffer", "bytearray",
-		"callable", "chr", "classmethod", "coerce", "compile", "complex",
-		"del", "delattr", "dict", "dir", "divmod",
-		"enumerate", "eval", "execfile",
-		"file", "filter", "float", "format", "frozenset",
-		"getattr", "globals",
+		"abs", "all", "and", "any", "apply", "as", "assert",
+		"bin", "bool", "break", "buffer", "bytearray",
+		"callable", "chr", "class", "classmethod", "coerce", "compile", "complex", "continue",
+		"def", "del", "delattr", "dict", "dir", "divmod",
+		"elif", "else", "enumerate", "eval", "execfile", "except",
+		"file", "filter", "finally", "float", "for", "format", "from", "frozenset",
+		"getattr", "global", "globals",
 		"hasattr", "hash", "help", "hex",
-		"id", "input", "int", "intern", "isinstance", "issubclass", "iter",
-		"len", "list", "locals",
-		"map", "max", "min", "next",
-		"memoryview",
-		"object", "oct", "open", "ord",
-		"pow", "print", "property",
-		"range", "raw_input", "reduce", "reload", "repr", "return", "reversed", "round",
+		"id", "if", "import", "in", "input", "int", "intern", "is", "isinstance", "issubclass", "iter",
+		"lambda", "len", "list", "locals",
+		"map", "max", "min", "memoryview",
+		"next", "nonlocal", "not",
+		"object", "oct", "open", "or", "ord",
+		"pass", "pow", "print", "property",
+		"raise", "range", "raw_input", "reduce", "reload", "repr", "return", "reversed", "round",
 		"set", "setattr", "slice", "sorted", "staticmethod", "str", "sum", "super",
-		"tuple", "type",
+		"try", "tuple", "type",
 		"unichr", "unicode",
 		"vars",
-		"with",
+		"with", "while",
+		"yield",
 		"zip",
 		"__import__",
 		"True", "False", "None"