diff --git a/.travis.yml b/.travis.yml index 2897de583..5cd9f7eb6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -151,32 +151,32 @@ matrix: - os: linux jdk: openjdk7 env: TARGET=csharp - stage: extended-test + stage: main-test - os: linux jdk: oraclejdk8 dist: trusty env: - TARGET=dotnet - GROUP=LEXER - stage: main-test + stage: extended-test - os: linux jdk: openjdk8 dist: trusty env: - TARGET=dotnet - GROUP=PARSER - stage: main-test + stage: extended-test - os: linux jdk: oraclejdk8 dist: trusty env: - TARGET=dotnet - GROUP=RECURSION - stage: main-test + stage: extended-test - os: linux jdk: openjdk7 env: TARGET=python2 - stage: extended-test + stage: main-test - os: linux jdk: openjdk7 env: TARGET=python3 diff --git a/contributors.txt b/contributors.txt index f3a9f8d43..05599c45d 100644 --- a/contributors.txt +++ b/contributors.txt @@ -180,5 +180,17 @@ YYYY/MM/DD, github id, Full name, email 2017/12/03, oranoran, Oran Epelbaum, oran / epelbaum me 2017/12/20, kbsletten, Kyle Sletten, kbsletten@gmail.com 2017/12/27, jkmar, Jakub Marciniszyn, marciniszyn.jk@gmail.com +2018/01/06, kasbah, Kaspar Emanuel, kaspar@monostable.co.uk +2018/02/08, razfriman, Raz Friman, raz@razfriman.com 2018/02/11, io7m, Mark Raynsford, code@io7m.com 2018/04/24, solussd, Joe Smith, joe@uwcreations.com +2018/05/15, johnvanderholt, jan dillingh johnvanderholte@gmail.com +2018/06/16, EternalPhane, Zongyuan Zuo, eternalphane@gmail.com +2018/05/15, johnvanderholt, jan dillingh johnvanderholte@gmail.com +2018/05/17, sinopsysHK, Eric Bardes, sinofwd@gmail.com +2018/05/23, srvance, Stephen Vance, steve@vance.com +2018/06/14, alecont, Alessandro Contenti, alecontenti@hotmail.com +2018/06/16, EternalPhane, Zongyuan Zuo, eternalphane@gmail.com +2018/07/03, jgoppert, James Goppert, james.goppert@gmail.com +2018/07/27, Maksim Novikov, mnovikov.work@gmail.com +2018/08/03, ENDOH takanao, djmchl@gmail.com \ No newline at end of file diff --git a/doc/csharp-target.md b/doc/csharp-target.md index 9158b0211..40be51840 100644 --- a/doc/csharp-target.md +++ b/doc/csharp-target.md @@ -21,7 +21,7 @@ You will find full instructions on the [Git repo page for ANTLR C# runtime](http Let's suppose that your grammar is named `MyGrammar`. The tool will generate for you the following files: -* MyGrammarLexer.cs +* MyGrammarLexer.cs * MyGrammarParser.cs * MyGrammarListener.cs (if you have not activated the -no-listener option) * MyGrammarBaseListener.cs (if you have not activated the -no-listener option) @@ -32,6 +32,7 @@ Now a fully functioning code might look like the following for start rule `Start ``` using Antlr4.Runtime; +using Antlr4.Runtime.Tree; public void MyParseMethod() { String input = "your text to parse here"; @@ -39,7 +40,7 @@ public void MyParseMethod() { ITokenSource lexer = new MyGrammarLexer(stream); ITokenStream tokens = new CommonTokenStream(lexer); MyGrammarParser parser = new MyGrammarParser(tokens); - parser.buildParseTrees = true; + parser.BuildParseTree = true; IParseTree tree = parser.StartRule(); } ``` diff --git a/doc/javascript-target.md b/doc/javascript-target.md index 3473e95b5..2d9973be6 100644 --- a/doc/javascript-target.md +++ b/doc/javascript-target.md @@ -62,7 +62,7 @@ The steps to create your parsing code are the following: You are now ready to bundle your parsing code as follows: - following webpack specs, create a webpack.config file - in the webpack.config file, exclude node.js only modules using: node: { module: "empty", net: "empty", fs: "empty" } - - from the cmd line, nag-vigate to the directory containing webpack.config and type: webpack + - from the cmd line, navigate to the directory containing webpack.config and type: webpack This will produce a single js file containing all your parsing code. Easy to include in your web pages! @@ -95,11 +95,16 @@ Let's suppose that your grammar is named, as above, "MyGrammar". Let's suppose t Now a fully functioning script might look like the following: ```javascript + var antlr4 = require('antlr4'); + var MyGrammarLexer = require('./MyGrammarLexer').MyGrammarLexer; + var MyGrammarParser = require('./MyGrammarParser').MyGrammarParser; + var MyGrammarListener = require('./MyGrammarListener').MyGrammarListener; + var input = "your text to parse here" var chars = new antlr4.InputStream(input); - var lexer = new MyGrammarLexer.MyGrammarLexer(chars); + var lexer = new MyGrammarLexer(chars); var tokens = new antlr4.CommonTokenStream(lexer); - var parser = new MyGrammarParser.MyGrammarParser(tokens); + var parser = new MyGrammarParser(tokens); parser.buildParseTrees = true; var tree = parser.MyStartRule(); ``` @@ -128,30 +133,30 @@ Let's suppose your MyGrammar grammar comprises 2 rules: "key" and "value". The a ``` In order to provide custom behavior, you might want to create the following class: - + ```javascript - KeyPrinter = function() { - MyGrammarListener.call(this); // inherit default listener - return this; - }; - -// inherit default listener +var KeyPrinter = function() { + MyGrammarListener.call(this); // inherit default listener + return this; +}; + +// continue inheriting default listener KeyPrinter.prototype = Object.create(MyGrammarListener.prototype); KeyPrinter.prototype.constructor = KeyPrinter; - + // override default listener behavior - KeyPrinter.prototype.exitKey = function(ctx) { - console.log("Oh, a key!"); - }; +KeyPrinter.prototype.exitKey = function(ctx) { + console.log("Oh, a key!"); +}; ``` In order to execute this listener, you would simply add the following lines to the above code: - + ```javascript - ... - tree = parser.StartRule() - only repeated here for reference - var printer = new KeyPrinter(); - antlr4.tree.ParseTreeWalker.DEFAULT.walk(printer, tree); + ... + tree = parser.StartRule() // only repeated here for reference +var printer = new KeyPrinter(); +antlr4.tree.ParseTreeWalker.DEFAULT.walk(printer, tree); ``` ## What about TypeScript? diff --git a/doc/releasing-antlr.md b/doc/releasing-antlr.md index e3ed7915b..239a57476 100644 --- a/doc/releasing-antlr.md +++ b/doc/releasing-antlr.md @@ -172,6 +172,7 @@ alias java='/Library/Java/JavaVirtualMachines/jdk1.7.0_21.jdk/Contents/Home/bin/ alias javac='/Library/Java/JavaVirtualMachines/jdk1.7.0_21.jdk/Contents/Home/bin/javac' alias javadoc='/Library/Java/JavaVirtualMachines/jdk1.7.0_21.jdk/Contents/Home/bin/javadoc' alias jar='/Library/Java/JavaVirtualMachines/jdk1.7.0_21.jdk/Contents/Home/bin/jar' +export JAVA_HOME=`/usr/libexec/java_home -v 1.7` ``` You should see 0x33 in generated .class files after 0xCAFEBABE; see [Java SE 7 = 51 (0x33 hex)](https://en.wikipedia.org/wiki/Java_class_file): diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/ParserExecDescriptors.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/ParserExecDescriptors.java index 297d2989e..ca6e393dd 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/ParserExecDescriptors.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/ParserExecDescriptors.java @@ -854,4 +854,39 @@ public class ParserExecDescriptors { @CommentHasStringValue public String grammar; } + + /** + * This is a regression test for antlr/antlr4#2301. + */ + public static class OrderingPredicates extends BaseParserTestDescriptor { + public String input = "POINT AT X"; + public String output = null; + public String errors = null; + public String startRule = "expr"; + public String grammarName = "Issue2301"; + + /** + grammar Issue2301; + + SPACES: [ \t\r\n]+ -> skip; + + AT: 'AT'; + X : 'X'; + Y : 'Y'; + + ID: [A-Z]+; + + constant + : 'DUMMY' + ; + + expr + : ID constant? + | expr AT X + | expr AT Y + ; + */ + @CommentHasStringValue + public String grammar; + } } diff --git a/runtime/CSharp/README.md b/runtime/CSharp/README.md index 265f2b4ee..3985a298b 100644 --- a/runtime/CSharp/README.md +++ b/runtime/CSharp/README.md @@ -51,7 +51,13 @@ This is just a quick start. The tool has many useful options to control generati The Antlr 4 standard runtime for C# is now available from NuGet. We trust that you know how to do add NuGet references to your project :-). -The package id is Antlr4.Runtime.Standard. We do not support other packages. +The package id is [Antlr4.Runtime.Standard](https://www.nuget.org/packages/Antlr4.Runtime.Standard/). We do not support other packages. + +Use the GUI or the following command in the Package Manager Console: + +``` +Install-Package Antlr4.Runtime.Standard +``` ### Step 6: You're done! diff --git a/runtime/Cpp/CMakeLists.txt b/runtime/Cpp/CMakeLists.txt index f4940c0c5..963250601 100644 --- a/runtime/Cpp/CMakeLists.txt +++ b/runtime/Cpp/CMakeLists.txt @@ -136,3 +136,7 @@ endif() install(FILES README.md VERSION DESTINATION "share/doc/libantlr4") + +set(CPACK_PACKAGE_CONTACT "antlr-discussion@googlegroups.com") +set(CPACK_PACKAGE_VERSION ${ANTLR_VERSION}) +include(CPack) diff --git a/runtime/Cpp/demo/Windows/antlr4-cpp-demo/antlr4-cpp-demo-vs2015.vcxproj..filters b/runtime/Cpp/demo/Windows/antlr4-cpp-demo/antlr4-cpp-demo-vs2015.vcxproj.filters similarity index 100% rename from runtime/Cpp/demo/Windows/antlr4-cpp-demo/antlr4-cpp-demo-vs2015.vcxproj..filters rename to runtime/Cpp/demo/Windows/antlr4-cpp-demo/antlr4-cpp-demo-vs2015.vcxproj.filters diff --git a/runtime/Cpp/runtime/src/support/Any.cpp b/runtime/Cpp/runtime/src/support/Any.cpp index 3dd1a94bf..b324cc15d 100644 --- a/runtime/Cpp/runtime/src/support/Any.cpp +++ b/runtime/Cpp/runtime/src/support/Any.cpp @@ -11,6 +11,3 @@ Any::~Any() { delete _ptr; } - -Any::Base::~Base() { -} diff --git a/runtime/Cpp/runtime/src/support/Any.h b/runtime/Cpp/runtime/src/support/Any.h index 3d8845c70..817490a3a 100644 --- a/runtime/Cpp/runtime/src/support/Any.h +++ b/runtime/Cpp/runtime/src/support/Any.h @@ -100,7 +100,7 @@ struct ANTLR4CPP_PUBLIC Any private: struct Base { - virtual ~Base(); + virtual ~Base() {}; virtual Base* clone() const = 0; }; @@ -112,10 +112,21 @@ private: T value; + Base* clone() const { + return clone<>(); + } + + private: + template::value, int>::type = 0> Base* clone() const { return new Derived(value); } + template::value, int>::type = 0> + Base* clone() const { + return nullptr; + } + }; Base* clone() const diff --git a/runtime/JavaScript/src/antlr4/tree/Tree.js b/runtime/JavaScript/src/antlr4/tree/Tree.js index 468d7a8a7..515b122b5 100644 --- a/runtime/JavaScript/src/antlr4/tree/Tree.js +++ b/runtime/JavaScript/src/antlr4/tree/Tree.js @@ -73,7 +73,11 @@ ParseTreeVisitor.prototype.visit = function(ctx) { }; ParseTreeVisitor.prototype.visitChildren = function(ctx) { - return this.visit(ctx.children); + if (ctx.children) { + return this.visit(ctx.children); + } else { + return null; + } } ParseTreeVisitor.prototype.visitTerminal = function(node) { diff --git a/runtime/Python3/src/antlr4/StdinStream.py b/runtime/Python3/src/antlr4/StdinStream.py new file mode 100644 index 000000000..f044fc4d7 --- /dev/null +++ b/runtime/Python3/src/antlr4/StdinStream.py @@ -0,0 +1,11 @@ +import codecs +import sys + +from antlr4.InputStream import InputStream + + +class StdinStream(InputStream): + def __init__(self, encoding:str='ascii', errors:str='strict') -> None: + bytes = sys.stdin.buffer.read() + data = codecs.decode(bytes, encoding, errors) + super().__init__(data) diff --git a/runtime/Python3/src/antlr4/atn/ParserATNSimulator.py b/runtime/Python3/src/antlr4/atn/ParserATNSimulator.py index d15629252..a3b2c7c14 100644 --- a/runtime/Python3/src/antlr4/atn/ParserATNSimulator.py +++ b/runtime/Python3/src/antlr4/atn/ParserATNSimulator.py @@ -1608,7 +1608,7 @@ class ParserATNSimulator(ATNSimulator): def reportAttemptingFullContext(self, dfa:DFA, conflictingAlts:set, configs:ATNConfigSet, startIndex:int, stopIndex:int): if ParserATNSimulator.debug or ParserATNSimulator.retry_debug: - interval = range(startIndex, stopIndex + 1) + interval = (startIndex, stopIndex + 1) print("reportAttemptingFullContext decision=" + str(dfa.decision) + ":" + str(configs) + ", input=" + self.parser.getTokenStream().getText(interval)) if self.parser is not None: @@ -1616,7 +1616,7 @@ class ParserATNSimulator(ATNSimulator): def reportContextSensitivity(self, dfa:DFA, prediction:int, configs:ATNConfigSet, startIndex:int, stopIndex:int): if ParserATNSimulator.debug or ParserATNSimulator.retry_debug: - interval = range(startIndex, stopIndex + 1) + interval = (startIndex, stopIndex + 1) print("reportContextSensitivity decision=" + str(dfa.decision) + ":" + str(configs) + ", input=" + self.parser.getTokenStream().getText(interval)) if self.parser is not None: @@ -1642,7 +1642,7 @@ class ParserATNSimulator(ATNSimulator): # } # i++; # } - interval = range(startIndex, stopIndex + 1) + interval = (startIndex, stopIndex + 1) print("reportAmbiguity " + str(ambigAlts) + ":" + str(configs) + ", input=" + self.parser.getTokenStream().getText(interval)) if self.parser is not None: diff --git a/runtime/Python3/src/antlr4/atn/SemanticContext.py b/runtime/Python3/src/antlr4/atn/SemanticContext.py index 548d7eaba..d4593195e 100644 --- a/runtime/Python3/src/antlr4/atn/SemanticContext.py +++ b/runtime/Python3/src/antlr4/atn/SemanticContext.py @@ -135,8 +135,8 @@ class PrecedencePredicate(SemanticContext): else: return None - def __cmp__(self, other): - return self.precedence - other.precedence + def __lt__(self, other): + return self.precedence < other.precedence def __hash__(self): return 31 diff --git a/tool/pom.xml b/tool/pom.xml index ae82015d2..dd3d5a9e6 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -44,7 +44,7 @@ com.ibm.icu icu4j - 58.2 + 61.1 diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg index 492c56644..1683a18f8 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg @@ -236,9 +236,12 @@ public class extends { public static final int = }; separator=", ", wrap, anchor>; - public static final String[] ruleNames = { - "}; separator=", ", wrap, anchor> - }; + private static String[] makeRuleNames() { + return new String[] { + "}; separator=", ", wrap, anchor> + }; + } + public static final String[] ruleNames = makeRuleNames(); @@ -275,12 +278,18 @@ case : >> vocabulary(literalNames, symbolicNames) ::= << -private static final String[] _LITERAL_NAMES = { - }; null="null", separator=", ", wrap, anchor> -}; -private static final String[] _SYMBOLIC_NAMES = { - }; null="null", separator=", ", wrap, anchor> -}; +private static String[] makeLiteralNames() { + return new String[] { + }; null="null", separator=", ", wrap, anchor> + }; +} +private static final String[] _LITERAL_NAMES = makeLiteralNames(); +private static String[] makeSymbolicNames() { + return new String[] { + }; null="null", separator=", ", wrap, anchor> + }; +} +private static final String[] _SYMBOLIC_NAMES = makeSymbolicNames(); public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES); /** @@ -914,9 +923,12 @@ public class extends { "}; separator=", ", wrap, anchor> }; - public static final String[] ruleNames = { - "}; separator=", ", wrap, anchor> - }; + private static String[] makeRuleNames() { + return new String[] { + "}; separator=", ", wrap, anchor> + }; + } + public static final String[] ruleNames = makeRuleNames(); diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg index 570f1659f..dcaa4463d 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Python2/Python2.stg @@ -749,6 +749,10 @@ import sys >> Lexer(lexer, atn, actionFuncs, sempredFuncs, superClass) ::= << + +from . import + + diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg index 34e525b85..8e03da79f 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Python3/Python3.stg @@ -119,7 +119,10 @@ Parser(parser, funcs, atn, sempredFuncs, superClass) ::= << Parser_(parser, funcs, atn, sempredFuncs, ctor, superClass) ::= << -from . import +if __name__ is not None and "." in __name__: + from . import +else: + from import @@ -756,7 +759,13 @@ import sys >> Lexer(lexer, atn, actionFuncs, sempredFuncs, superClass) ::= << + +if __name__ is not None and "." in __name__: + from . import +else: + from import + class (Lexer): diff --git a/tool/src/org/antlr/v4/codegen/target/Python2Target.java b/tool/src/org/antlr/v4/codegen/target/Python2Target.java index 3fc6a3541..28dfaae60 100644 --- a/tool/src/org/antlr/v4/codegen/target/Python2Target.java +++ b/tool/src/org/antlr/v4/codegen/target/Python2Target.java @@ -24,27 +24,28 @@ import java.util.Set; */ public class Python2Target extends Target { protected static final String[] python2Keywords = { - "abs", "all", "any", "apply", "as", - "bin", "bool", "buffer", "bytearray", - "callable", "chr", "classmethod", "coerce", "compile", "complex", - "del", "delattr", "dict", "dir", "divmod", - "enumerate", "eval", "execfile", - "file", "filter", "float", "format", "frozenset", - "getattr", "globals", + "abs", "all", "and", "any", "apply", "as", "assert", + "bin", "bool", "break", "buffer", "bytearray", + "callable", "chr", "class", "classmethod", "coerce", "compile", "complex", "continue", + "def", "del", "delattr", "dict", "dir", "divmod", + "elif", "else", "enumerate", "eval", "except", "exec", "execfile", + "file", "filter", "finally", "float", "for", "format", "from", "frozenset", + "getattr", "global", "globals", "hasattr", "hash", "help", "hex", - "id", "input", "int", "intern", "isinstance", "issubclass", "iter", - "len", "list", "locals", - "map", "max", "min", "next", + "id", "if", "import", "in", "input", "int", "intern", "is", "isinstance", "issubclass", "iter", + "lambda", "len", "list", "locals", + "map", "max", "min", "next", "not", "memoryview", - "object", "oct", "open", "ord", - "pow", "print", "property", - "range", "raw_input", "reduce", "reload", "repr", "return", "reversed", "round", + "object", "oct", "open", "or", "ord", + "pass", "pow", "print", "property", + "raise", "range", "raw_input", "reduce", "reload", "repr", "return", "reversed", "round", "set", "setattr", "slice", "sorted", "staticmethod", "str", "sum", "super", - "tuple", "type", + "try", "tuple", "type", "unichr", "unicode", "vars", - "with", + "while", "with", "xrange", + "yield", "zip", "__import__", "True", "False", "None" diff --git a/tool/src/org/antlr/v4/codegen/target/Python3Target.java b/tool/src/org/antlr/v4/codegen/target/Python3Target.java index 388269a73..1dffc4e42 100644 --- a/tool/src/org/antlr/v4/codegen/target/Python3Target.java +++ b/tool/src/org/antlr/v4/codegen/target/Python3Target.java @@ -24,26 +24,27 @@ import java.util.Set; */ public class Python3Target extends Target { protected static final String[] python3Keywords = { - "abs", "all", "any", "apply", "as", - "bin", "bool", "buffer", "bytearray", - "callable", "chr", "classmethod", "coerce", "compile", "complex", - "del", "delattr", "dict", "dir", "divmod", - "enumerate", "eval", "execfile", - "file", "filter", "float", "format", "frozenset", - "getattr", "globals", + "abs", "all", "and", "any", "apply", "as", "assert", + "bin", "bool", "break", "buffer", "bytearray", + "callable", "chr", "class", "classmethod", "coerce", "compile", "complex", "continue", + "def", "del", "delattr", "dict", "dir", "divmod", + "elif", "else", "enumerate", "eval", "execfile", "except", + "file", "filter", "finally", "float", "for", "format", "from", "frozenset", + "getattr", "global", "globals", "hasattr", "hash", "help", "hex", - "id", "input", "int", "intern", "isinstance", "issubclass", "iter", - "len", "list", "locals", - "map", "max", "min", "next", - "memoryview", - "object", "oct", "open", "ord", - "pow", "print", "property", - "range", "raw_input", "reduce", "reload", "repr", "return", "reversed", "round", + "id", "if", "import", "in", "input", "int", "intern", "is", "isinstance", "issubclass", "iter", + "lambda", "len", "list", "locals", + "map", "max", "min", "memoryview", + "next", "nonlocal", "not", + "object", "oct", "open", "or", "ord", + "pass", "pow", "print", "property", + "raise", "range", "raw_input", "reduce", "reload", "repr", "return", "reversed", "round", "set", "setattr", "slice", "sorted", "staticmethod", "str", "sum", "super", - "tuple", "type", + "try", "tuple", "type", "unichr", "unicode", "vars", - "with", + "with", "while", + "yield", "zip", "__import__", "True", "False", "None"