From a60fcfed0ff356306c54c9f4c30598e827e3af2d Mon Sep 17 00:00:00 2001 From: Mike Lischke Date: Sun, 1 Jan 2017 17:39:44 +0100 Subject: [PATCH 01/23] Removed the need of a shadow s0 copy in the DFA class. --- .../runtime/src/atn/ParserATNSimulator.cpp | 2 ++ .../Cpp/runtime/src/atn/PredictionContext.cpp | 4 +++- runtime/Cpp/runtime/src/dfa/DFA.cpp | 21 +++++++++---------- runtime/Cpp/runtime/src/dfa/DFA.h | 5 ++--- .../v4/tool/templates/codegen/Cpp/Cpp.stg | 2 +- 5 files changed, 18 insertions(+), 16 deletions(-) diff --git a/runtime/Cpp/runtime/src/atn/ParserATNSimulator.cpp b/runtime/Cpp/runtime/src/atn/ParserATNSimulator.cpp index 5d4aa146d..b15346ffe 100755 --- a/runtime/Cpp/runtime/src/atn/ParserATNSimulator.cpp +++ b/runtime/Cpp/runtime/src/atn/ParserATNSimulator.cpp @@ -124,6 +124,8 @@ size_t ParserATNSimulator::adaptivePredict(TokenStream *input, size_t decision, } else { dfa::DFAState *newState = new dfa::DFAState(std::move(s0_closure)); /* mem-check: managed by the DFA or deleted below */ s0 = addDFAState(dfa, newState); + + delete dfa.s0; // Delete existing s0 DFA state, if there's any. dfa.s0 = s0; if (s0 != newState) { delete newState; // If there was already a state with this config set we don't need the new one. diff --git a/runtime/Cpp/runtime/src/atn/PredictionContext.cpp b/runtime/Cpp/runtime/src/atn/PredictionContext.cpp index 9bee78451..86639d289 100755 --- a/runtime/Cpp/runtime/src/atn/PredictionContext.cpp +++ b/runtime/Cpp/runtime/src/atn/PredictionContext.cpp @@ -350,8 +350,10 @@ Ref PredictionContext::mergeArrays(const Ref(mergedParents, mergedReturnStates); + } if (mergeCache != nullptr) { mergeCache->put(a, b, M); diff --git a/runtime/Cpp/runtime/src/dfa/DFA.cpp b/runtime/Cpp/runtime/src/dfa/DFA.cpp index 309fdb27a..a637bf72c 100755 --- a/runtime/Cpp/runtime/src/dfa/DFA.cpp +++ b/runtime/Cpp/runtime/src/dfa/DFA.cpp @@ -26,35 +26,34 @@ DFA::DFA(atn::DecisionState *atnStartState, size_t decision) if (static_cast(atnStartState)->isPrecedenceDecision) { _precedenceDfa = true; s0 = new DFAState(std::unique_ptr(new atn::ATNConfigSet())); - _s0Shadow = s0; s0->isAcceptState = false; s0->requiresFullContext = false; } } } -DFA::DFA(DFA &&other) : atnStartState(std::move(other.atnStartState)), decision(std::move(other.decision)) { +DFA::DFA(DFA &&other) : atnStartState(other.atnStartState), decision(other.decision) { // Source states are implicitly cleared by the move. states = std::move(other.states); - // Manually move s0 pointers. + other.atnStartState = nullptr; + other.decision = 0; s0 = other.s0; other.s0 = nullptr; - _s0Shadow = other._s0Shadow; - other._s0Shadow = nullptr; - _precedenceDfa = other._precedenceDfa; + other._precedenceDfa = false; } DFA::~DFA() { - // ml: s0 can be set either in our constructor or by external code, so we need a way to track our own creation. - // We could use a shared pointer again (and force that on all external assignments etc.) or just shadow it. - // I hesitate moving s0 to the normal states list as this might conflict with consumers of that list. - delete _s0Shadow; - + bool s0InList = (s0 == nullptr); for (auto state : states) { + if (state == s0) + s0InList = true; delete state; } + + if (!s0InList) + delete s0; } bool DFA::isPrecedenceDfa() const { diff --git a/runtime/Cpp/runtime/src/dfa/DFA.h b/runtime/Cpp/runtime/src/dfa/DFA.h index b5d061ac4..7d2f653c1 100755 --- a/runtime/Cpp/runtime/src/dfa/DFA.h +++ b/runtime/Cpp/runtime/src/dfa/DFA.h @@ -20,10 +20,10 @@ namespace dfa { /// Set only allows you to see if it's there. /// From which ATN state did we create this DFA? - atn::DecisionState *const atnStartState; + atn::DecisionState *atnStartState; std::unordered_set states; // States are owned by this class. DFAState *s0; - const size_t decision; + size_t decision; DFA(atn::DecisionState *atnStartState); DFA(atn::DecisionState *atnStartState, size_t decision); @@ -85,7 +85,6 @@ namespace dfa { * {@code false}. This is the backing field for {@link #isPrecedenceDfa}. */ bool _precedenceDfa; - DFAState *_s0Shadow = nullptr; // ml: assigned when we created s0 ourselves. }; } // namespace atn diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg index 78f961083..4e9352475 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg @@ -1014,7 +1014,7 @@ ContextRuleListIndexedGetterDecl(r) ::= << >> -LexerRuleContext() ::= "antlr4::RuleContext" +LexerRuleContext() ::= "RuleContext" // The rule context name is the rule followed by a suffix; e.g. r becomes rContext. RuleContextNameSuffix() ::= "Context" From 47e43cc8d6b5d1dfe5ff494503e6d0eec63d4b39 Mon Sep 17 00:00:00 2001 From: Mike Lischke Date: Wed, 19 Apr 2017 13:47:12 +0200 Subject: [PATCH 02/23] Simplify output path generation The twisted way to create a javac like output path breaks many use cases, including very simple ones with lexer + parser grammar in a relative subdir (where parser generation doesn't find the lexer tokens). Additionally, not all projects prefer to have their generated files in an output folder which is comprised of the given output path and the subdir given for the grammar files. The check for absolute paths as decision criterion is a weak one anyway - it makes simple usage difficult and doesn't deal with path expansion on all platforms. If a javac like output path is required it should be easy to construct this upfront and provide it as the output path parameter, instead of implicitely creating it. This patch simplifies things without any WTF moments because generated files end up and unexpected locations. It now can deal with simple cases (no output, no grammar subdir) up to complicated cases (nested and/or absolute output dir, multiple grammar subfolders). --- tool/src/org/antlr/v4/Tool.java | 26 +++----------------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/tool/src/org/antlr/v4/Tool.java b/tool/src/org/antlr/v4/Tool.java index 42ace2ecb..6638ff5a1 100644 --- a/tool/src/org/antlr/v4/Tool.java +++ b/tool/src/org/antlr/v4/Tool.java @@ -761,18 +761,11 @@ public class Tool { File outputDir; String fileDirectory; - // Some files are given to us without a PATH but should should - // still be written to the output directory in the relative path of - // the output directory. The file directory is either the set of sub directories - // or just or the relative path recorded for the parent grammar. This means - // that when we write the tokens files, or the .java files for imported grammars - // taht we will write them in the correct place. if (fileNameWithPath.lastIndexOf(File.separatorChar) == -1) { // No path is included in the file name, so make the file - // directory the same as the parent grammar (which might sitll be just "" + // directory the same as the parent grammar (which might still be just "" // but when it is not, we will write the file in the correct place. fileDirectory = "."; - } else { fileDirectory = fileNameWithPath.substring(0, fileNameWithPath.lastIndexOf(File.separatorChar)); @@ -781,21 +774,8 @@ public class Tool { // -o /tmp /var/lib/t.g4 => /tmp/T.java // -o subdir/output /usr/lib/t.g4 => subdir/output/T.java // -o . /usr/lib/t.g4 => ./T.java - if (fileDirectory != null && - (new File(fileDirectory).isAbsolute() || - fileDirectory.startsWith("~"))) { // isAbsolute doesn't count this :( - // somebody set the dir, it takes precendence; write new file there - outputDir = new File(outputDirectory); - } - else { - // -o /tmp subdir/t.g4 => /tmp/subdir/t.g4 - if (fileDirectory != null) { - outputDir = new File(outputDirectory, fileDirectory); - } - else { - outputDir = new File(outputDirectory); - } - } + // -o /tmp subdir/t.g4 => /tmp/t.g4 + outputDir = new File(outputDirectory); } else { // they didn't specify a -o dir so just write to location From defad74649538a5eff872ee4af486a5ef88bd145 Mon Sep 17 00:00:00 2001 From: Mike Lischke Date: Wed, 19 Apr 2017 15:48:07 +0200 Subject: [PATCH 03/23] Extended tokenVocab search to the grammar subfolder. --- .../org/antlr/v4/parse/TokenVocabParser.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tool/src/org/antlr/v4/parse/TokenVocabParser.java b/tool/src/org/antlr/v4/parse/TokenVocabParser.java index 4267a8d96..cf84a923a 100644 --- a/tool/src/org/antlr/v4/parse/TokenVocabParser.java +++ b/tool/src/org/antlr/v4/parse/TokenVocabParser.java @@ -146,6 +146,22 @@ public class TokenVocabParser { // files are generated (in the base, not relative to the input // location.) f = new File(g.tool.outputDirectory, vocabName + CodeGenerator.VOCAB_FILE_EXTENSION); - return f; + if ( f.exists() ) { + return f; + } + + // Still not found? Use the grammar's subfolder then. + String fileDirectory; + + if (g.fileName.lastIndexOf(File.separatorChar) == -1) { + // No path is included in the file name, so make the file + // directory the same as the parent grammar (which might still be just "" + // but when it is not, we will write the file in the correct place. + fileDirectory = "."; + } + else { + fileDirectory = g.fileName.substring(0, g.fileName.lastIndexOf(File.separatorChar)); + } + return new File(fileDirectory, vocabName + CodeGenerator.VOCAB_FILE_EXTENSION); } } From d12e755c022ac5fef6a3db91ca00965544c139ad Mon Sep 17 00:00:00 2001 From: Mike Lischke Date: Wed, 26 Apr 2017 16:18:50 +0200 Subject: [PATCH 04/23] Properly sorted exceptions in the forward declaration list. Removed some duplicate classes there too. --- runtime/Cpp/runtime/src/support/Declarations.h | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/runtime/Cpp/runtime/src/support/Declarations.h b/runtime/Cpp/runtime/src/support/Declarations.h index d80891c15..1534c919e 100644 --- a/runtime/Cpp/runtime/src/support/Declarations.h +++ b/runtime/Cpp/runtime/src/support/Declarations.h @@ -6,16 +6,6 @@ #pragma once namespace antlr4 { - class IllegalStateException; - class IllegalArgumentException; - class NoSuchElementException; - class NullPointerException; - class InputMismatchException; - class ParseCancellationException; - class InputMismatchException; - class EmptyStackException; - class LexerNoViableAltException; - class ANTLRErrorListener; class ANTLRErrorStrategy; class ANTLRFileStream; @@ -30,7 +20,10 @@ namespace antlr4 { class ConsoleErrorListener; class DefaultErrorStrategy; class DiagnosticErrorListener; + class EmptyStackException; class FailedPredicateException; + class IllegalArgumentException; + class IllegalStateException; class InputMismatchException; class IntStream; class InterpreterRuleContext; @@ -38,7 +31,10 @@ namespace antlr4 { class LexerInterpreter; class LexerNoViableAltException; class ListTokenSource; + class NoSuchElementException; class NoViableAltException; + class NullPointerException; + class ParseCancellationException; class Parser; class ParserInterpreter; class ParserRuleContext; @@ -59,7 +55,6 @@ namespace antlr4 { class Interval; class IntervalSet; class MurmurHash; - class ParseCancellationException; class Utils; class Predicate; } From 67f683d0ac35019e76cdfac9cd71fe37b548c35d Mon Sep 17 00:00:00 2001 From: Hanzhou Shi Date: Tue, 9 May 2017 22:06:36 -0700 Subject: [PATCH 05/23] Adding generation of dyanmically linked lib. --- runtime/Swift/Package.swift | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/runtime/Swift/Package.swift b/runtime/Swift/Package.swift index a0c9a926d..5c2e28b12 100644 --- a/runtime/Swift/Package.swift +++ b/runtime/Swift/Package.swift @@ -7,3 +7,13 @@ import PackageDescription let package = Package( name: "Antlr4" ) + +products.append( + Product( + name: "Antlr4", + type: .Library(.Dynamic), + modules: [ + "Antlr4" + ] + ) +) From d66f89db527c3335852459063f43c529c93a66cf Mon Sep 17 00:00:00 2001 From: Hanzhou Shi Date: Tue, 9 May 2017 22:54:25 -0700 Subject: [PATCH 06/23] Removed xcworkspace. --- runtime/Swift/Antlr4.xcworkspace/contents.xcworkspacedata | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 runtime/Swift/Antlr4.xcworkspace/contents.xcworkspacedata diff --git a/runtime/Swift/Antlr4.xcworkspace/contents.xcworkspacedata b/runtime/Swift/Antlr4.xcworkspace/contents.xcworkspacedata deleted file mode 100644 index 6584c73ac..000000000 --- a/runtime/Swift/Antlr4.xcworkspace/contents.xcworkspacedata +++ /dev/null @@ -1,7 +0,0 @@ - - - - - From 6acec92866552a46afea96d23352ebfcd26f6a0f Mon Sep 17 00:00:00 2001 From: Hanzhou Shi Date: Wed, 10 May 2017 23:29:25 -0700 Subject: [PATCH 07/23] Refactoring swift test framework. --- .../antlr/v4/test/runtime/go/BaseGoTest.java | 2 +- .../v4/test/runtime/swift/BaseSwiftTest.java | 182 ++++++------------ 2 files changed, 59 insertions(+), 125 deletions(-) diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/go/BaseGoTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/go/BaseGoTest.java index 48791461f..03c41b2f9 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/go/BaseGoTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/go/BaseGoTest.java @@ -326,7 +326,7 @@ public class BaseGoTest implements RuntimeTestSupport { // writeFile(overall_tmpdir, "input", input); // rawBuildRecognizerTestFile(parserName, lexerName, listenerName, // visitorName, startRuleName, debug); -// return execRecognizer(); +// return execParser(); // } @Override diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java index a3da071dc..f044602d7 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java @@ -6,7 +6,6 @@ package org.antlr.v4.test.runtime.swift; -import org.antlr.v4.Tool; import org.antlr.v4.test.runtime.ErrorQueue; import org.antlr.v4.test.runtime.RuntimeTestSupport; import org.antlr.v4.test.runtime.StreamVacuum; @@ -29,16 +28,21 @@ import static org.junit.Assert.assertTrue; public class BaseSwiftTest implements RuntimeTestSupport { /** - * The base test directory is the directory where generated files get placed - * during unit test execution. + * Path of the ANTLR runtime. */ - private static final String BASE_TEST_DIR; + private static String ANTLR_RUNTIME_PATH; - private static String ANTLR_FRAMEWORK_DIR; + private static void exec(String workingDir, String... command) { + ProcessBuilder builder = new ProcessBuilder(command); + builder.directory(new File(workingDir)); + try { + Process p = builder.start(); + p.waitFor(); + } catch (Exception e) { + e.printStackTrace(); + } + } - /** - * Common routine to setup the ANTLR4 runtime. - */ static { String baseTestDir = System.getProperty("antlr-swift-test-dir"); if (baseTestDir == null || baseTestDir.isEmpty()) { @@ -49,84 +53,41 @@ public class BaseSwiftTest implements RuntimeTestSupport { throw new UnsupportedOperationException("The specified base test directory does not exist: " + baseTestDir); } - BASE_TEST_DIR = baseTestDir; - - //add antlr.swift final ClassLoader loader = Thread.currentThread().getContextClassLoader(); - final URL swiftRuntime = loader.getResource("Swift/Sources/Antlr4"); + // build swift runtime + final URL swiftRuntime = loader.getResource("Swift"); if (swiftRuntime == null) { throw new RuntimeException("Swift runtime file not found at:" + swiftRuntime.getPath()); } - String swiftRuntimePath = swiftRuntime.getPath(); + ANTLR_RUNTIME_PATH = swiftRuntime.getPath(); + exec(ANTLR_RUNTIME_PATH, "swift", "build"); - try { - String commandLine = "find " + swiftRuntimePath + "/ -iname *.swift -not -name merge.swift -exec cat {} ;"; - ProcessBuilder builder = new ProcessBuilder(commandLine.split(" ")); - builder.redirectError(ProcessBuilder.Redirect.INHERIT); - Process p = builder.start(); - StreamVacuum stdoutVacuum = new StreamVacuum(p.getInputStream()); - stdoutVacuum.start(); - p.waitFor(); - stdoutVacuum.join(); - - String antlrSwift = stdoutVacuum.toString(); - //write to Antlr4 - ANTLR_FRAMEWORK_DIR = new File(BASE_TEST_DIR, "Antlr4").getAbsolutePath(); - mkdir(ANTLR_FRAMEWORK_DIR); - writeFile(ANTLR_FRAMEWORK_DIR, "Antlr4.swift", antlrSwift); - //compile Antlr4 module - buildAntlr4Framework(); - String argsString; - } - catch (Exception e) { - e.printStackTrace(System.err); - } + // shutdown logic Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { - // shutdown logic - eraseAntlrFrameWorkDir(); + exec(ANTLR_RUNTIME_PATH, "swift", "package", "clean"); } }); } - private static void eraseFilesIn(String dirName) { - if (dirName == null) { - return; - } - - File dir = new File(dirName); - String[] files = dir.list(); - if (files != null) for (String file : files) { - new File(dirName + "/" + file).delete(); - } - } - - private static void eraseAntlrFrameWorkDir() { - File frameworkdir = new File(ANTLR_FRAMEWORK_DIR); - if (frameworkdir.exists()) { - eraseFilesIn(ANTLR_FRAMEWORK_DIR); - frameworkdir.delete(); - } - } - - private static boolean buildAntlr4Framework() throws Exception { - String argsString = "xcrun -sdk macosx swiftc -emit-library -emit-module Antlr4.swift -module-name Antlr4 -module-link-name Antlr4 -Xlinker -install_name -Xlinker " + ANTLR_FRAMEWORK_DIR + "/libAntlr4.dylib "; - return runProcess(argsString, ANTLR_FRAMEWORK_DIR); - } - public String tmpdir = null; + /** + * If error during parser execution, store stderr here; can't return + * stdout and stderr. This doesn't trap errors from running antlr. + */ + private String stderrDuringParse; + /** * Errors found while running antlr */ private StringBuilder antlrToolErrors; /** - * If error during parser execution, store stderr here; can't return - * stdout and stderr. This doesn't trap errors from running antlr. + * Source files used in each small swift project. */ - protected String stderrDuringParse; + private Set sourceFiles = new HashSet(); @Override public void testSetUp() throws Exception { @@ -137,21 +98,20 @@ public class BaseSwiftTest implements RuntimeTestSupport { tmpdir = prop; } else { - tmpdir = new File(System.getProperty("java.io.tmpdir"), getClass().getSimpleName() + - "-" + Thread.currentThread().getName() + "-" + System.currentTimeMillis()).getAbsolutePath(); + String classSimpleName = getClass().getSimpleName(); + String threadName = Thread.currentThread().getName(); + String childPath = String.format("%s-%s-%s", classSimpleName, threadName, System.currentTimeMillis()); + tmpdir = new File(System.getProperty("java.io.tmpdir"), childPath).getAbsolutePath(); } antlrToolErrors = new StringBuilder(); - } @Override public void testTearDown() throws Exception { - } @Override public void eraseTempDir() { - } @Override @@ -179,7 +139,7 @@ public class BaseSwiftTest implements RuntimeTestSupport { @Override public String execLexer(String grammarFileName, String grammarStr, String lexerName, String input, boolean showDFA) { - boolean success = rawGenerateRecognizer(grammarFileName, + boolean success = generateParser(grammarFileName, grammarStr, null, lexerName); @@ -188,16 +148,17 @@ public class BaseSwiftTest implements RuntimeTestSupport { writeLexerTestFile(lexerName, showDFA); addSourceFiles("main.swift"); - compile(); - String output = execTest(); - return output; +// compile(); +// String output = execTest(); +// return output; + return ""; // TODO: } private String execTest() { try { String exec = tmpdir + "/" + EXEC_NAME; String[] args = - new String[]{exec, "input"};//new File(tmpdir, "input").getAbsolutePath() + new String[]{exec, "input"}; ProcessBuilder pb = new ProcessBuilder(args); pb.directory(new File(tmpdir)); Process p = pb.start(); @@ -224,13 +185,11 @@ public class BaseSwiftTest implements RuntimeTestSupport { return null; } - private Set sourceFiles = new HashSet(); - private void addSourceFiles(String... files) { Collections.addAll(this.sourceFiles, files); } - public boolean compile() { + private boolean compile() { try { return buildProject(); } catch (Exception e) { @@ -244,13 +203,13 @@ public class BaseSwiftTest implements RuntimeTestSupport { String fileList = sourceFiles.toString().replace("[", "").replace("]", "") .replace(", ", " "); - String argsString = "xcrun -sdk macosx swiftc " + fileList + " -o " + EXEC_NAME + " -I " + ANTLR_FRAMEWORK_DIR + " -L " + ANTLR_FRAMEWORK_DIR + " -module-link-name Antlr4 -suppress-warnings"; - return runProcess(argsString, tmpdir); +// String argsString = "xcrun -sdk macosx swiftc " + fileList + " -o " + EXEC_NAME + " -I " + ANTLR_FRAMEWORK_DIR + " -L " + ANTLR_FRAMEWORK_DIR + " -module-link-name Antlr4 -suppress-warnings"; +// return runProcess(argsString, tmpdir); + return true; } private static boolean runProcess(String argsString, String execPath) throws IOException, InterruptedException { String[] args = argsString.split(" "); -// System.err.println("Starting build " + argsString);//Utils.join(args, " ")) Process process = Runtime.getRuntime().exec(args, null, new File(execPath)); StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream()); StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream()); @@ -272,31 +231,31 @@ public class BaseSwiftTest implements RuntimeTestSupport { lexerName, startRuleName, input, showDiagnosticErrors, false); } - protected String execParser(String grammarFileName, + private String execParser(String grammarFileName, String grammarStr, String parserName, String lexerName, String startRuleName, String input, boolean debug,boolean profile) { - boolean success = rawGenerateRecognizer(grammarFileName, + boolean success = generateParser(grammarFileName, grammarStr, parserName, lexerName, "-visitor"); assertTrue(success); writeFile(tmpdir, "input", input); - return rawExecRecognizer(parserName, + return execParser(parserName, lexerName, startRuleName, debug,profile); } - protected String rawExecRecognizer(String parserName, - String lexerName, - String parserStartRuleName, - boolean debug, - boolean profile) + private String execParser(String parserName, + String lexerName, + String parserStartRuleName, + boolean debug, + boolean profile) { this.stderrDuringParse = null; if ( parserName==null ) { @@ -311,15 +270,15 @@ public class BaseSwiftTest implements RuntimeTestSupport { } addSourceFiles("main.swift"); - return execRecognizer(); + return execParser(); } - public String execRecognizer() { + private String execParser() { compile(); return execTest(); } - protected void writeParserTestFile(String parserName, + private void writeParserTestFile(String parserName, String lexerName, String parserStartRuleName, boolean debug, @@ -384,7 +343,7 @@ public class BaseSwiftTest implements RuntimeTestSupport { writeFile(tmpdir, "main.swift", outputFileST.render()); } - protected void writeLexerTestFile(String lexerName, boolean showDFA) { + private void writeLexerTestFile(String lexerName, boolean showDFA) { ST outputFileST = new ST( "import Antlr4\n" + "import Foundation\n" + @@ -417,24 +376,12 @@ public class BaseSwiftTest implements RuntimeTestSupport { /** * Return true if all is well */ - private boolean rawGenerateRecognizer(String grammarFileName, - String grammarStr, - String parserName, - String lexerName, - String... extraOptions) { - return rawGenerateRecognizer(grammarFileName, grammarStr, parserName, lexerName, false, extraOptions); - } - - /** - * Return true if all is well - */ - private boolean rawGenerateRecognizer(String grammarFileName, - String grammarStr, - String parserName, - String lexerName, - boolean defaultListener, - String... extraOptions) { - ErrorQueue equeue = antlrOnString(getTmpDir(), "Swift", grammarFileName, grammarStr, defaultListener, extraOptions); + private boolean generateParser(String grammarFileName, + String grammarStr, + String parserName, + String lexerName, + String... extraOptions) { + ErrorQueue equeue = antlrOnString(getTmpDir(), "Swift", grammarFileName, grammarStr, false, extraOptions); if (!equeue.errors.isEmpty()) { return false; } @@ -461,17 +408,4 @@ public class BaseSwiftTest implements RuntimeTestSupport { addSourceFiles(files.toArray(new String[files.size()])); return true; } - - protected static void mkdir(String dir) { - File f = new File(dir); - f.mkdirs(); - } - - protected Tool newTool(String[] args) { - return new Tool(args); - } - - protected Tool newTool() { - return new Tool(new String[]{"-o", tmpdir}); - } } From 8cbd7c0693445c554ea3d65f5beda9cdc3e26799 Mon Sep 17 00:00:00 2001 From: Hanzhou Shi Date: Thu, 11 May 2017 23:32:49 -0700 Subject: [PATCH 08/23] Get parser tests working. --- .../v4/test/runtime/swift/BaseSwiftTest.java | 140 +++++++----------- 1 file changed, 56 insertions(+), 84 deletions(-) diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java index f044602d7..78dd08c2a 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Set; import static org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString; +import static org.antlr.v4.test.runtime.BaseRuntimeTest.mkdir; import static org.antlr.v4.test.runtime.BaseRuntimeTest.writeFile; import static org.junit.Assert.assertTrue; @@ -87,7 +88,7 @@ public class BaseSwiftTest implements RuntimeTestSupport { /** * Source files used in each small swift project. */ - private Set sourceFiles = new HashSet(); + private Set sourceFiles = new HashSet<>(); @Override public void testSetUp() throws Exception { @@ -139,11 +140,10 @@ public class BaseSwiftTest implements RuntimeTestSupport { @Override public String execLexer(String grammarFileName, String grammarStr, String lexerName, String input, boolean showDFA) { - boolean success = generateParser(grammarFileName, + generateParser(grammarFileName, grammarStr, null, lexerName); - assertTrue(success); writeFile(tmpdir, "input", input); writeLexerTestFile(lexerName, showDFA); addSourceFiles("main.swift"); @@ -154,32 +154,12 @@ public class BaseSwiftTest implements RuntimeTestSupport { return ""; // TODO: } - private String execTest() { + private String execTest(String projectDir, String projectName) { try { - String exec = tmpdir + "/" + EXEC_NAME; - String[] args = - new String[]{exec, "input"}; - ProcessBuilder pb = new ProcessBuilder(args); - pb.directory(new File(tmpdir)); - Process p = pb.start(); - StreamVacuum stdoutVacuum = new StreamVacuum(p.getInputStream()); - StreamVacuum stderrVacuum = new StreamVacuum(p.getErrorStream()); - stdoutVacuum.start(); - stderrVacuum.start(); - p.waitFor(); - stdoutVacuum.join(); - stderrVacuum.join(); - String output = stdoutVacuum.toString(); - if ( output.length()==0 ) { - output = null; - } - if (stderrVacuum.toString().length() > 0) { - this.stderrDuringParse = stderrVacuum.toString(); - } - return output; + return runProcess(projectDir, "./.build/debug/" + projectName, "input"); } catch (Exception e) { - System.err.println("can't exec recognizer"); + System.err.println("Execution of testcase failed."); e.printStackTrace(System.err); } return null; @@ -189,27 +169,35 @@ public class BaseSwiftTest implements RuntimeTestSupport { Collections.addAll(this.sourceFiles, files); } - private boolean compile() { - try { - return buildProject(); - } catch (Exception e) { - return false; + private void listSourceFiles() { + for (String sourceFile: this.sourceFiles) { + System.out.println(sourceFile); } } private static final String EXEC_NAME = "Test"; - private boolean buildProject() throws Exception { - String fileList = sourceFiles.toString().replace("[", "").replace("]", "") - .replace(", ", " "); + private void buildProject(String projectDir) { + mkdir(projectDir); + exec(projectDir, "swift", "package", "init", "--type", "executable"); + for (String sourceFile: sourceFiles) { + String absPath = getTmpDir() + "/" + sourceFile; + exec(getTmpDir(), "mv", "-f", absPath, projectDir + "/Sources/"); + } + exec(getTmpDir(), "mv", "-f", "input", projectDir); -// String argsString = "xcrun -sdk macosx swiftc " + fileList + " -o " + EXEC_NAME + " -I " + ANTLR_FRAMEWORK_DIR + " -L " + ANTLR_FRAMEWORK_DIR + " -module-link-name Antlr4 -suppress-warnings"; -// return runProcess(argsString, tmpdir); - return true; + try { + String dylibPath = ANTLR_RUNTIME_PATH + "/.build/debug/"; + runProcess(projectDir, "swift", "build", + "-Xswiftc", "-I"+dylibPath, + "-Xlinker", "-L"+dylibPath, + "-Xlinker", "-lAntlr4"); + } catch (IOException | InterruptedException e) { + e.printStackTrace(); + } } - private static boolean runProcess(String argsString, String execPath) throws IOException, InterruptedException { - String[] args = argsString.split(" "); + private static String runProcess(String execPath, String... args) throws IOException, InterruptedException { Process process = Runtime.getRuntime().exec(args, null, new File(execPath)); StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream()); StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream()); @@ -219,43 +207,30 @@ public class BaseSwiftTest implements RuntimeTestSupport { stdoutVacuum.join(); stderrVacuum.join(); if (stderrVacuum.toString().length() > 0) { - //this.stderrDuringParse = stderrVacuum.toString(); - System.err.println("buildProject stderrVacuum: " + stderrVacuum); + throw new RuntimeException(stderrVacuum.toString()); } - return process.exitValue() == 0; + return stdoutVacuum.toString(); } @Override public String execParser(String grammarFileName, String grammarStr, String parserName, String lexerName, String listenerName, String visitorName, String startRuleName, String input, boolean showDiagnosticErrors) { - return execParser(grammarFileName, grammarStr, parserName, - lexerName, startRuleName, input, showDiagnosticErrors, false); - } - - private String execParser(String grammarFileName, - String grammarStr, - String parserName, - String lexerName, - String startRuleName, - String input, boolean debug,boolean profile) - { - boolean success = generateParser(grammarFileName, + generateParser(grammarFileName, grammarStr, parserName, lexerName, "-visitor"); - assertTrue(success); - writeFile(tmpdir, "input", input); + writeFile(getTmpDir(), "input", input); return execParser(parserName, lexerName, startRuleName, - debug,profile); + showDiagnosticErrors,false); } private String execParser(String parserName, - String lexerName, - String parserStartRuleName, - boolean debug, - boolean profile) + String lexerName, + String parserStartRuleName, + boolean debug, + boolean profile) { this.stderrDuringParse = null; if ( parserName==null ) { @@ -270,19 +245,17 @@ public class BaseSwiftTest implements RuntimeTestSupport { } addSourceFiles("main.swift"); - return execParser(); - } - - private String execParser() { - compile(); - return execTest(); + String projectName = "testcase-" + System.currentTimeMillis(); + String projectDir = getTmpDir() + "/" + projectName; + buildProject(projectDir); + return execTest(projectDir, projectName); } private void writeParserTestFile(String parserName, - String lexerName, - String parserStartRuleName, - boolean debug, - boolean profile) { + String lexerName, + String parserStartRuleName, + boolean debug, + boolean profile) { ST outputFileST = new ST( "import Antlr4\n" + @@ -345,7 +318,7 @@ public class BaseSwiftTest implements RuntimeTestSupport { private void writeLexerTestFile(String lexerName, boolean showDFA) { ST outputFileST = new ST( - "import Antlr4\n" + + "import Antlr4\n" + "import Foundation\n" + "setbuf(__stdoutp, nil)\n" + @@ -374,27 +347,27 @@ public class BaseSwiftTest implements RuntimeTestSupport { } /** - * Return true if all is well + * Generates the parser for one test case. */ - private boolean generateParser(String grammarFileName, - String grammarStr, - String parserName, - String lexerName, - String... extraOptions) { + private void generateParser(String grammarFileName, + String grammarStr, + String parserName, + String lexerName, + String... extraOptions) { ErrorQueue equeue = antlrOnString(getTmpDir(), "Swift", grammarFileName, grammarStr, false, extraOptions); - if (!equeue.errors.isEmpty()) { - return false; - } + assertTrue(equeue.errors.isEmpty()); + System.out.println(getTmpDir()); - List files = new ArrayList(); + List files = new ArrayList<>(); if (lexerName != null) { files.add(lexerName + ".swift"); files.add(lexerName + "ATN.swift"); } + if (parserName != null) { files.add(parserName + ".swift"); files.add(parserName + "ATN.swift"); - Set optionsSet = new HashSet(Arrays.asList(extraOptions)); + Set optionsSet = new HashSet<>(Arrays.asList(extraOptions)); String grammarName = grammarFileName.substring(0, grammarFileName.lastIndexOf('.')); if (!optionsSet.contains("-no-listener")) { files.add(grammarName + "Listener.swift"); @@ -406,6 +379,5 @@ public class BaseSwiftTest implements RuntimeTestSupport { } } addSourceFiles(files.toArray(new String[files.size()])); - return true; } } From 3095de982efe28ddd2d7dfb1578287bee8103561 Mon Sep 17 00:00:00 2001 From: Hanzhou Shi Date: Fri, 12 May 2017 11:58:29 -0700 Subject: [PATCH 09/23] All tests passing. But slow. --- .../v4/test/runtime/swift/BaseSwiftTest.java | 148 +++++++++--------- 1 file changed, 71 insertions(+), 77 deletions(-) diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java index 78dd08c2a..4fb37ea62 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java @@ -6,6 +6,7 @@ package org.antlr.v4.test.runtime.swift; +import org.antlr.v4.runtime.misc.Pair; import org.antlr.v4.test.runtime.ErrorQueue; import org.antlr.v4.test.runtime.RuntimeTestSupport; import org.antlr.v4.test.runtime.StreamVacuum; @@ -33,17 +34,6 @@ public class BaseSwiftTest implements RuntimeTestSupport { */ private static String ANTLR_RUNTIME_PATH; - private static void exec(String workingDir, String... command) { - ProcessBuilder builder = new ProcessBuilder(command); - builder.directory(new File(workingDir)); - try { - Process p = builder.start(); - p.waitFor(); - } catch (Exception e) { - e.printStackTrace(); - } - } - static { String baseTestDir = System.getProperty("antlr-swift-test-dir"); if (baseTestDir == null || baseTestDir.isEmpty()) { @@ -62,12 +52,12 @@ public class BaseSwiftTest implements RuntimeTestSupport { throw new RuntimeException("Swift runtime file not found at:" + swiftRuntime.getPath()); } ANTLR_RUNTIME_PATH = swiftRuntime.getPath(); - exec(ANTLR_RUNTIME_PATH, "swift", "build"); + fastFailRunProcess(ANTLR_RUNTIME_PATH, "swift", "build"); // shutdown logic Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { - exec(ANTLR_RUNTIME_PATH, "swift", "package", "clean"); + fastFailRunProcess(ANTLR_RUNTIME_PATH, "swift", "package", "clean"); } }); } @@ -148,68 +138,10 @@ public class BaseSwiftTest implements RuntimeTestSupport { writeLexerTestFile(lexerName, showDFA); addSourceFiles("main.swift"); -// compile(); -// String output = execTest(); -// return output; - return ""; // TODO: - } - - private String execTest(String projectDir, String projectName) { - try { - return runProcess(projectDir, "./.build/debug/" + projectName, "input"); - } - catch (Exception e) { - System.err.println("Execution of testcase failed."); - e.printStackTrace(System.err); - } - return null; - } - - private void addSourceFiles(String... files) { - Collections.addAll(this.sourceFiles, files); - } - - private void listSourceFiles() { - for (String sourceFile: this.sourceFiles) { - System.out.println(sourceFile); - } - } - - private static final String EXEC_NAME = "Test"; - - private void buildProject(String projectDir) { - mkdir(projectDir); - exec(projectDir, "swift", "package", "init", "--type", "executable"); - for (String sourceFile: sourceFiles) { - String absPath = getTmpDir() + "/" + sourceFile; - exec(getTmpDir(), "mv", "-f", absPath, projectDir + "/Sources/"); - } - exec(getTmpDir(), "mv", "-f", "input", projectDir); - - try { - String dylibPath = ANTLR_RUNTIME_PATH + "/.build/debug/"; - runProcess(projectDir, "swift", "build", - "-Xswiftc", "-I"+dylibPath, - "-Xlinker", "-L"+dylibPath, - "-Xlinker", "-lAntlr4"); - } catch (IOException | InterruptedException e) { - e.printStackTrace(); - } - } - - private static String runProcess(String execPath, String... args) throws IOException, InterruptedException { - Process process = Runtime.getRuntime().exec(args, null, new File(execPath)); - StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream()); - StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream()); - stdoutVacuum.start(); - stderrVacuum.start(); - process.waitFor(); - stdoutVacuum.join(); - stderrVacuum.join(); - if (stderrVacuum.toString().length() > 0) { - throw new RuntimeException(stderrVacuum.toString()); - } - return stdoutVacuum.toString(); + String projectName = "testcase-" + System.currentTimeMillis(); + String projectDir = getTmpDir() + "/" + projectName; + buildProject(projectDir); + return execTest(projectDir, projectName); } @Override @@ -226,13 +158,75 @@ public class BaseSwiftTest implements RuntimeTestSupport { showDiagnosticErrors,false); } + private String execTest(String projectDir, String projectName) { + try { + Pair output = runProcess(projectDir, "./.build/debug/" + projectName, "input"); + if (output.b.length() > 0) { + stderrDuringParse = output.b; + } + String stdout = output.a; + return stdout.length() > 0 ? stdout : null; + } + catch (Exception e) { + System.err.println("Execution of testcase failed."); + e.printStackTrace(System.err); + } + return null; + } + + private void addSourceFiles(String... files) { + Collections.addAll(this.sourceFiles, files); + } + + private void buildProject(String projectDir) { + mkdir(projectDir); + fastFailRunProcess(projectDir, "swift", "package", "init", "--type", "executable"); + for (String sourceFile: sourceFiles) { + String absPath = getTmpDir() + "/" + sourceFile; + fastFailRunProcess(getTmpDir(), "mv", "-f", absPath, projectDir + "/Sources/"); + } + fastFailRunProcess(getTmpDir(), "mv", "-f", "input", projectDir); + + try { + String dylibPath = ANTLR_RUNTIME_PATH + "/.build/debug/"; + runProcess(projectDir, "swift", "build", + "-Xswiftc", "-I"+dylibPath, + "-Xlinker", "-L"+dylibPath, + "-Xlinker", "-lAntlr4"); + } catch (IOException | InterruptedException e) { + e.printStackTrace(); + } + } + + private static Pair runProcess(String execPath, String... args) throws IOException, InterruptedException { + Process process = Runtime.getRuntime().exec(args, null, new File(execPath)); + StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream()); + StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream()); + stdoutVacuum.start(); + stderrVacuum.start(); + process.waitFor(); + stdoutVacuum.join(); + stderrVacuum.join(); + return new Pair<>(stdoutVacuum.toString(), stderrVacuum.toString()); + } + + private static void fastFailRunProcess(String workingDir, String... command) { + ProcessBuilder builder = new ProcessBuilder(command); + builder.directory(new File(workingDir)); + try { + Process p = builder.start(); + p.waitFor(); + } catch (Exception e) { + e.printStackTrace(); + } + } + private String execParser(String parserName, String lexerName, String parserStartRuleName, boolean debug, boolean profile) { - this.stderrDuringParse = null; if ( parserName==null ) { writeLexerTestFile(lexerName, false); } @@ -356,7 +350,7 @@ public class BaseSwiftTest implements RuntimeTestSupport { String... extraOptions) { ErrorQueue equeue = antlrOnString(getTmpDir(), "Swift", grammarFileName, grammarStr, false, extraOptions); assertTrue(equeue.errors.isEmpty()); - System.out.println(getTmpDir()); +// System.out.println(getTmpDir()); List files = new ArrayList<>(); if (lexerName != null) { From a879297bb5c3a47f390c6798c9e5cdd6f4d971c8 Mon Sep 17 00:00:00 2001 From: Hanzhou Shi Date: Sun, 7 May 2017 20:50:54 -0700 Subject: [PATCH 10/23] swift build & swift test support Linux platform --- .../Swift/Sources/Antlr4/CommonToken.swift | 6 +- .../Sources/Antlr4/DefaultErrorStrategy.swift | 6 +- .../Antlr4/DiagnosticErrorListener.swift | 11 ++- runtime/Swift/Sources/Antlr4/Parser.swift | 6 +- runtime/Swift/Sources/Antlr4/Recognizer.swift | 10 +-- .../Sources/Antlr4/atn/ATNDeserializer.swift | 80 +++++++++---------- .../Sources/Antlr4/atn/LL1Analyzer.swift | 18 +---- .../Antlr4/atn/LexerATNSimulator.swift | 2 +- .../Antlr4/atn/ParserATNSimulator.swift | 2 +- runtime/Swift/Sources/Antlr4/dfa/DFA.swift | 2 +- runtime/Swift/Sources/Antlr4/misc/Utils.swift | 12 ++- .../misc/extension/StringExtension.swift | 31 ++----- .../Antlr4/misc/utils/CommonUtil.swift | 8 +- .../Antlr4/misc/utils/CrossPlatform.swift | 41 ++++++++++ .../pattern/ParseTreePatternMatcher.swift | 2 +- .../TokenStreamRewriterTests.swift | 48 ++++++++++- .../Tests/Antlr4Tests/TokenStreamTests.swift | 4 + .../Tests/Antlr4Tests/VisitorTests.swift | 8 ++ runtime/Swift/Tests/LinuxMain.swift | 13 +++ 19 files changed, 188 insertions(+), 122 deletions(-) create mode 100644 runtime/Swift/Sources/Antlr4/misc/utils/CrossPlatform.swift create mode 100644 runtime/Swift/Tests/LinuxMain.swift diff --git a/runtime/Swift/Sources/Antlr4/CommonToken.swift b/runtime/Swift/Sources/Antlr4/CommonToken.swift index 5c2bff18d..b7cbe3cc6 100644 --- a/runtime/Swift/Sources/Antlr4/CommonToken.swift +++ b/runtime/Swift/Sources/Antlr4/CommonToken.swift @@ -236,9 +236,9 @@ public class CommonToken: WritableToken { } var txt: String if let tokenText = getText() { - txt = tokenText.replaceAll("\n", replacement: "\\n") - txt = txt.replaceAll("\r", replacement: "\\r") - txt = txt.replaceAll("\t", replacement: "\\t") + txt = tokenText.replacingOccurrences(of: "\n", with: "\\n") + txt = txt.replacingOccurrences(of: "\r", with: "\\r") + txt = txt.replacingOccurrences(of: "\t", with: "\\t") } else { txt = "" } diff --git a/runtime/Swift/Sources/Antlr4/DefaultErrorStrategy.swift b/runtime/Swift/Sources/Antlr4/DefaultErrorStrategy.swift index 420f3b39c..a7221b8dc 100644 --- a/runtime/Swift/Sources/Antlr4/DefaultErrorStrategy.swift +++ b/runtime/Swift/Sources/Antlr4/DefaultErrorStrategy.swift @@ -574,9 +574,9 @@ public class DefaultErrorStrategy: ANTLRErrorStrategy { internal func escapeWSAndQuote(_ s: String) -> String { var s = s - s = s.replaceAll("\n", replacement: "\\n") - s = s.replaceAll("\r", replacement: "\\r") - s = s.replaceAll("\t", replacement: "\\t") + s = s.replacingOccurrences(of: "\n", with: "\\n") + s = s.replacingOccurrences(of: "\r", with: "\\r") + s = s.replacingOccurrences(of: "\t", with: "\\t") return "'" + s + "'" } diff --git a/runtime/Swift/Sources/Antlr4/DiagnosticErrorListener.swift b/runtime/Swift/Sources/Antlr4/DiagnosticErrorListener.swift index a6bcf745c..7e05f519a 100644 --- a/runtime/Swift/Sources/Antlr4/DiagnosticErrorListener.swift +++ b/runtime/Swift/Sources/Antlr4/DiagnosticErrorListener.swift @@ -60,8 +60,7 @@ public class DiagnosticErrorListener: BaseErrorListener { let decision: String = getDecisionDescription(recognizer, dfa) let conflictingAlts: BitSet = try getConflictingAlts(ambigAlts, configs) let text: String = try recognizer.getTokenStream()!.getText(Interval.of(startIndex, stopIndex)) - - let message: String = NSString(format: format as NSString, decision, conflictingAlts.description, text) as String + let message = makeString(fromFormat: format, decision, conflictingAlts.description, text) try recognizer.notifyErrorListeners(message) } @@ -75,7 +74,7 @@ public class DiagnosticErrorListener: BaseErrorListener { let format: String = "reportAttemptingFullContext d=%@, input='%@'" let decision: String = getDecisionDescription(recognizer, dfa) let text: String = try recognizer.getTokenStream()!.getText(Interval.of(startIndex, stopIndex)) - let message: String = NSString(format: format as NSString, decision, text) as String + let message = makeString(fromFormat: format, decision, text) try recognizer.notifyErrorListeners(message) } @@ -89,7 +88,7 @@ public class DiagnosticErrorListener: BaseErrorListener { let format: String = "reportContextSensitivity d=%@, input='%@'" let decision: String = getDecisionDescription(recognizer, dfa) let text: String = try recognizer.getTokenStream()!.getText(Interval.of(startIndex, stopIndex)) - let message: String = NSString(format: format as NSString, decision, text) as String + let message = makeString(fromFormat: format, decision, text) try recognizer.notifyErrorListeners(message) } @@ -107,8 +106,7 @@ public class DiagnosticErrorListener: BaseErrorListener { if ruleName.isEmpty { return String(decision) } - - return NSString(format: "%d (%@)", decision, ruleName) as String + return makeString(fromFormat: "%@ (%@)", decision.description, ruleName) } /// Computes the set of conflicting or ambiguous alternatives from a @@ -127,4 +125,5 @@ public class DiagnosticErrorListener: BaseErrorListener { let result = try configs.getAltBitSet() return result } + } diff --git a/runtime/Swift/Sources/Antlr4/Parser.swift b/runtime/Swift/Sources/Antlr4/Parser.swift index cc267a25f..7c5ee0467 100644 --- a/runtime/Swift/Sources/Antlr4/Parser.swift +++ b/runtime/Swift/Sources/Antlr4/Parser.swift @@ -440,7 +440,7 @@ open class Parser: Recognizer { let serializedAtn: String = getSerializedATN() var result: ATN? = bypassAltsAtnCache[serializedAtn] - synced(bypassAltsAtnCache) { + synchronized { [unowned self] in if result == nil { let deserializationOptions: ATNDeserializationOptions = ATNDeserializationOptions() @@ -988,7 +988,7 @@ open class Parser: Recognizer { guard let _interp = _interp else { return s } - synced(_interp.decisionToDFA as AnyObject) { + synchronized { [unowned self] in for d in 0..<_interp.decisionToDFA.count { @@ -1005,7 +1005,7 @@ open class Parser: Recognizer { guard let _interp = _interp else { return } - synced(_interp.decisionToDFA as AnyObject) { + synchronized { [unowned self] in var seenOne: Bool = false diff --git a/runtime/Swift/Sources/Antlr4/Recognizer.swift b/runtime/Swift/Sources/Antlr4/Recognizer.swift index adc7f5457..bd7d5d07f 100644 --- a/runtime/Swift/Sources/Antlr4/Recognizer.swift +++ b/runtime/Swift/Sources/Antlr4/Recognizer.swift @@ -57,7 +57,7 @@ open class Recognizer { public func getTokenTypeMap() -> Dictionary { let vocabulary: Vocabulary = getVocabulary() var result: Dictionary? = self.tokenTypeMapCache[vocabulary] - synced(tokenTypeMapCache) { + synchronized { [unowned self] in if result == nil { result = Dictionary() @@ -96,7 +96,7 @@ open class Recognizer { let ruleNames: [String] = getRuleNames() let result: Dictionary? = self.ruleIndexMapCache[ArrayWrapper(ruleNames)] - synced(ruleIndexMapCache) { + synchronized { [unowned self] in if result == nil { self.ruleIndexMapCache[ArrayWrapper(ruleNames)] = Utils.toMap(ruleNames) @@ -212,9 +212,9 @@ open class Recognizer { s = "<\(t.getType())>" } } - s = s.replaceAll("\n", replacement: "\\n") - s = s.replaceAll("\r", replacement: "\\r") - s = s.replaceAll("\t", replacement: "\\t") + s = s.replacingOccurrences(of: "\n", with: "\\n") + s = s.replacingOccurrences(of: "\r", with: "\\r") + s = s.replacingOccurrences(of: "\t", with: "\\t") return "\(s)" } diff --git a/runtime/Swift/Sources/Antlr4/atn/ATNDeserializer.swift b/runtime/Swift/Sources/Antlr4/atn/ATNDeserializer.swift index 004141f0f..a28219b01 100644 --- a/runtime/Swift/Sources/Antlr4/atn/ATNDeserializer.swift +++ b/runtime/Swift/Sources/Antlr4/atn/ATNDeserializer.swift @@ -554,22 +554,19 @@ public class ATNDeserializer { } public func deserializeFromJson(_ jsonStr: String) -> ATN { - // let jsonStr = Utils.readFile2String(jsonFileName) guard !jsonStr.isEmpty else { fatalError("ATN Serialization is empty,Please include *LexerATN.json and *ParserATN.json in TARGETS-Build Phases-Copy Bundle Resources") } if let JSONData = jsonStr.data(using: String.Encoding.utf8) { do { let JSON = try JSONSerialization.jsonObject(with: JSONData, options: JSONSerialization.ReadingOptions(rawValue: 0)) - guard let JSONDictionary: NSDictionary = JSON as? NSDictionary else { - print("Not a Dictionary") - // put in function + guard let JSONDictionary = JSON as? Dictionary else { fatalError("deserializeFromJson Not a Dictionary") } return try dictToJson(JSONDictionary) - } catch let JSONError as NSError { + } catch let JSONError { print("\(JSONError)") } } @@ -577,10 +574,10 @@ public class ATNDeserializer { fatalError("Could not deserialize ATN ") } - public func dictToJson(_ dict: NSDictionary) throws -> ATN { + public func dictToJson(_ dict: Dictionary) throws -> ATN { - let version: Int = dict.object(forKey: "version") as! Int + let version: Int = dict["version"] as! Int if version != ATNDeserializer.SERIALIZED_VERSION { let reason: String = "Could not deserialize ATN with version \(version) (expected \(ATNDeserializer.SERIALIZED_VERSION))." @@ -588,7 +585,7 @@ public class ATNDeserializer { throw ANTLRError.unsupportedOperation(msg: reason) } - let uuid: UUID = UUID(uuidString: dict.object(forKey: "uuid") as! String)! + let uuid: UUID = UUID(uuidString: dict["uuid"] as! String)! if !ATNDeserializer.SUPPORTED_UUIDS.contains(uuid) { let reason: String = "Could not deserialize ATN with UUID \(uuid) (expected \(ATNDeserializer.SERIALIZED_UUID) or a legacy UUID)." @@ -599,8 +596,8 @@ public class ATNDeserializer { let supportsPrecedencePredicates: Bool = isFeatureSupported(ATNDeserializer.ADDED_PRECEDENCE_TRANSITIONS, uuid) let supportsLexerActions: Bool = isFeatureSupported(ATNDeserializer.ADDED_LEXER_ACTIONS, uuid) - let grammarType: ATNType = ATNType(rawValue: dict.object(forKey: "grammarType") as! Int)! - let maxTokenType: Int = dict.object(forKey: "maxTokenType") as! Int + let grammarType: ATNType = ATNType(rawValue: dict["grammarType"] as! Int)! + let maxTokenType: Int = dict["maxTokenType"] as! Int let atn: ATN = ATN(grammarType, maxTokenType) // @@ -609,22 +606,22 @@ public class ATNDeserializer { var loopBackStateNumbers: Array<(LoopEndState, Int)> = Array<(LoopEndState, Int)>() var endStateNumbers: Array<(BlockStartState, Int)> = Array<(BlockStartState, Int)>() - let states = dict.object(forKey: "states") as! [NSDictionary] - + let states = dict["states"] as! [Dictionary] + for state in states { - let ruleIndex: Int = state.object(forKey: "ruleIndex") as! Int + let ruleIndex: Int = state["ruleIndex"] as! Int - let stype: Int = state.object(forKey: "stateType") as! Int + let stype: Int = state["stateType"] as! Int let s: ATNState = try stateFactory(stype, ruleIndex)! if stype == ATNState.LOOP_END { // special case - let loopBackStateNumber: Int = state.object(forKey: "detailStateNumber") as! Int + let loopBackStateNumber: Int = state["detailStateNumber"] as! Int loopBackStateNumbers.append((s as! LoopEndState, loopBackStateNumber)) } else { if s is BlockStartState { - let endStateNumber: Int = state.object(forKey: "detailStateNumber") as! Int + let endStateNumber: Int = state["detailStateNumber"] as! Int endStateNumbers.append((s as! BlockStartState, endStateNumber)) } } @@ -642,13 +639,13 @@ public class ATNDeserializer { pair.0.endState = atn.states[pair.1] as? BlockEndState } - let numNonGreedyStates = dict.object(forKey: "nonGreedyStates") as! [Int] + let numNonGreedyStates = dict["nonGreedyStates"] as! [Int] for numNonGreedyState in numNonGreedyStates { (atn.states[numNonGreedyState] as! DecisionState).nonGreedy = true } if supportsPrecedencePredicates { - let numPrecedenceStates = dict.object(forKey: "precedenceStates") as! [Int] + let numPrecedenceStates = dict["precedenceStates"] as! [Int] for numPrecedenceState in numPrecedenceStates { (atn.states[numPrecedenceState] as! RuleStartState).isPrecedenceRule = true } @@ -658,7 +655,7 @@ public class ATNDeserializer { // // RULES // - let ruleToStartState = dict.object(forKey: "ruleToStartState") as! [NSDictionary] + let ruleToStartState = dict["ruleToStartState"] as! [Dictionary] let nrules: Int = ruleToStartState.count if atn.grammarType == ATNType.lexer { @@ -668,11 +665,11 @@ public class ATNDeserializer { atn.ruleToStartState = [RuleStartState](repeating: RuleStartState(), count: nrules) // [nrules]; for i in 0.. = Array() - let nsets: Int = dict.object(forKey: "nsets") as! Int - let intervalSet = dict.object(forKey: "IntervalSet") as! [NSDictionary] + let nsets: Int = dict["nsets"] as! Int + let intervalSet = dict["IntervalSet"] as! [Dictionary] for i in 0..] for j in 0..]] for transitionsBuilder in allTransitions { for transition in transitionsBuilder { - let src: Int = transition.object(forKey: "src") as! Int - let trg: Int = transition.object(forKey: "trg") as! Int - let ttype: Int = transition.object(forKey: "edgeType") as! Int - let arg1: Int = transition.object(forKey: "arg1") as! Int - let arg2: Int = transition.object(forKey: "arg2") as! Int - let arg3: Int = transition.object(forKey: "arg3") as! Int + let src: Int = transition["src"] as! Int + let trg: Int = transition["trg"] as! Int + let ttype: Int = transition["edgeType"] as! Int + let arg1: Int = transition["arg1"] as! Int + let arg2: Int = transition["arg2"] as! Int + let arg3: Int = transition["arg3"] as! Int let trans: Transition = try edgeFactory(atn, ttype, src, trg, arg1, arg2, arg3, sets) let srcState: ATNState = atn.states[src]! @@ -830,7 +824,7 @@ public class ATNDeserializer { // // DECISIONS // - let ndecisions: [Int] = dict.object(forKey: "decisionToState") as! [Int] + let ndecisions: [Int] = dict["decisionToState"] as! [Int] let length = ndecisions.count for i in 0..] if supportsLexerActions { atn.lexerActions = [LexerAction](repeating: LexerAction(), count: lexerActionsBuilder.count) //[toInt(data[p++])]; let length = atn.lexerActions.count for i in 0.. [IntervalSet?]? { -// print("LOOK("+s.stateNumber+")"); - + guard let s = s else { return nil } let length = s.getNumberOfTransitions() var look: [IntervalSet?] = [IntervalSet?](repeating: nil, count: length) - //new IntervalSet[s.getNumberOfTransitions()]; for alt in 0.. = Set() @@ -138,10 +136,6 @@ public class LL1Analyzer { lookBusy.insert(c) } -// if ( !lookBusy.insert (c) ) { -// return; -// } - if s == stopState { guard let ctx = ctx else { try look.add(CommonToken.EPSILON) @@ -175,11 +169,8 @@ public class LL1Analyzer { var removed: Bool = try calledRuleStack.get(returnState.ruleIndex!) - //TODO try - //try { try calledRuleStack.clear(returnState.ruleIndex!) try self._LOOK(returnState, stopState, ctx.getParent(i), look, &lookBusy, calledRuleStack, seeThruPreds, addEOF) - //} defer { if removed { try! calledRuleStack.set(returnState.ruleIndex!) @@ -193,18 +184,15 @@ public class LL1Analyzer { var n: Int = s.getNumberOfTransitions() for i in 0.. \(q) upon \(t)") } - synced(p) { + synchronized { if p.edges == nil { // make room for tokens 1..n and -1 masquerading as index 0 //TODO ARRAY COUNT diff --git a/runtime/Swift/Sources/Antlr4/atn/ParserATNSimulator.swift b/runtime/Swift/Sources/Antlr4/atn/ParserATNSimulator.swift index 43c0aa888..4bc319ea0 100644 --- a/runtime/Swift/Sources/Antlr4/atn/ParserATNSimulator.swift +++ b/runtime/Swift/Sources/Antlr4/atn/ParserATNSimulator.swift @@ -1972,7 +1972,7 @@ open class ParserATNSimulator: ATNSimulator { guard let from = from else { return to } - synced(from) { + synchronized { [unowned self] in if from.edges == nil { from.edges = [DFAState?](repeating: nil, count: self.atn.maxTokenType + 1 + 1) //new DFAState[atn.maxTokenType+1+1]; diff --git a/runtime/Swift/Sources/Antlr4/dfa/DFA.swift b/runtime/Swift/Sources/Antlr4/dfa/DFA.swift index afeebf39e..93eca7bc0 100644 --- a/runtime/Swift/Sources/Antlr4/dfa/DFA.swift +++ b/runtime/Swift/Sources/Antlr4/dfa/DFA.swift @@ -102,7 +102,7 @@ public class DFA: CustomStringConvertible { } // synchronization on s0 here is ok. when the DFA is turned into a // precedence DFA, s0 will be initialized once and not updated again - synced(s0) { + synchronized { // s0.edges is never null for a precedence DFA if precedence >= edges.count { let increase = [DFAState?](repeating: nil, count: (precedence + 1 - edges.count)) diff --git a/runtime/Swift/Sources/Antlr4/misc/Utils.swift b/runtime/Swift/Sources/Antlr4/misc/Utils.swift index 6529fe609..d8ad9bfd7 100644 --- a/runtime/Swift/Sources/Antlr4/misc/Utils.swift +++ b/runtime/Swift/Sources/Antlr4/misc/Utils.swift @@ -52,7 +52,7 @@ public class Utils { do { fileContents = try String(contentsOfFile: path, encoding: encoding) - } catch _ as NSError { + } catch { return [Character]() } @@ -68,22 +68,20 @@ public class Utils { var fileContents: String? = nil do { fileContents = try String(contentsOfFile: path!, encoding: encoding) - } catch _ as NSError { + } catch { return "" } - - return fileContents ?? "" } public static func readFile2StringByPath(_ path: String, _ encoding: String.Encoding = String.Encoding.utf8) -> String { - //let path = fileName.stringByExpandingTildeInPath var fileContents: String? = nil + do { - fileContents = try NSString(contentsOfFile: path, encoding: String.Encoding.utf8.rawValue) as String //try String(contentsOfFile: path!, encoding: encoding) - } catch _ as NSError { + fileContents = try String(contentsOfFile: path, encoding: String.Encoding.utf8) + } catch { return "" } diff --git a/runtime/Swift/Sources/Antlr4/misc/extension/StringExtension.swift b/runtime/Swift/Sources/Antlr4/misc/extension/StringExtension.swift index 46788278a..a5f8ebea3 100644 --- a/runtime/Swift/Sources/Antlr4/misc/extension/StringExtension.swift +++ b/runtime/Swift/Sources/Antlr4/misc/extension/StringExtension.swift @@ -2,36 +2,17 @@ /// Use of this file is governed by the BSD 3-clause license that /// can be found in the LICENSE.txt file in the project root. -//import Cocoa - -#if os(OSX) - -import Cocoa - -#elseif os(iOS) - -import UIKit - -#endif +import Foundation //http://stackoverflow.com/questions/28182441/swift-how-to-get-substring-from-start-to-last-index-of-character //https://github.com/williamFalcon/Bolt_Swift/blob/master/Bolt/BoltLibrary/String/String.swift extension String { - func trim() -> String { - return self.trimmingCharacters(in: CharacterSet.whitespaces) - } - func split(_ separator: String) -> [String] { return self.components(separatedBy: separator) } - func replaceAll(_ from: String, replacement: String) -> String { - - return self.replacingOccurrences(of: from, with: replacement, options: NSString.CompareOptions.literal, range: nil) - } - func containsIgnoreCase(_ find: String) -> Bool { return self.lowercased().range(of: find.lowercased()) != nil } @@ -53,7 +34,7 @@ extension String { func indexOf(_ target: String, startIndex: Int) -> Int { let startRange = self.characters.index(self.startIndex, offsetBy: startIndex) - let range = self.range(of: target, options: NSString.CompareOptions.literal, range: startRange..", - "<": "<"] + "<": "<" + ] public var escapedHtmlString: String { var newString = "\(self)" for (key, value) in String.htmlEscapedDictionary { - newString = newString.replaceAll(value, replacement: key) + newString = newString.replacingOccurrences(of: value, with: key) } return newString } diff --git a/runtime/Swift/Sources/Antlr4/misc/utils/CommonUtil.swift b/runtime/Swift/Sources/Antlr4/misc/utils/CommonUtil.swift index c59be5d92..f1126c589 100644 --- a/runtime/Swift/Sources/Antlr4/misc/utils/CommonUtil.swift +++ b/runtime/Swift/Sources/Antlr4/misc/utils/CommonUtil.swift @@ -13,7 +13,7 @@ import Foundation func errPrint(_ msg: String) { - fputs(msg + "\n", __stderrp) + fputs(msg + "\n", stderr) } public func +(lhs: String, rhs: Int) -> String { @@ -57,12 +57,6 @@ func >>>(lhs: Int, rhs: Int) -> Int { return Int(bitPattern: left >> right) } -func synced(_ lock: AnyObject, closure: () -> ()) { - objc_sync_enter(lock) - closure() - objc_sync_exit(lock) -} - func intChar2String(_ i: Int) -> String { return String(Character(integerLiteral: i)) } diff --git a/runtime/Swift/Sources/Antlr4/misc/utils/CrossPlatform.swift b/runtime/Swift/Sources/Antlr4/misc/utils/CrossPlatform.swift new file mode 100644 index 000000000..294376869 --- /dev/null +++ b/runtime/Swift/Sources/Antlr4/misc/utils/CrossPlatform.swift @@ -0,0 +1,41 @@ +import Foundation + +// MARK: - String + +/// Cross-platform String formatting method. +/// We make this method because on Linux String is not conforming to +/// CVarArg, thus cannot be mapped to "%@" in format string. +/// This method implements a work-around that maps "%@" to "%s" in +/// our format string and then convert NSString to CString. +/// +/// - Parameters: +/// - format: printf-like format string +/// - args: argument strings +/// - Returns: formatted string +func makeString(fromFormat format: String, _ args: String...) -> String { + #if os(Linux) + let linuxFormat = format.replacingOccurrences(of: "%@", with: "%s") + let cStrings = args.map { $0.withCString { $0 } } + return String(format: linuxFormat, arguments: cStrings) + #else + return String(format: format, args) + #endif +} + + +// MARK: - Multithread + +fileprivate var _GLOBAL_MUTEX = pthread_mutex_t() + +/// Cross-platform synchronized execution of a closure. +/// Using naive locking that uses a global mutex lock. +/// +/// - Parameter closure: closure needs to be executed. +func synchronized(closure: () -> R) { + pthread_mutex_lock(&_GLOBAL_MUTEX) + defer { + pthread_mutex_unlock(&_GLOBAL_MUTEX) + } + _ = closure() +} + diff --git a/runtime/Swift/Sources/Antlr4/tree/pattern/ParseTreePatternMatcher.swift b/runtime/Swift/Sources/Antlr4/tree/pattern/ParseTreePatternMatcher.swift index 978bc020f..cb42b5c52 100644 --- a/runtime/Swift/Sources/Antlr4/tree/pattern/ParseTreePatternMatcher.swift +++ b/runtime/Swift/Sources/Antlr4/tree/pattern/ParseTreePatternMatcher.swift @@ -482,7 +482,7 @@ public class ParseTreePatternMatcher { let c: Chunk = chunks[i] if c is TextChunk { let tc: TextChunk = c as! TextChunk - let unescaped: String = tc.getText().replaceAll(escape, replacement: "") + let unescaped = tc.getText().replacingOccurrences(of: escape, with: "") if unescaped.length < tc.getText().length { chunks[i] = TextChunk(unescaped) } diff --git a/runtime/Swift/Tests/Antlr4Tests/TokenStreamRewriterTests.swift b/runtime/Swift/Tests/Antlr4Tests/TokenStreamRewriterTests.swift index 568de8c98..d309e523f 100644 --- a/runtime/Swift/Tests/Antlr4Tests/TokenStreamRewriterTests.swift +++ b/runtime/Swift/Tests/Antlr4Tests/TokenStreamRewriterTests.swift @@ -6,7 +6,53 @@ import XCTest import Antlr4 class TokenStreamRewriterTests: XCTestCase { - + + static let allTests = [ + ("testPreservesOrderOfContiguousInserts", testPreservesOrderOfContiguousInserts), + ("testDistinguishBetweenInsertAfterAndInsertBeforeToPreserverOrder2", testDistinguishBetweenInsertAfterAndInsertBeforeToPreserverOrder2), + ("testDistinguishBetweenInsertAfterAndInsertBeforeToPreserverOrder", testDistinguishBetweenInsertAfterAndInsertBeforeToPreserverOrder), + ("testInsertBeforeTokenThenDeleteThatToken", testInsertBeforeTokenThenDeleteThatToken), + ("testLeaveAloneDisjointInsert2", testLeaveAloneDisjointInsert2), + ("testLeaveAloneDisjointInsert", testLeaveAloneDisjointInsert), + ("testDropPrevCoveredInsert", testDropPrevCoveredInsert), + ("testDropIdenticalReplace", testDropIdenticalReplace), + ("testOverlappingReplace4", testOverlappingReplace4), + ("testOverlappingReplace3", testOverlappingReplace3), + ("testOverlappingReplace2", testOverlappingReplace2), + ("testOverlappingReplace", testOverlappingReplace), + ("testDisjointInserts", testDisjointInserts), + ("testCombineInsertOnLeftWithDelete", testCombineInsertOnLeftWithDelete), + ("testCombineInsertOnLeftWithReplace", testCombineInsertOnLeftWithReplace), + ("testCombine3Inserts", testCombine3Inserts), + ("testCombineInserts", testCombineInserts), + ("testReplaceSingleMiddleThenOverlappingSuperset", testReplaceSingleMiddleThenOverlappingSuperset), + ("testReplaceThenReplaceLowerIndexedSuperset", testReplaceThenReplaceLowerIndexedSuperset), + ("testReplaceThenReplaceSuperset", testReplaceThenReplaceSuperset), + ("testReplaceSubsetThenFetch", testReplaceSubsetThenFetch), + ("testReplaceAll", testReplaceAll), + ("testReplaceRangeThenInsertAfterRightEdge", testReplaceRangeThenInsertAfterRightEdge), + ("testReplaceRangeThenInsertAtRightEdge", testReplaceRangeThenInsertAtRightEdge), + ("testReplaceThenInsertAtLeftEdge", testReplaceThenInsertAtLeftEdge), + ("testReplaceThenInsertAfterLastIndex", testReplaceThenInsertAfterLastIndex), + ("testInsertThenReplaceLastIndex", testInsertThenReplaceLastIndex), + ("testReplaceThenInsertBeforeLastIndex", testReplaceThenInsertBeforeLastIndex), + ("test2InsertThenReplaceIndex0", test2InsertThenReplaceIndex0), + ("test2InsertMiddleIndex", test2InsertMiddleIndex), + ("testInsertThenReplaceSameIndex", testInsertThenReplaceSameIndex), + ("testInsertInPriorReplace", testInsertInPriorReplace), + ("testReplaceThenDeleteMiddleIndex", testReplaceThenDeleteMiddleIndex), + ("test2ReplaceMiddleIndex1InsertBefore", test2ReplaceMiddleIndex1InsertBefore), + ("test2ReplaceMiddleIndex", test2ReplaceMiddleIndex), + ("testToStringStartStop2", testToStringStartStop2), + ("testToStringStartStop", testToStringStartStop), + ("testReplaceMiddleIndex", testReplaceMiddleIndex), + ("testReplaceLastIndex", testReplaceLastIndex), + ("testReplaceIndex0", testReplaceIndex0), + ("test2InsertBeforeAfterMiddleIndex", test2InsertBeforeAfterMiddleIndex), + ("testInsertAfterLastIndex", testInsertAfterLastIndex), + ("testInsertBeforeIndex0", testInsertBeforeIndex0) + ] + func testInsertBeforeIndex0() throws { let input = ANTLRInputStream("abc") let lexer = LexerA(input) diff --git a/runtime/Swift/Tests/Antlr4Tests/TokenStreamTests.swift b/runtime/Swift/Tests/Antlr4Tests/TokenStreamTests.swift index 68f966b78..c87501e66 100644 --- a/runtime/Swift/Tests/Antlr4Tests/TokenStreamTests.swift +++ b/runtime/Swift/Tests/Antlr4Tests/TokenStreamTests.swift @@ -6,6 +6,10 @@ import XCTest import Antlr4 class TokenStreamTests: XCTestCase { + + static let allTests = [ + ("testBufferedTokenStreamClearFetchEOFWithNewSource", testBufferedTokenStreamClearFetchEOFWithNewSource) + ] /// Test fetchEOF reset after setTokenSource func testBufferedTokenStreamClearFetchEOFWithNewSource() throws { diff --git a/runtime/Swift/Tests/Antlr4Tests/VisitorTests.swift b/runtime/Swift/Tests/Antlr4Tests/VisitorTests.swift index 33c58d5c4..e22488797 100644 --- a/runtime/Swift/Tests/Antlr4Tests/VisitorTests.swift +++ b/runtime/Swift/Tests/Antlr4Tests/VisitorTests.swift @@ -6,6 +6,14 @@ import XCTest import Antlr4 class VisitorTests: XCTestCase { + static let allTests = [ + ("testCalculatorVisitor", testCalculatorVisitor), + ("testShouldNotVisitTerminal", testShouldNotVisitTerminal), + ("testShouldNotVisitEOF", testShouldNotVisitEOF), + ("testVisitErrorNode", testVisitErrorNode), + ("testVisitTerminalNode", testVisitTerminalNode) + ] + /// /// This test verifies the basic behavior of visitors, with an emphasis on /// {@link AbstractParseTreeVisitor#visitTerminal}. diff --git a/runtime/Swift/Tests/LinuxMain.swift b/runtime/Swift/Tests/LinuxMain.swift new file mode 100644 index 000000000..92fa3a52f --- /dev/null +++ b/runtime/Swift/Tests/LinuxMain.swift @@ -0,0 +1,13 @@ +#if os(Linux) + +import XCTest +@testable import Antlr4Tests + +XCTMain([ + // Antlr4Tests + testCase(TokenStreamTests.allTests), + testCase(TokenStreamRewriterTests.allTests), + testCase(VisitorTests.allTests) +]) + +#endif From a64ec12e739cec51c200e5578bc95000e2345756 Mon Sep 17 00:00:00 2001 From: Hanzhou Shi Date: Sun, 7 May 2017 21:25:16 -0700 Subject: [PATCH 11/23] fixing tests, and some cleanups. --- .../Antlr4/DiagnosticErrorListener.swift | 20 +++++++++---------- .../Antlr4/misc/utils/CrossPlatform.swift | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/runtime/Swift/Sources/Antlr4/DiagnosticErrorListener.swift b/runtime/Swift/Sources/Antlr4/DiagnosticErrorListener.swift index 7e05f519a..21e8da8c5 100644 --- a/runtime/Swift/Sources/Antlr4/DiagnosticErrorListener.swift +++ b/runtime/Swift/Sources/Antlr4/DiagnosticErrorListener.swift @@ -56,10 +56,10 @@ public class DiagnosticErrorListener: BaseErrorListener { return } - let format: String = "reportAmbiguity d=%@: ambigAlts=%@, input='%@'" - let decision: String = getDecisionDescription(recognizer, dfa) - let conflictingAlts: BitSet = try getConflictingAlts(ambigAlts, configs) - let text: String = try recognizer.getTokenStream()!.getText(Interval.of(startIndex, stopIndex)) + let format = "reportAmbiguity d=%@: ambigAlts=%@, input='%@'" + let decision = getDecisionDescription(recognizer, dfa) + let conflictingAlts = try getConflictingAlts(ambigAlts, configs) + let text = try recognizer.getTokenStream()!.getText(Interval.of(startIndex, stopIndex)) let message = makeString(fromFormat: format, decision, conflictingAlts.description, text) try recognizer.notifyErrorListeners(message) } @@ -71,9 +71,9 @@ public class DiagnosticErrorListener: BaseErrorListener { _ stopIndex: Int, _ conflictingAlts: BitSet?, _ configs: ATNConfigSet) throws { - let format: String = "reportAttemptingFullContext d=%@, input='%@'" - let decision: String = getDecisionDescription(recognizer, dfa) - let text: String = try recognizer.getTokenStream()!.getText(Interval.of(startIndex, stopIndex)) + let format = "reportAttemptingFullContext d=%@, input='%@'" + let decision = getDecisionDescription(recognizer, dfa) + let text = try recognizer.getTokenStream()!.getText(Interval.of(startIndex, stopIndex)) let message = makeString(fromFormat: format, decision, text) try recognizer.notifyErrorListeners(message) } @@ -85,9 +85,9 @@ public class DiagnosticErrorListener: BaseErrorListener { _ stopIndex: Int, _ prediction: Int, _ configs: ATNConfigSet) throws { - let format: String = "reportContextSensitivity d=%@, input='%@'" - let decision: String = getDecisionDescription(recognizer, dfa) - let text: String = try recognizer.getTokenStream()!.getText(Interval.of(startIndex, stopIndex)) + let format = "reportContextSensitivity d=%@, input='%@'" + let decision = getDecisionDescription(recognizer, dfa) + let text = try recognizer.getTokenStream()!.getText(Interval.of(startIndex, stopIndex)) let message = makeString(fromFormat: format, decision, text) try recognizer.notifyErrorListeners(message) } diff --git a/runtime/Swift/Sources/Antlr4/misc/utils/CrossPlatform.swift b/runtime/Swift/Sources/Antlr4/misc/utils/CrossPlatform.swift index 294376869..9c7473b0a 100644 --- a/runtime/Swift/Sources/Antlr4/misc/utils/CrossPlatform.swift +++ b/runtime/Swift/Sources/Antlr4/misc/utils/CrossPlatform.swift @@ -18,7 +18,7 @@ func makeString(fromFormat format: String, _ args: String...) -> String { let cStrings = args.map { $0.withCString { $0 } } return String(format: linuxFormat, arguments: cStrings) #else - return String(format: format, args) + return String(format: format, arguments: args) #endif } From 10a3ee3f49bce0563475cfb7aaea981eaf61b49c Mon Sep 17 00:00:00 2001 From: Hanzhou Shi Date: Fri, 12 May 2017 21:43:52 -0700 Subject: [PATCH 12/23] adding travis configuration for swift under linux --- .travis.yml | 5 +++++ .travis/before-install-linux-swift.sh | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100755 .travis/before-install-linux-swift.sh diff --git a/.travis.yml b/.travis.yml index 25dc7ece7..850e979fd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,6 +36,11 @@ matrix: compiler: clang osx_image: xcode8.1 env: TARGET=swift + - os: linux + compiler: clang + env: + - TARGET=swift + - SWIFT_VERSION=swift-3.1.1 - os: osx osx_image: xcode8.2 env: TARGET=dotnet diff --git a/.travis/before-install-linux-swift.sh b/.travis/before-install-linux-swift.sh new file mode 100755 index 000000000..b5566b65b --- /dev/null +++ b/.travis/before-install-linux-swift.sh @@ -0,0 +1,19 @@ +set -euo pipefail + +# download swift +mkdir swift +curl https://swift.org/builds/$SWIFT_VERSION-release/ubuntu1404/$SWIFT_VERSION-RELEASE/$SWIFT_VERSION-RELEASE-ubuntu14.04.tar.gz -s | tar xz -C swift &> /dev/null + +# install dependencies +sudo apt-get install clang libicu52 + +# update libstdc++6, travis uses some old version +echo "\n" | sudo add-apt-repository ppa:ubuntu-toolchain-r/test +sudo apt-get update +sudo apt-get install gcc-4.9 +sudo apt-get install libstdc++6 + +# check swift +export PATH=$(pwd)/swift/$SWIFT_VERSION-RELEASE-ubuntu14.04/usr/bin:$PATH +swift --version +swift build --version From e8962dad7e0c1db2a58a1e565518a4b7fd33d6c5 Mon Sep 17 00:00:00 2001 From: Hanzhou Shi Date: Sat, 13 May 2017 14:55:03 -0700 Subject: [PATCH 13/23] reads SWIFT_HOME from environment variables. --- .travis/before-install-linux-swift.sh | 16 +++++---- .../v4/test/runtime/swift/BaseSwiftTest.java | 35 +++++++++++-------- 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/.travis/before-install-linux-swift.sh b/.travis/before-install-linux-swift.sh index b5566b65b..0c1425919 100755 --- a/.travis/before-install-linux-swift.sh +++ b/.travis/before-install-linux-swift.sh @@ -4,16 +4,18 @@ set -euo pipefail mkdir swift curl https://swift.org/builds/$SWIFT_VERSION-release/ubuntu1404/$SWIFT_VERSION-RELEASE/$SWIFT_VERSION-RELEASE-ubuntu14.04.tar.gz -s | tar xz -C swift &> /dev/null -# install dependencies -sudo apt-get install clang libicu52 +# make sure we use trusty repositories (travis by default uses precise) +curl https://repogen.simplylinux.ch/txt/trusty/sources_c4aa56bd26c0f54f391d8fae3e687ef5f6e97c26.txt | sudo tee /etc/apt/sources.list -# update libstdc++6, travis uses some old version -echo "\n" | sudo add-apt-repository ppa:ubuntu-toolchain-r/test +# install dependencies +# some packages below will be update, swift assumes newer versions +# of, for example, sqlite3 and libicu, without the update some +# tools will not work sudo apt-get update -sudo apt-get install gcc-4.9 -sudo apt-get install libstdc++6 +sudo apt-get install clang libicu-dev libxml2 sqlite3 # check swift -export PATH=$(pwd)/swift/$SWIFT_VERSION-RELEASE-ubuntu14.04/usr/bin:$PATH +export SWIFT_HOME=$(pwd)/swift/$SWIFT_VERSION-RELEASE-ubuntu14.04/usr/bin/ +export PATH=${SWIFT_HOME):$PATH swift --version swift build --version diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java index 4fb37ea62..bbf363d51 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java @@ -20,6 +20,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import static org.antlr.v4.test.runtime.BaseRuntimeTest.antlrOnString; @@ -34,30 +35,34 @@ public class BaseSwiftTest implements RuntimeTestSupport { */ private static String ANTLR_RUNTIME_PATH; + /** + * Absolute path to swift command. + */ + private static String SWIFT_CMD; + + /** + * Environment variable name for swift home. + */ + private static final String SWIFT_HOME_ENV_KEY = "SWIFT_HOME"; + static { - String baseTestDir = System.getProperty("antlr-swift-test-dir"); - if (baseTestDir == null || baseTestDir.isEmpty()) { - baseTestDir = System.getProperty("java.io.tmpdir"); - } - - if (!new File(baseTestDir).isDirectory()) { - throw new UnsupportedOperationException("The specified base test directory does not exist: " + baseTestDir); - } - - final ClassLoader loader = Thread.currentThread().getContextClassLoader(); + Map env = System.getenv(); + String swiftHome = env.containsKey(SWIFT_HOME_ENV_KEY) ? env.get(SWIFT_HOME_ENV_KEY) : ""; + SWIFT_CMD = swiftHome + "swift"; + ClassLoader loader = Thread.currentThread().getContextClassLoader(); // build swift runtime - final URL swiftRuntime = loader.getResource("Swift"); + URL swiftRuntime = loader.getResource("Swift"); if (swiftRuntime == null) { throw new RuntimeException("Swift runtime file not found at:" + swiftRuntime.getPath()); } ANTLR_RUNTIME_PATH = swiftRuntime.getPath(); - fastFailRunProcess(ANTLR_RUNTIME_PATH, "swift", "build"); + fastFailRunProcess(ANTLR_RUNTIME_PATH, SWIFT_CMD, "build"); // shutdown logic Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { - fastFailRunProcess(ANTLR_RUNTIME_PATH, "swift", "package", "clean"); + fastFailRunProcess(ANTLR_RUNTIME_PATH, SWIFT_CMD, "package", "clean"); } }); } @@ -180,7 +185,7 @@ public class BaseSwiftTest implements RuntimeTestSupport { private void buildProject(String projectDir) { mkdir(projectDir); - fastFailRunProcess(projectDir, "swift", "package", "init", "--type", "executable"); + fastFailRunProcess(projectDir, SWIFT_CMD, "package", "init", "--type", "executable"); for (String sourceFile: sourceFiles) { String absPath = getTmpDir() + "/" + sourceFile; fastFailRunProcess(getTmpDir(), "mv", "-f", absPath, projectDir + "/Sources/"); @@ -189,7 +194,7 @@ public class BaseSwiftTest implements RuntimeTestSupport { try { String dylibPath = ANTLR_RUNTIME_PATH + "/.build/debug/"; - runProcess(projectDir, "swift", "build", + runProcess(projectDir, SWIFT_CMD, "build", "-Xswiftc", "-I"+dylibPath, "-Xlinker", "-L"+dylibPath, "-Xlinker", "-lAntlr4"); From 77eddc8e764a40b8da4dde0b82beab649c7e4e57 Mon Sep 17 00:00:00 2001 From: Hanzhou Shi Date: Sat, 13 May 2017 17:00:27 -0700 Subject: [PATCH 14/23] fix linker issue on travis ubuntu --- .travis.yml | 4 +--- .travis/before-install-linux-swift.sh | 12 +++--------- .travis/run-tests-swift.sh | 18 +++++++++++++++++- .../v4/test/runtime/swift/BaseSwiftTest.java | 13 +++++++++---- 4 files changed, 30 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index 850e979fd..6e6f65cce 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,9 +38,7 @@ matrix: env: TARGET=swift - os: linux compiler: clang - env: - - TARGET=swift - - SWIFT_VERSION=swift-3.1.1 + env: TARGET=swift - os: osx osx_image: xcode8.2 env: TARGET=dotnet diff --git a/.travis/before-install-linux-swift.sh b/.travis/before-install-linux-swift.sh index 0c1425919..607f04449 100755 --- a/.travis/before-install-linux-swift.sh +++ b/.travis/before-install-linux-swift.sh @@ -1,9 +1,5 @@ set -euo pipefail -# download swift -mkdir swift -curl https://swift.org/builds/$SWIFT_VERSION-release/ubuntu1404/$SWIFT_VERSION-RELEASE/$SWIFT_VERSION-RELEASE-ubuntu14.04.tar.gz -s | tar xz -C swift &> /dev/null - # make sure we use trusty repositories (travis by default uses precise) curl https://repogen.simplylinux.ch/txt/trusty/sources_c4aa56bd26c0f54f391d8fae3e687ef5f6e97c26.txt | sudo tee /etc/apt/sources.list @@ -14,8 +10,6 @@ curl https://repogen.simplylinux.ch/txt/trusty/sources_c4aa56bd26c0f54f391d8fae3 sudo apt-get update sudo apt-get install clang libicu-dev libxml2 sqlite3 -# check swift -export SWIFT_HOME=$(pwd)/swift/$SWIFT_VERSION-RELEASE-ubuntu14.04/usr/bin/ -export PATH=${SWIFT_HOME):$PATH -swift --version -swift build --version +# This would fix a know linker issue mentioned in: +# https://bugs.swift.org/browse/SR-2299 +sudo ln -sf ld.gold /usr/bin/ld diff --git a/.travis/run-tests-swift.sh b/.travis/run-tests-swift.sh index 78c38ba7f..f0ffae029 100755 --- a/.travis/run-tests-swift.sh +++ b/.travis/run-tests-swift.sh @@ -1,4 +1,20 @@ #!/bin/bash -# only test swift as we develop on os x so likely well tested and its dog slow on travis +# linux specific setup, those setup have to be +# here since environment variables doesn't pass +# across scripts +if [ $TRAVIS_OS_NAME == "linux" ]; then + export SWIFT_VERSION=swift-3.1.1 + export SWIFT_HOME=$(pwd)/swift/$SWIFT_VERSION-RELEASE-ubuntu14.04/usr/bin/ + export PATH=$SWIFT_HOME:$PATH + + # download swift + mkdir swift + curl https://swift.org/builds/$SWIFT_VERSION-release/ubuntu1404/$SWIFT_VERSION-RELEASE/$SWIFT_VERSION-RELEASE-ubuntu14.04.tar.gz -s | tar xz -C swift &> /dev/null +fi + +# check swift +swift --version +swift build --version + mvn -q -Dtest=swift.* test diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java index bbf363d51..f6b890931 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java @@ -194,10 +194,15 @@ public class BaseSwiftTest implements RuntimeTestSupport { try { String dylibPath = ANTLR_RUNTIME_PATH + "/.build/debug/"; - runProcess(projectDir, SWIFT_CMD, "build", + Pair buildResult = runProcess(projectDir, SWIFT_CMD, "build", "-Xswiftc", "-I"+dylibPath, "-Xlinker", "-L"+dylibPath, - "-Xlinker", "-lAntlr4"); + "-Xlinker", "-lAntlr4", + "-Xlinker", "-rpath", + "-Xlinker", dylibPath); + if (buildResult.b.length() > 0) { + throw new RuntimeException("unit test build failed: " + buildResult.b); + } } catch (IOException | InterruptedException e) { e.printStackTrace(); } @@ -259,7 +264,7 @@ public class BaseSwiftTest implements RuntimeTestSupport { ST outputFileST = new ST( "import Antlr4\n" + "import Foundation\n" + - "setbuf(__stdoutp, nil)\n" + + "setbuf(stdout, nil)\n" + "class TreeShapeListener: ParseTreeListener{\n" + " func visitTerminal(_ node: TerminalNode){ }\n" + " func visitErrorNode(_ node: ErrorNode){ }\n" + @@ -320,7 +325,7 @@ public class BaseSwiftTest implements RuntimeTestSupport { "import Antlr4\n" + "import Foundation\n" + - "setbuf(__stdoutp, nil)\n" + + "setbuf(stdout, nil)\n" + "let args = CommandLine.arguments\n" + "let input = ANTLRFileStream(args[1])\n" + "let lex = (input)\n" + From 13769268c96a47e2ca0425cbe5611f1102f65772 Mon Sep 17 00:00:00 2001 From: Hanzhou Shi Date: Sat, 13 May 2017 21:34:11 -0700 Subject: [PATCH 15/23] fixing string issue. --- .../Antlr4/DiagnosticErrorListener.swift | 11 ++++----- .../Antlr4/misc/utils/CrossPlatform.swift | 23 ------------------- 2 files changed, 4 insertions(+), 30 deletions(-) diff --git a/runtime/Swift/Sources/Antlr4/DiagnosticErrorListener.swift b/runtime/Swift/Sources/Antlr4/DiagnosticErrorListener.swift index 21e8da8c5..75681f617 100644 --- a/runtime/Swift/Sources/Antlr4/DiagnosticErrorListener.swift +++ b/runtime/Swift/Sources/Antlr4/DiagnosticErrorListener.swift @@ -56,11 +56,10 @@ public class DiagnosticErrorListener: BaseErrorListener { return } - let format = "reportAmbiguity d=%@: ambigAlts=%@, input='%@'" let decision = getDecisionDescription(recognizer, dfa) let conflictingAlts = try getConflictingAlts(ambigAlts, configs) let text = try recognizer.getTokenStream()!.getText(Interval.of(startIndex, stopIndex)) - let message = makeString(fromFormat: format, decision, conflictingAlts.description, text) + let message = "reportAmbiguity d=\(decision): ambigAlts=\(conflictingAlts), input='\(text)'" try recognizer.notifyErrorListeners(message) } @@ -71,10 +70,9 @@ public class DiagnosticErrorListener: BaseErrorListener { _ stopIndex: Int, _ conflictingAlts: BitSet?, _ configs: ATNConfigSet) throws { - let format = "reportAttemptingFullContext d=%@, input='%@'" let decision = getDecisionDescription(recognizer, dfa) let text = try recognizer.getTokenStream()!.getText(Interval.of(startIndex, stopIndex)) - let message = makeString(fromFormat: format, decision, text) + let message = "reportAttemptingFullContext d=\(decision), input='\(text)'" try recognizer.notifyErrorListeners(message) } @@ -85,10 +83,9 @@ public class DiagnosticErrorListener: BaseErrorListener { _ stopIndex: Int, _ prediction: Int, _ configs: ATNConfigSet) throws { - let format = "reportContextSensitivity d=%@, input='%@'" let decision = getDecisionDescription(recognizer, dfa) let text = try recognizer.getTokenStream()!.getText(Interval.of(startIndex, stopIndex)) - let message = makeString(fromFormat: format, decision, text) + let message = "reportContextSensitivity d=\(decision), input='\(text)'" try recognizer.notifyErrorListeners(message) } @@ -106,7 +103,7 @@ public class DiagnosticErrorListener: BaseErrorListener { if ruleName.isEmpty { return String(decision) } - return makeString(fromFormat: "%@ (%@)", decision.description, ruleName) + return "\(decision) (\(ruleName))" } /// Computes the set of conflicting or ambiguous alternatives from a diff --git a/runtime/Swift/Sources/Antlr4/misc/utils/CrossPlatform.swift b/runtime/Swift/Sources/Antlr4/misc/utils/CrossPlatform.swift index 9c7473b0a..b60a38346 100644 --- a/runtime/Swift/Sources/Antlr4/misc/utils/CrossPlatform.swift +++ b/runtime/Swift/Sources/Antlr4/misc/utils/CrossPlatform.swift @@ -1,28 +1,5 @@ import Foundation -// MARK: - String - -/// Cross-platform String formatting method. -/// We make this method because on Linux String is not conforming to -/// CVarArg, thus cannot be mapped to "%@" in format string. -/// This method implements a work-around that maps "%@" to "%s" in -/// our format string and then convert NSString to CString. -/// -/// - Parameters: -/// - format: printf-like format string -/// - args: argument strings -/// - Returns: formatted string -func makeString(fromFormat format: String, _ args: String...) -> String { - #if os(Linux) - let linuxFormat = format.replacingOccurrences(of: "%@", with: "%s") - let cStrings = args.map { $0.withCString { $0 } } - return String(format: linuxFormat, arguments: cStrings) - #else - return String(format: format, arguments: args) - #endif -} - - // MARK: - Multithread fileprivate var _GLOBAL_MUTEX = pthread_mutex_t() From f15d9a31a338846bc14f419ce9ae6c82cae9763e Mon Sep 17 00:00:00 2001 From: Hanzhou Shi Date: Sat, 13 May 2017 22:55:46 -0700 Subject: [PATCH 16/23] revert one trivial change in go test. --- .../test/org/antlr/v4/test/runtime/go/BaseGoTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime-testsuite/test/org/antlr/v4/test/runtime/go/BaseGoTest.java b/runtime-testsuite/test/org/antlr/v4/test/runtime/go/BaseGoTest.java index 03c41b2f9..48791461f 100644 --- a/runtime-testsuite/test/org/antlr/v4/test/runtime/go/BaseGoTest.java +++ b/runtime-testsuite/test/org/antlr/v4/test/runtime/go/BaseGoTest.java @@ -326,7 +326,7 @@ public class BaseGoTest implements RuntimeTestSupport { // writeFile(overall_tmpdir, "input", input); // rawBuildRecognizerTestFile(parserName, lexerName, listenerName, // visitorName, startRuleName, debug); -// return execParser(); +// return execRecognizer(); // } @Override From 53b83d7184f9e1d5cf12c518405e88e7ca70d2b3 Mon Sep 17 00:00:00 2001 From: Hanzhou Shi Date: Wed, 17 May 2017 22:52:05 -0700 Subject: [PATCH 17/23] Refactored synchronization by using mutex. --- runtime/Swift/Sources/Antlr4/Parser.swift | 22 ++++---- runtime/Swift/Sources/Antlr4/Recognizer.swift | 13 +++-- .../Antlr4/atn/LexerATNSimulator.swift | 34 +++++++----- .../Antlr4/atn/ParserATNSimulator.swift | 54 ++++++++++--------- runtime/Swift/Sources/Antlr4/dfa/DFA.swift | 7 +-- .../Antlr4/misc/utils/CrossPlatform.swift | 18 ------- .../Sources/Antlr4/misc/utils/Mutex.swift | 30 +++++++++++ 7 files changed, 102 insertions(+), 76 deletions(-) delete mode 100644 runtime/Swift/Sources/Antlr4/misc/utils/CrossPlatform.swift create mode 100644 runtime/Swift/Sources/Antlr4/misc/utils/Mutex.swift diff --git a/runtime/Swift/Sources/Antlr4/Parser.swift b/runtime/Swift/Sources/Antlr4/Parser.swift index 7c5ee0467..783562b37 100644 --- a/runtime/Swift/Sources/Antlr4/Parser.swift +++ b/runtime/Swift/Sources/Antlr4/Parser.swift @@ -62,6 +62,12 @@ open class Parser: Recognizer { // TODO: Print exit info. } } + + /// mutex for bypassAltsAtnCache updates + private var bypassAltsAtnCacheMutex = Mutex() + + /// mutex for decisionToDFA updates + private var decisionToDFAMutex = Mutex() /** * This field maps from the serialized ATN string to the deserialized {@link org.antlr.v4.runtime.atn.ATN} with @@ -71,6 +77,7 @@ open class Parser: Recognizer { */ private let bypassAltsAtnCache: HashMap = HashMap() + /** * The error handling strategy for the parser. The default value is a new * instance of {@link org.antlr.v4.runtime.DefaultErrorStrategy}. @@ -110,7 +117,6 @@ open class Parser: Recognizer { */ internal var _buildParseTrees: Bool = true - /** * When {@link #setTrace}{@code (true)} is called, a reference to the * {@link org.antlr.v4.runtime.Parser.TraceListener} is stored here so it can be easily removed in a @@ -355,12 +361,6 @@ open class Parser: Recognizer { _parseListeners = nil } } - -// if (_parseListeners.remove(listener)) { -// if (_parseListeners.isEmpty) { -// _parseListeners = nil; -// } -// } } } @@ -440,7 +440,7 @@ open class Parser: Recognizer { let serializedAtn: String = getSerializedATN() var result: ATN? = bypassAltsAtnCache[serializedAtn] - synchronized { + try! bypassAltsAtnCacheMutex.synchronized { [unowned self] in if result == nil { let deserializationOptions: ATNDeserializationOptions = ATNDeserializationOptions() @@ -448,8 +448,6 @@ open class Parser: Recognizer { result = try! ATNDeserializer(deserializationOptions).deserialize(Array(serializedAtn.characters)) self.bypassAltsAtnCache[serializedAtn] = result! } - - } return result! } @@ -988,7 +986,7 @@ open class Parser: Recognizer { guard let _interp = _interp else { return s } - synchronized { + try! decisionToDFAMutex.synchronized { [unowned self] in for d in 0..<_interp.decisionToDFA.count { @@ -1005,7 +1003,7 @@ open class Parser: Recognizer { guard let _interp = _interp else { return } - synchronized { + try! decisionToDFAMutex.synchronized { [unowned self] in var seenOne: Bool = false diff --git a/runtime/Swift/Sources/Antlr4/Recognizer.swift b/runtime/Swift/Sources/Antlr4/Recognizer.swift index bd7d5d07f..bb7ddc0c5 100644 --- a/runtime/Swift/Sources/Antlr4/Recognizer.swift +++ b/runtime/Swift/Sources/Antlr4/Recognizer.swift @@ -19,6 +19,12 @@ open class Recognizer { public var _interp: ATNInterpreter! private var _stateNumber: Int = -1 + + /// mutex for tokenTypeMapCache updates + private var tokenTypeMapCacheMutex = Mutex() + + /// mutex for ruleIndexMapCacheMutex updates + private var ruleIndexMapCacheMutex = Mutex() /** Used to print out token names like ID during debugging and * error reporting. The generated parsers implement a method @@ -57,7 +63,7 @@ open class Recognizer { public func getTokenTypeMap() -> Dictionary { let vocabulary: Vocabulary = getVocabulary() var result: Dictionary? = self.tokenTypeMapCache[vocabulary] - synchronized { + try! tokenTypeMapCacheMutex.synchronized { [unowned self] in if result == nil { result = Dictionary() @@ -80,8 +86,6 @@ open class Recognizer { self.tokenTypeMapCache[vocabulary] = result! } - - } return result! @@ -96,12 +100,11 @@ open class Recognizer { let ruleNames: [String] = getRuleNames() let result: Dictionary? = self.ruleIndexMapCache[ArrayWrapper(ruleNames)] - synchronized { + try! ruleIndexMapCacheMutex.synchronized { [unowned self] in if result == nil { self.ruleIndexMapCache[ArrayWrapper(ruleNames)] = Utils.toMap(ruleNames) } - } return result! diff --git a/runtime/Swift/Sources/Antlr4/atn/LexerATNSimulator.swift b/runtime/Swift/Sources/Antlr4/atn/LexerATNSimulator.swift index dd3b42172..7d34cc1a4 100644 --- a/runtime/Swift/Sources/Antlr4/atn/LexerATNSimulator.swift +++ b/runtime/Swift/Sources/Antlr4/atn/LexerATNSimulator.swift @@ -59,7 +59,14 @@ open class LexerATNSimulator: ATNSimulator { public var charPositionInLine: Int = 0 public final var decisionToDFA: [DFA] + internal var mode: Int = Lexer.DEFAULT_MODE + + /// mutex for DFAState change + private var dfaStateMutex = Mutex() + + /// mutex for changes to all DFAStates map + private var dfaStatesMutex = Mutex() /// Used during DFA/ATN exec to record the most recent accept configuration info @@ -648,7 +655,7 @@ open class LexerATNSimulator: ATNSimulator { print("EDGE \(p) -> \(q) upon \(t)") } - synchronized { + try! dfaStateMutex.synchronized { if p.edges == nil { // make room for tokens 1..n and -1 masquerading as index 0 //TODO ARRAY COUNT @@ -678,20 +685,19 @@ open class LexerATNSimulator: ATNSimulator { } let dfa: DFA = decisionToDFA[mode] - //synced (dfa.states) { - let existing = dfa.states[proposed] - if existing != nil { - return existing!! + + return try! dfaStatesMutex.synchronized { + if let existing = dfa.states[proposed] { + return existing! + } + + let newState: DFAState = proposed + newState.stateNumber = dfa.states.count + configs.setReadonly(true) + newState.configs = configs + dfa.states[newState] = newState + return newState } - - let newState: DFAState = proposed - - newState.stateNumber = dfa.states.count - configs.setReadonly(true) - newState.configs = configs - dfa.states[newState] = newState - return newState - //} } diff --git a/runtime/Swift/Sources/Antlr4/atn/ParserATNSimulator.swift b/runtime/Swift/Sources/Antlr4/atn/ParserATNSimulator.swift index 4bc319ea0..0cc5b0507 100644 --- a/runtime/Swift/Sources/Antlr4/atn/ParserATNSimulator.swift +++ b/runtime/Swift/Sources/Antlr4/atn/ParserATNSimulator.swift @@ -265,13 +265,18 @@ open class ParserATNSimulator: ATNSimulator { internal var _startIndex: Int = 0 internal var _outerContext: ParserRuleContext! internal var _dfa: DFA? + + /// mutex for DFAState change + private var dfaStateMutex = Mutex() + + /// mutex for changes in a DFAStates map + private var dfaStatesMutex = Mutex() - /// Testing only! - // public convenience init(_ atn : ATN, _ decisionToDFA : [DFA], - // _ sharedContextCache : PredictionContextCache) - // { - // self.init(nil, atn, decisionToDFA, sharedContextCache); - // } +// /// Testing only! +// public convenience init(_ atn : ATN, _ decisionToDFA : [DFA], +// _ sharedContextCache : PredictionContextCache) { +// self.init(nil, atn, decisionToDFA, sharedContextCache); +// } public init(_ parser: Parser, _ atn: ATN, _ decisionToDFA: [DFA], @@ -1972,7 +1977,7 @@ open class ParserATNSimulator: ATNSimulator { guard let from = from else { return to } - synchronized { + try! dfaStateMutex.synchronized { [unowned self] in if from.edges == nil { from.edges = [DFAState?](repeating: nil, count: self.atn.maxTokenType + 1 + 1) //new DFAState[atn.maxTokenType+1+1]; @@ -2006,25 +2011,26 @@ open class ParserATNSimulator: ATNSimulator { if D == ATNSimulator.ERROR { return D } - //TODO: synced (dfa.states) { - //synced (dfa.states) { - let existing = dfa.states[D] - if existing != nil { - return existing!! - } + + return try dfaStatesMutex.synchronized { + if let existing = dfa.states[D] { + return existing! + } - D.stateNumber = dfa.states.count - if !D.configs.isReadonly() { - try D.configs.optimizeConfigs(self) - D.configs.setReadonly(true) - } - dfa.states[D] = D - if debug { - print("adding new DFA state: \(D)") - } + D.stateNumber = dfa.states.count + + if !D.configs.isReadonly() { + try D.configs.optimizeConfigs(self) + D.configs.setReadonly(true) + } + + dfa.states[D] = D + if debug { + print("adding new DFA state: \(D)") + } - //} - return D + return D + } } func reportAttemptingFullContext(_ dfa: DFA, _ conflictingAlts: BitSet?, _ configs: ATNConfigSet, _ startIndex: Int, _ stopIndex: Int) throws { diff --git a/runtime/Swift/Sources/Antlr4/dfa/DFA.swift b/runtime/Swift/Sources/Antlr4/dfa/DFA.swift index 93eca7bc0..ab9f17e14 100644 --- a/runtime/Swift/Sources/Antlr4/dfa/DFA.swift +++ b/runtime/Swift/Sources/Antlr4/dfa/DFA.swift @@ -20,6 +20,9 @@ public class DFA: CustomStringConvertible { /// {@code true} if this DFA is for a precedence decision; otherwise, /// {@code false}. This is the backing field for {@link #isPrecedenceDfa}. private final var precedenceDfa: Bool + + /// mutex for DFAState changes. + private var dfaStateMutex = Mutex() public convenience init(_ atnStartState: DecisionState) { self.init(atnStartState, 0) @@ -102,13 +105,11 @@ public class DFA: CustomStringConvertible { } // synchronization on s0 here is ok. when the DFA is turned into a // precedence DFA, s0 will be initialized once and not updated again - synchronized { + try! dfaStateMutex.synchronized { // s0.edges is never null for a precedence DFA if precedence >= edges.count { let increase = [DFAState?](repeating: nil, count: (precedence + 1 - edges.count)) s0.edges = edges + increase - //Array( self.s0!.edges![0..(closure: () -> R) { - pthread_mutex_lock(&_GLOBAL_MUTEX) - defer { - pthread_mutex_unlock(&_GLOBAL_MUTEX) - } - _ = closure() -} - diff --git a/runtime/Swift/Sources/Antlr4/misc/utils/Mutex.swift b/runtime/Swift/Sources/Antlr4/misc/utils/Mutex.swift new file mode 100644 index 000000000..598e4d08d --- /dev/null +++ b/runtime/Swift/Sources/Antlr4/misc/utils/Mutex.swift @@ -0,0 +1,30 @@ +import Foundation + + +/// Using class so it can be shared even if +/// it appears to be a field in a class. +class Mutex { + + /// The mutex instance. + private var mutex = pthread_mutex_t() + + /// Initialization + init() { + pthread_mutex_init(&mutex, nil) + } + + /// Running the supplied closure synchronously. + /// + /// - Parameter closure: the closure to run + /// - Returns: the value returned by the closure + /// - Throws: the exception populated by the closure run + @discardableResult + func synchronized(closure: () throws -> R) throws -> R { + pthread_mutex_lock(&mutex) + defer { + pthread_mutex_unlock(&mutex) + } + return try closure() + } + +} From 4c0bbfd768711955eb25109321e7475c17bffb60 Mon Sep 17 00:00:00 2001 From: Hanzhou Shi Date: Sat, 20 May 2017 23:37:59 -0700 Subject: [PATCH 18/23] Marking Mutex.synchronized with rethrow. --- runtime/Swift/Sources/Antlr4/Parser.swift | 8 ++++---- runtime/Swift/Sources/Antlr4/Recognizer.swift | 4 ++-- runtime/Swift/Sources/Antlr4/atn/LexerATNSimulator.swift | 4 ++-- runtime/Swift/Sources/Antlr4/atn/ParserATNSimulator.swift | 2 +- runtime/Swift/Sources/Antlr4/dfa/DFA.swift | 2 +- runtime/Swift/Sources/Antlr4/misc/utils/Mutex.swift | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/runtime/Swift/Sources/Antlr4/Parser.swift b/runtime/Swift/Sources/Antlr4/Parser.swift index 783562b37..81ce2ac4f 100644 --- a/runtime/Swift/Sources/Antlr4/Parser.swift +++ b/runtime/Swift/Sources/Antlr4/Parser.swift @@ -440,12 +440,12 @@ open class Parser: Recognizer { let serializedAtn: String = getSerializedATN() var result: ATN? = bypassAltsAtnCache[serializedAtn] - try! bypassAltsAtnCacheMutex.synchronized { + bypassAltsAtnCacheMutex.synchronized { [unowned self] in if result == nil { let deserializationOptions: ATNDeserializationOptions = ATNDeserializationOptions() try! deserializationOptions.setGenerateRuleBypassTransitions(true) - result = try! ATNDeserializer(deserializationOptions).deserialize(Array(serializedAtn.characters)) + result = try! ATNDeserializer(deserializationOptions).deserialize(Array(serializedAtn.characters)) self.bypassAltsAtnCache[serializedAtn] = result! } } @@ -986,7 +986,7 @@ open class Parser: Recognizer { guard let _interp = _interp else { return s } - try! decisionToDFAMutex.synchronized { + decisionToDFAMutex.synchronized { [unowned self] in for d in 0..<_interp.decisionToDFA.count { @@ -1003,7 +1003,7 @@ open class Parser: Recognizer { guard let _interp = _interp else { return } - try! decisionToDFAMutex.synchronized { + decisionToDFAMutex.synchronized { [unowned self] in var seenOne: Bool = false diff --git a/runtime/Swift/Sources/Antlr4/Recognizer.swift b/runtime/Swift/Sources/Antlr4/Recognizer.swift index bb7ddc0c5..525e46756 100644 --- a/runtime/Swift/Sources/Antlr4/Recognizer.swift +++ b/runtime/Swift/Sources/Antlr4/Recognizer.swift @@ -63,7 +63,7 @@ open class Recognizer { public func getTokenTypeMap() -> Dictionary { let vocabulary: Vocabulary = getVocabulary() var result: Dictionary? = self.tokenTypeMapCache[vocabulary] - try! tokenTypeMapCacheMutex.synchronized { + tokenTypeMapCacheMutex.synchronized { [unowned self] in if result == nil { result = Dictionary() @@ -100,7 +100,7 @@ open class Recognizer { let ruleNames: [String] = getRuleNames() let result: Dictionary? = self.ruleIndexMapCache[ArrayWrapper(ruleNames)] - try! ruleIndexMapCacheMutex.synchronized { + ruleIndexMapCacheMutex.synchronized { [unowned self] in if result == nil { self.ruleIndexMapCache[ArrayWrapper(ruleNames)] = Utils.toMap(ruleNames) diff --git a/runtime/Swift/Sources/Antlr4/atn/LexerATNSimulator.swift b/runtime/Swift/Sources/Antlr4/atn/LexerATNSimulator.swift index 7d34cc1a4..4d8694a96 100644 --- a/runtime/Swift/Sources/Antlr4/atn/LexerATNSimulator.swift +++ b/runtime/Swift/Sources/Antlr4/atn/LexerATNSimulator.swift @@ -655,7 +655,7 @@ open class LexerATNSimulator: ATNSimulator { print("EDGE \(p) -> \(q) upon \(t)") } - try! dfaStateMutex.synchronized { + dfaStateMutex.synchronized { if p.edges == nil { // make room for tokens 1..n and -1 masquerading as index 0 //TODO ARRAY COUNT @@ -686,7 +686,7 @@ open class LexerATNSimulator: ATNSimulator { let dfa: DFA = decisionToDFA[mode] - return try! dfaStatesMutex.synchronized { + return dfaStatesMutex.synchronized { if let existing = dfa.states[proposed] { return existing! } diff --git a/runtime/Swift/Sources/Antlr4/atn/ParserATNSimulator.swift b/runtime/Swift/Sources/Antlr4/atn/ParserATNSimulator.swift index 0cc5b0507..edc3c38fc 100644 --- a/runtime/Swift/Sources/Antlr4/atn/ParserATNSimulator.swift +++ b/runtime/Swift/Sources/Antlr4/atn/ParserATNSimulator.swift @@ -1977,7 +1977,7 @@ open class ParserATNSimulator: ATNSimulator { guard let from = from else { return to } - try! dfaStateMutex.synchronized { + dfaStateMutex.synchronized { [unowned self] in if from.edges == nil { from.edges = [DFAState?](repeating: nil, count: self.atn.maxTokenType + 1 + 1) //new DFAState[atn.maxTokenType+1+1]; diff --git a/runtime/Swift/Sources/Antlr4/dfa/DFA.swift b/runtime/Swift/Sources/Antlr4/dfa/DFA.swift index ab9f17e14..a966fe25b 100644 --- a/runtime/Swift/Sources/Antlr4/dfa/DFA.swift +++ b/runtime/Swift/Sources/Antlr4/dfa/DFA.swift @@ -105,7 +105,7 @@ public class DFA: CustomStringConvertible { } // synchronization on s0 here is ok. when the DFA is turned into a // precedence DFA, s0 will be initialized once and not updated again - try! dfaStateMutex.synchronized { + dfaStateMutex.synchronized { // s0.edges is never null for a precedence DFA if precedence >= edges.count { let increase = [DFAState?](repeating: nil, count: (precedence + 1 - edges.count)) diff --git a/runtime/Swift/Sources/Antlr4/misc/utils/Mutex.swift b/runtime/Swift/Sources/Antlr4/misc/utils/Mutex.swift index 598e4d08d..84bdafe28 100644 --- a/runtime/Swift/Sources/Antlr4/misc/utils/Mutex.swift +++ b/runtime/Swift/Sources/Antlr4/misc/utils/Mutex.swift @@ -19,7 +19,7 @@ class Mutex { /// - Returns: the value returned by the closure /// - Throws: the exception populated by the closure run @discardableResult - func synchronized(closure: () throws -> R) throws -> R { + func synchronized(closure: () throws -> R) rethrows -> R { pthread_mutex_lock(&mutex) defer { pthread_mutex_unlock(&mutex) From 19f584da05bcc277f9804a596765acfc3c8172c5 Mon Sep 17 00:00:00 2001 From: Mike Lischke Date: Sun, 4 Jun 2017 12:24:06 +0200 Subject: [PATCH 19/23] Fixed build after UTF32 string convesion patch. Also made converter local vars in conversion routines, instead of static global vars. --- runtime/Cpp/runtime/src/ANTLRInputStream.h | 6 +-- runtime/Cpp/runtime/src/antlr4-common.h | 10 +++-- .../Cpp/runtime/src/support/StringUtils.cpp | 16 ++++---- runtime/Cpp/runtime/src/support/StringUtils.h | 41 +++++++++++-------- 4 files changed, 40 insertions(+), 33 deletions(-) diff --git a/runtime/Cpp/runtime/src/ANTLRInputStream.h b/runtime/Cpp/runtime/src/ANTLRInputStream.h index e904ff9b7..e9850504d 100755 --- a/runtime/Cpp/runtime/src/ANTLRInputStream.h +++ b/runtime/Cpp/runtime/src/ANTLRInputStream.h @@ -16,11 +16,7 @@ namespace antlr4 { protected: /// The data being scanned. // UTF-32 -#if defined(_MSC_VER) && _MSC_VER == 1900 - i32string _data; // Custom type for VS 2015. -#else - std::u32string _data; -#endif + UTF32String _data; /// 0..n-1 index into string of next char size_t p; diff --git a/runtime/Cpp/runtime/src/antlr4-common.h b/runtime/Cpp/runtime/src/antlr4-common.h index 9272d1958..85477d813 100644 --- a/runtime/Cpp/runtime/src/antlr4-common.h +++ b/runtime/Cpp/runtime/src/antlr4-common.h @@ -55,11 +55,13 @@ typedef __int32 ssize_t; #endif - #if _MSC_VER == 1900 + #if _MSC_VER >= 1900 && _MSC_VER < 2000 // VS 2015 has a known bug when using std::codecvt_utf8 // so we have to temporarily use __int32 instead. // https://connect.microsoft.com/VisualStudio/feedback/details/1403302/unresolved-external-when-using-codecvt-utf8 typedef std::basic_string<__int32> i32string; + + typedef i32string UTF32String; #endif #ifdef ANTLR4CPP_EXPORTS @@ -72,11 +74,11 @@ #endif #endif -#ifdef _MSC_VER class ANTLR4CPP_PUBLIC std::exception; // Needed for VS 2015. -#endif #elif __APPLE__ + typedef std::u32string UTF32String; + #define GUID_CFUUID #if __GNUC__ >= 4 #define ANTLR4CPP_PUBLIC __attribute__ ((visibility ("default"))) @@ -84,6 +86,8 @@ #define ANTLR4CPP_PUBLIC #endif #else + typedef std::u32string UTF32String; + #define GUID_LIBUUID #if __GNUC__ >= 6 #define ANTLR4CPP_PUBLIC __attribute__ ((visibility ("default"))) diff --git a/runtime/Cpp/runtime/src/support/StringUtils.cpp b/runtime/Cpp/runtime/src/support/StringUtils.cpp index 5f4a0bce7..552f1031a 100644 --- a/runtime/Cpp/runtime/src/support/StringUtils.cpp +++ b/runtime/Cpp/runtime/src/support/StringUtils.cpp @@ -7,29 +7,29 @@ namespace antlrcpp { -void replaceAll(std::string& str, const std::string& from, const std::string& to) +void replaceAll(std::string& str, std::string const& from, std::string const& to) { - if(from.empty()) { + if (from.empty()) return; - } + size_t start_pos = 0; - while((start_pos = str.find(from, start_pos)) != std::string::npos) { + while ((start_pos = str.find(from, start_pos)) != std::string::npos) { str.replace(start_pos, from.length(), to); - start_pos += to.length(); // In case 'to' contains 'from', like replacing 'x' with 'yx' + start_pos += to.length(); // In case 'to' contains 'from', like replacing 'x' with 'yx'. } } -std::string ws2s(const std::wstring &wstr) { +std::string ws2s(std::wstring const& wstr) { std::wstring_convert> converter; - std::string narrow = converter.to_bytes(wstr); + return narrow; } std::wstring s2ws(const std::string &str) { std::wstring_convert> converter; - std::wstring wide = converter.from_bytes(str); + return wide; } diff --git a/runtime/Cpp/runtime/src/support/StringUtils.h b/runtime/Cpp/runtime/src/support/StringUtils.h index cd4d81c1d..ac0fcc765 100644 --- a/runtime/Cpp/runtime/src/support/StringUtils.h +++ b/runtime/Cpp/runtime/src/support/StringUtils.h @@ -8,40 +8,47 @@ #include "antlr4-common.h" namespace antlrcpp { - // For all conversions utf8 <-> utf32. - // VS 2015 and VS 2017 have different bugs in std::codecvt_utf8 (VS 2013 works fine). + +// For all conversions utf8 <-> utf32. +// VS 2015 and VS 2017 have different bugs in std::codecvt_utf8 (VS 2013 works fine). #if defined(_MSC_VER) && _MSC_VER >= 1900 && _MSC_VER < 2000 - static std::wstring_convert, __int32> utfConverter; + typedef std::wstring_convert, __int32> UTF32Converter; #else - static std::wstring_convert, char32_t> utfConverter; + typedef std::wstring_convert, char32_t> UTF32Converter; #endif - //the conversion functions fails in VS2017, so we explicitly use a workaround + // The conversion functions fails in VS2017, so we explicitly use a workaround. template - inline std::string utf32_to_utf8(T _data) + inline std::string utf32_to_utf8(T const& data) { + // Don't make the converter static or we have to serialize access to it. + UTF32Converter converter; + #if _MSC_VER > 1900 && _MSC_VER < 2000 - auto p = reinterpret_cast(_data.data()); - return antlrcpp::utfConverter.to_bytes(p, p + _data.size()); + auto p = reinterpret_cast(data.data()); + return converter.to_bytes(p, p + data.size()); #else - return antlrcpp::utfConverter.to_bytes(_data); + return converter.to_bytes(data); #endif } - inline auto utf8_to_utf32(const char* first, const char* last) + inline UTF32String utf8_to_utf32(const char* first, const char* last) { + UTF32Converter converter; + #if _MSC_VER > 1900 && _MSC_VER < 2000 - auto r = antlrcpp::utfConverter.from_bytes(first, last); - std::u32string s = reinterpret_cast(r.data()); - return s; + auto r = converter.from_bytes(first, last); + i32string s = reinterpret_cast(r.data()); #else - return antlrcpp::utfConverter.from_bytes(first, last); + std::u32string s = converter.from_bytes(first, last); #endif + + return s; } - void replaceAll(std::string& str, const std::string& from, const std::string& to); + void replaceAll(std::string &str, std::string const& from, std::string const& to); // string <-> wstring conversion (UTF-16), e.g. for use with Window's wide APIs. - ANTLR4CPP_PUBLIC std::string ws2s(const std::wstring &wstr); - ANTLR4CPP_PUBLIC std::wstring s2ws(const std::string &str); + ANTLR4CPP_PUBLIC std::string ws2s(std::wstring const& wstr); + ANTLR4CPP_PUBLIC std::wstring s2ws(std::string const& str); } From cec714f7454c2736c5508f6e12fbd8399914dc26 Mon Sep 17 00:00:00 2001 From: Mike Lischke Date: Sun, 4 Jun 2017 14:26:55 +0200 Subject: [PATCH 20/23] Corrections for VS 2015. --- runtime/Cpp/runtime/src/support/StringUtils.h | 4 ++-- runtime/Cpp/runtime/src/tree/TerminalNodeImpl.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/runtime/Cpp/runtime/src/support/StringUtils.h b/runtime/Cpp/runtime/src/support/StringUtils.h index ac0fcc765..dc80c077b 100644 --- a/runtime/Cpp/runtime/src/support/StringUtils.h +++ b/runtime/Cpp/runtime/src/support/StringUtils.h @@ -24,7 +24,7 @@ namespace antlrcpp { // Don't make the converter static or we have to serialize access to it. UTF32Converter converter; - #if _MSC_VER > 1900 && _MSC_VER < 2000 + #if _MSC_VER >= 1900 && _MSC_VER < 2000 auto p = reinterpret_cast(data.data()); return converter.to_bytes(p, p + data.size()); #else @@ -36,7 +36,7 @@ namespace antlrcpp { { UTF32Converter converter; - #if _MSC_VER > 1900 && _MSC_VER < 2000 + #if _MSC_VER >= 1900 && _MSC_VER < 2000 auto r = converter.from_bytes(first, last); i32string s = reinterpret_cast(r.data()); #else diff --git a/runtime/Cpp/runtime/src/tree/TerminalNodeImpl.cpp b/runtime/Cpp/runtime/src/tree/TerminalNodeImpl.cpp index a7fe749db..0f806d4e7 100755 --- a/runtime/Cpp/runtime/src/tree/TerminalNodeImpl.cpp +++ b/runtime/Cpp/runtime/src/tree/TerminalNodeImpl.cpp @@ -20,8 +20,8 @@ Token* TerminalNodeImpl::getSymbol() { return symbol; } -void TerminalNodeImpl::setParent(RuleContext *parent) { - this->parent = parent; +void TerminalNodeImpl::setParent(RuleContext *parent_) { + this->parent = parent_; } misc::Interval TerminalNodeImpl::getSourceInterval() { From aad5faf8196c1eb0b2ac2530f963756993b02788 Mon Sep 17 00:00:00 2001 From: Mike Lischke Date: Sun, 11 Jun 2017 12:57:49 +0200 Subject: [PATCH 21/23] Revert "Extended tokenVocab search to the grammar subfolder." This reverts commit defad74649538a5eff872ee4af486a5ef88bd145. --- .../org/antlr/v4/parse/TokenVocabParser.java | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/tool/src/org/antlr/v4/parse/TokenVocabParser.java b/tool/src/org/antlr/v4/parse/TokenVocabParser.java index cf84a923a..4267a8d96 100644 --- a/tool/src/org/antlr/v4/parse/TokenVocabParser.java +++ b/tool/src/org/antlr/v4/parse/TokenVocabParser.java @@ -146,22 +146,6 @@ public class TokenVocabParser { // files are generated (in the base, not relative to the input // location.) f = new File(g.tool.outputDirectory, vocabName + CodeGenerator.VOCAB_FILE_EXTENSION); - if ( f.exists() ) { - return f; - } - - // Still not found? Use the grammar's subfolder then. - String fileDirectory; - - if (g.fileName.lastIndexOf(File.separatorChar) == -1) { - // No path is included in the file name, so make the file - // directory the same as the parent grammar (which might still be just "" - // but when it is not, we will write the file in the correct place. - fileDirectory = "."; - } - else { - fileDirectory = g.fileName.substring(0, g.fileName.lastIndexOf(File.separatorChar)); - } - return new File(fileDirectory, vocabName + CodeGenerator.VOCAB_FILE_EXTENSION); + return f; } } From bcb0a8f1503a673d480404d69b0460b6108032b6 Mon Sep 17 00:00:00 2001 From: Mike Lischke Date: Sun, 11 Jun 2017 13:01:04 +0200 Subject: [PATCH 22/23] Revert "Simplify output path generation" This reverts commit 47e43cc8d6b5d1dfe5ff494503e6d0eec63d4b39. --- tool/src/org/antlr/v4/Tool.java | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/tool/src/org/antlr/v4/Tool.java b/tool/src/org/antlr/v4/Tool.java index 6638ff5a1..42ace2ecb 100644 --- a/tool/src/org/antlr/v4/Tool.java +++ b/tool/src/org/antlr/v4/Tool.java @@ -761,11 +761,18 @@ public class Tool { File outputDir; String fileDirectory; + // Some files are given to us without a PATH but should should + // still be written to the output directory in the relative path of + // the output directory. The file directory is either the set of sub directories + // or just or the relative path recorded for the parent grammar. This means + // that when we write the tokens files, or the .java files for imported grammars + // taht we will write them in the correct place. if (fileNameWithPath.lastIndexOf(File.separatorChar) == -1) { // No path is included in the file name, so make the file - // directory the same as the parent grammar (which might still be just "" + // directory the same as the parent grammar (which might sitll be just "" // but when it is not, we will write the file in the correct place. fileDirectory = "."; + } else { fileDirectory = fileNameWithPath.substring(0, fileNameWithPath.lastIndexOf(File.separatorChar)); @@ -774,8 +781,21 @@ public class Tool { // -o /tmp /var/lib/t.g4 => /tmp/T.java // -o subdir/output /usr/lib/t.g4 => subdir/output/T.java // -o . /usr/lib/t.g4 => ./T.java - // -o /tmp subdir/t.g4 => /tmp/t.g4 - outputDir = new File(outputDirectory); + if (fileDirectory != null && + (new File(fileDirectory).isAbsolute() || + fileDirectory.startsWith("~"))) { // isAbsolute doesn't count this :( + // somebody set the dir, it takes precendence; write new file there + outputDir = new File(outputDirectory); + } + else { + // -o /tmp subdir/t.g4 => /tmp/subdir/t.g4 + if (fileDirectory != null) { + outputDir = new File(outputDirectory, fileDirectory); + } + else { + outputDir = new File(outputDirectory); + } + } } else { // they didn't specify a -o dir so just write to location From ef49f61819e06bd6633e9a7193c88348cbf7b8e0 Mon Sep 17 00:00:00 2001 From: Mike Lischke Date: Sun, 11 Jun 2017 13:31:33 +0200 Subject: [PATCH 23/23] Formatting --- runtime/Cpp/runtime/src/support/StringUtils.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runtime/Cpp/runtime/src/support/StringUtils.h b/runtime/Cpp/runtime/src/support/StringUtils.h index dc80c077b..ea1b1d0e1 100644 --- a/runtime/Cpp/runtime/src/support/StringUtils.h +++ b/runtime/Cpp/runtime/src/support/StringUtils.h @@ -9,14 +9,14 @@ namespace antlrcpp { -// For all conversions utf8 <-> utf32. -// VS 2015 and VS 2017 have different bugs in std::codecvt_utf8 (VS 2013 works fine). + // For all conversions utf8 <-> utf32. + // VS 2015 and VS 2017 have different bugs in std::codecvt_utf8 (VS 2013 works fine). #if defined(_MSC_VER) && _MSC_VER >= 1900 && _MSC_VER < 2000 typedef std::wstring_convert, __int32> UTF32Converter; #else typedef std::wstring_convert, char32_t> UTF32Converter; #endif - + // The conversion functions fails in VS2017, so we explicitly use a workaround. template inline std::string utf32_to_utf8(T const& data)