From 31aa7bf5c9c15821c92e66384b636e48d80af346 Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Sat, 28 Jun 2014 10:46:25 -0700 Subject: [PATCH 1/2] Re-factor loading profile in the streams so that I can reuse that load elsewhere. handy function. --- .../org/antlr/v4/runtime/ANTLRFileStream.java | 31 ++------------- .../src/org/antlr/v4/runtime/misc/Utils.java | 38 ++++++++++++++++++- 2 files changed, 41 insertions(+), 28 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/ANTLRFileStream.java b/runtime/Java/src/org/antlr/v4/runtime/ANTLRFileStream.java index 340a59e93..ba4a53136 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/ANTLRFileStream.java +++ b/runtime/Java/src/org/antlr/v4/runtime/ANTLRFileStream.java @@ -29,11 +29,9 @@ */ package org.antlr.v4.runtime; -import java.io.File; -import java.io.FileInputStream; +import org.antlr.v4.runtime.misc.Utils; + import java.io.IOException; -import java.io.InputStreamReader; -import java.util.Arrays; /** * This is an {@link ANTLRInputStream} that is loaded from a file all at once @@ -54,29 +52,8 @@ public class ANTLRFileStream extends ANTLRInputStream { public void load(String fileName, String encoding) throws IOException { - if ( fileName==null ) { - return; - } - File f = new File(fileName); - int size = (int)f.length(); - InputStreamReader isr; - FileInputStream fis = new FileInputStream(fileName); - if ( encoding!=null ) { - isr = new InputStreamReader(fis, encoding); - } - else { - isr = new InputStreamReader(fis); - } - try { - data = new char[size]; - n = isr.read(data); - if (n < data.length) { - data = Arrays.copyOf(data, n); - } - } - finally { - isr.close(); - } + data = Utils.readFile(fileName, encoding); + this.n = data.length; } @Override diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java b/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java index 7414bd028..427e581d4 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java @@ -30,13 +30,17 @@ package org.antlr.v4.runtime.misc; -import java.awt.*; +import java.awt.Window; import java.awt.event.WindowAdapter; import java.awt.event.WindowEvent; import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; import java.io.FileWriter; import java.io.IOException; +import java.io.InputStreamReader; import java.io.Writer; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.Iterator; @@ -104,6 +108,38 @@ public class Utils { } } + public static char[] readFile(String fileName) throws IOException { + return readFile(fileName, null); + } + + public static char[] readFile(String fileName, String encoding) throws IOException { + if ( fileName==null ) { + return null; + } + File f = new File(fileName); + int size = (int)f.length(); + InputStreamReader isr; + FileInputStream fis = new FileInputStream(fileName); + if ( encoding!=null ) { + isr = new InputStreamReader(fis, encoding); + } + else { + isr = new InputStreamReader(fis); + } + char[] data = null; + try { + data = new char[size]; + int n = isr.read(data); + if (n < data.length) { + data = Arrays.copyOf(data, n); + } + } + finally { + isr.close(); + } + return data; + } + public static void waitForClose(final Window window) throws InterruptedException { final Object lock = new Object(); From 2e352e269754cbd3985b35f0dcc52c58e25ffdd7 Mon Sep 17 00:00:00 2001 From: Terence Parr Date: Sat, 28 Jun 2014 14:00:54 -0700 Subject: [PATCH 2/2] add unit tests for -o and -lib; improve BaseTest: refactor writeFile to Utils. -o options and others not accepted by antlr(). --- .../src/org/antlr/v4/runtime/misc/Utils.java | 25 +++-- tool/test/org/antlr/v4/test/BaseTest.java | 56 +++++++----- .../antlr/v4/test/TestCompositeGrammars.java | 91 ++++++++++++++++++- 3 files changed, 138 insertions(+), 34 deletions(-) diff --git a/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java b/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java index 427e581d4..619afdcd6 100644 --- a/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java +++ b/runtime/Java/src/org/antlr/v4/runtime/misc/Utils.java @@ -33,13 +33,12 @@ package org.antlr.v4.runtime.misc; import java.awt.Window; import java.awt.event.WindowAdapter; import java.awt.event.WindowEvent; -import java.io.BufferedWriter; import java.io.File; import java.io.FileInputStream; -import java.io.FileWriter; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStreamReader; -import java.io.Writer; +import java.io.OutputStreamWriter; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; @@ -98,13 +97,25 @@ public class Utils { } public static void writeFile(String fileName, String content) throws IOException { - FileWriter fw = new FileWriter(fileName); - Writer w = new BufferedWriter(fw); + writeFile(fileName, content, null); + } + + public static void writeFile(String fileName, String content, String encoding) throws IOException { + File f = new File(fileName); + FileOutputStream fos = new FileOutputStream(f); + OutputStreamWriter osw; + if (encoding != null) { + osw = new OutputStreamWriter(fos, encoding); + } + else { + osw = new OutputStreamWriter(fos); + } + try { - w.write(content); + osw.write(content); } finally { - w.close(); + osw.close(); } } diff --git a/tool/test/org/antlr/v4/test/BaseTest.java b/tool/test/org/antlr/v4/test/BaseTest.java index cfd700512..d11d7a216 100644 --- a/tool/test/org/antlr/v4/test/BaseTest.java +++ b/tool/test/org/antlr/v4/test/BaseTest.java @@ -59,6 +59,7 @@ import org.antlr.v4.runtime.misc.Interval; import org.antlr.v4.runtime.misc.NotNull; import org.antlr.v4.runtime.misc.Nullable; import org.antlr.v4.runtime.misc.Pair; +import org.antlr.v4.runtime.misc.Utils; import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.semantics.SemanticPipeline; import org.antlr.v4.tool.ANTLRMessage; @@ -81,13 +82,10 @@ import javax.tools.JavaFileObject; import javax.tools.StandardJavaFileManager; import javax.tools.ToolProvider; import java.io.BufferedReader; -import java.io.BufferedWriter; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; -import java.io.OutputStreamWriter; import java.io.PipedInputStream; import java.io.PipedOutputStream; import java.io.PrintStream; @@ -442,19 +440,21 @@ public abstract class BaseTest { */ } - /** Return true if all is ok, no errors */ - protected ErrorQueue antlr(String fileName, String grammarFileName, String grammarStr, boolean defaultListener, String... extraOptions) { - System.out.println("dir "+tmpdir); - mkdir(tmpdir); - writeFile(tmpdir, fileName, grammarStr); + protected ErrorQueue antlr(String grammarFileName, boolean defaultListener, String... extraOptions) { final List options = new ArrayList(); Collections.addAll(options, extraOptions); - options.add("-o"); - options.add(tmpdir); - options.add("-lib"); - options.add(tmpdir); - options.add("-encoding"); - options.add("UTF-8"); + if ( !options.contains("-o") ) { + options.add("-o"); + options.add(tmpdir); + } + if ( !options.contains("-lib") ) { + options.add("-lib"); + options.add(tmpdir); + } + if ( !options.contains("-encoding") ) { + options.add("-encoding"); + options.add("UTF-8"); + } options.add(new File(tmpdir,grammarFileName).toString()); final String[] optionsA = new String[options.size()]; @@ -474,7 +474,12 @@ public abstract class BaseTest { System.err.println(msg); } System.out.println("!!!\ngrammar:"); - System.out.println(grammarStr); + try { + System.out.println(new String(Utils.readFile(tmpdir+"/"+grammarFileName))); + } + catch (IOException ioe) { + System.err.println(ioe.toString()); + } System.out.println("###"); } if ( !defaultListener && !equeue.warnings.isEmpty() ) { @@ -488,6 +493,13 @@ public abstract class BaseTest { return equeue; } + protected ErrorQueue antlr(String grammarFileName, String grammarStr, boolean defaultListener, String... extraOptions) { + System.out.println("dir "+tmpdir); + mkdir(tmpdir); + writeFile(tmpdir, grammarFileName, grammarStr); + return antlr(grammarFileName, defaultListener, extraOptions); + } + protected String execLexer(String grammarFileName, String grammarStr, String lexerName, @@ -599,7 +611,7 @@ public abstract class BaseTest { String input, boolean debug, boolean profile) { - boolean success = rawGenerateAndBuildRecognizer(grammarFileName, + boolean success = rawGenerateAndBuildRecognizer(grammarFileName, grammarStr, parserName, lexerName, @@ -632,7 +644,7 @@ public abstract class BaseTest { String... extraOptions) { ErrorQueue equeue = - antlr(grammarFileName, grammarFileName, grammarStr, defaultListener, extraOptions); + antlr(grammarFileName, grammarStr, defaultListener, extraOptions); if (!equeue.errors.isEmpty()) { return false; } @@ -789,7 +801,7 @@ public abstract class BaseTest { String[] lines = input.split("\n"); String fileName = getFilenameFromFirstLineOfGrammar(lines[0]); - ErrorQueue equeue = antlr(fileName, fileName, input, false); + ErrorQueue equeue = antlr(fileName, input, false); String actual = equeue.toString(true); actual = actual.replace(tmpdir + File.separator, ""); @@ -1027,13 +1039,7 @@ public abstract class BaseTest { public static void writeFile(String dir, String fileName, String content) { try { - File f = new File(dir, fileName); - FileOutputStream outputStream = new FileOutputStream(f); - OutputStreamWriter w = new OutputStreamWriter(outputStream, "UTF-8"); - BufferedWriter bw = new BufferedWriter(w); - bw.write(content); - bw.close(); - w.close(); + Utils.writeFile(dir+"/"+fileName, content, "UTF-8"); } catch (IOException ioe) { System.err.println("can't write file"); diff --git a/tool/test/org/antlr/v4/test/TestCompositeGrammars.java b/tool/test/org/antlr/v4/test/TestCompositeGrammars.java index 63174b438..b11d9e903 100644 --- a/tool/test/org/antlr/v4/test/TestCompositeGrammars.java +++ b/tool/test/org/antlr/v4/test/TestCompositeGrammars.java @@ -35,11 +35,98 @@ import org.antlr.v4.tool.Grammar; import org.antlr.v4.tool.GrammarSemanticsMessage; import org.junit.Test; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; public class TestCompositeGrammars extends BaseTest { protected boolean debug = false; + @Test public void testImportFileLocationInSubdir() throws Exception { + String slave = + "parser grammar S;\n" + + "a : B {System.out.println(\"S.a\");} ;\n"; + mkdir(tmpdir); + String subdir = tmpdir + "/sub"; + mkdir(subdir); + writeFile(subdir, "S.g4", slave); + String master = + "grammar M;\n" + + "import S;\n" + + "s : a ;\n" + + "B : 'b' ;" + // defines B from inherited token space + "WS : (' '|'\\n') -> skip ;\n" ; + writeFile(tmpdir, "M.g4", master); + ErrorQueue equeue = antlr("M.g4", false, "-lib", subdir); + assertEquals(equeue.size(), 0); + } + + @Test public void testImportFileNotSearchedForInOutputDir() throws Exception { + String slave = + "parser grammar S;\n" + + "a : B {System.out.println(\"S.a\");} ;\n"; + mkdir(tmpdir); + String outdir = tmpdir + "/out"; + mkdir(outdir); + writeFile(outdir, "S.g4", slave); + String master = + "grammar M;\n" + + "import S;\n" + + "s : a ;\n" + + "B : 'b' ;" + // defines B from inherited token space + "WS : (' '|'\\n') -> skip ;\n" ; + writeFile(tmpdir, "M.g4", master); + ErrorQueue equeue = antlr("M.g4", false, "-o", outdir); + assertEquals(ErrorType.CANNOT_FIND_IMPORTED_GRAMMAR, equeue.errors.get(0).getErrorType()); + } + + @Test public void testOutputDirShouldNotEffectImports() throws Exception { + String slave = + "parser grammar S;\n" + + "a : B {System.out.println(\"S.a\");} ;\n"; + mkdir(tmpdir); + String subdir = tmpdir + "/sub"; + mkdir(subdir); + writeFile(subdir, "S.g4", slave); + String master = + "grammar M;\n" + + "import S;\n" + + "s : a ;\n" + + "B : 'b' ;" + // defines B from inherited token space + "WS : (' '|'\\n') -> skip ;\n" ; + writeFile(tmpdir, "M.g4", master); + String outdir = tmpdir + "/out"; + mkdir(outdir); + ErrorQueue equeue = antlr("M.g4", false, "-o", outdir, "-lib", subdir); + assertEquals(0, equeue.size()); + } + + @Test public void testTokensFileInOutputDirAndImportFileInSubdir() throws Exception { + String slave = + "parser grammar S;\n" + + "a : B {System.out.println(\"S.a\");} ;\n"; + mkdir(tmpdir); + String subdir = tmpdir + "/sub"; + mkdir(subdir); + writeFile(subdir, "S.g4", slave); + String parser = + "parser grammar MParser;\n" + + "import S;\n" + + "options {tokenVocab=MLexer;}\n" + + "s : a ;\n"; + writeFile(tmpdir, "MParser.g4", parser); + String lexer = + "lexer grammar MLexer;\n" + + "B : 'b' ;" + // defines B from inherited token space + "WS : (' '|'\\n') -> skip ;\n" ; + writeFile(tmpdir, "MLexer.g4", lexer); + String outdir = tmpdir + "/out"; + mkdir(outdir); + ErrorQueue equeue = antlr("MLexer.g4", false, "-o", outdir); + assertEquals(0, equeue.size()); + equeue = antlr("MParser.g4", false, "-o", outdir, "-lib", subdir); + assertEquals(0, equeue.size()); + } + @Test public void testDelegatorInvokesDelegateRule() throws Exception { String slave = "parser grammar S;\n" + @@ -641,7 +728,7 @@ public class TestCompositeGrammars extends BaseTest { "s : a ;\n" + "B : 'b' ;" + // defines B from inherited token space "WS : (' '|'\\n') -> skip ;\n" ; - ErrorQueue equeue = antlr("M.g4", "M.g4", master, false); + ErrorQueue equeue = antlr("M.g4", master, false); int expecting = 0; // should be ok assertEquals(expecting, equeue.errors.size()); }