Merge pull request #799 from parrt/freshen-unit-test-gen

Freshen unit test gen

bild.py

@@ -153,8 +153,11 @@ def mkjar_runtime():
     print "Made jar OSGi-ready " + jarfile
 
 
-def mkjar():
+def mkjar(): # if called as root target
     rmdir("out")
+    _mkjar()
+
+def _mkjar(): # don't wipe out out dir if we know it's done like from all()
     mkjar_complete()
     # put it in JARCARCHE too so bild can find it during antlr4()
     copyfile(src="dist/antlr4-" + VERSION + "-complete.jar", trg=JARCACHE+"/antlr-"+VERSION+"-complete.jar") # note mvn wants antlr4-ver-... but I want antlr-ver-...
@@ -174,19 +177,51 @@ def mkjar():
     mkjar_runtime()   # now build the runtime jar
 
 
-def tests():
+def regen_tests():
-    require(mkjar)
+    require(_mkjar)
     junit_jar, hamcrest_jar = load_junitjars()
     cp = uniformpath("dist/antlr4-" + VERSION + "-complete.jar") \
-         + os.pathsep + uniformpath("out/test/Java") \
+         + os.pathsep + uniformpath("out/test") \
          + os.pathsep + junit_jar \
          + os.pathsep + hamcrest_jar
+    args = ["-nowarn", "-Xlint", "-Xlint:-serial", "-g"]
+    javac("tool/test", "out/test", version="1.6", cp=cp, args=args)  # all targets can use org.antlr.v4.test.*
+    java(classname="org.antlr.v4.test.rt.gen.Generator", cp="out/test:dist/antlr4-4.5-complete.jar")
+    print "test generation complete"
+    log("test generation complete")
+
+def tests():
+    require(regen_tests)
+    for t in TARGETS:
+        test_target(t)
+
+
+def test_java():
+    test_target("Java")
+
+
+def test_python2():
+    test_target("Python2")
+
+
+def test_python3():
+    test_target("Python3")
+
+
+def test_csharp():
+    test_target("CSharp")
+
+
+def test_javascript():
+    test_target("JavaScript")
+
+
+def test_target(t):
+    require(regen_tests)
+    cp = uniformpath("dist/antlr4-" + VERSION + "-complete.jar") \
+         + os.pathsep + uniformpath("out/test")
     juprops = ["-D%s=%s" % (p, test_properties[p]) for p in test_properties]
     args = ["-nowarn", "-Xlint", "-Xlint:-serial", "-g"]
-    # don't compile generator
-    skip = [ uniformpath(TARGETS['Java'] + "/tool/test/org/antlr/v4/test/rt/gen/") ]
-    javac("tool/test", "out/test/Java", version="1.6", cp=cp, args=args, skip=skip)  # all targets can use org.antlr.v4.test.*
-    for t in TARGETS:
     print "Testing %s ..." % t
     try:
         test(t, cp, juprops, args)
@@ -195,10 +230,12 @@ def tests():
         print t + " tests failed"
 
 def test(t, cp, juprops, args):
+    junit_jar, hamcrest_jar = load_junitjars()
     srcdir = uniformpath(TARGETS[t] + "/tool/test")
     dstdir = uniformpath( "out/test/" + t)
     # Prefix CLASSPATH with individual target tests
     thiscp = dstdir + os.pathsep + cp
+    thisjarwithjunit = thiscp + os.pathsep + hamcrest_jar + os.pathsep + junit_jar
     skip = []
     if t=='Java':
         # don't test generator
@@ -207,13 +244,13 @@ def test(t, cp, juprops, args):
         # need BaseTest located in Py3 target
         base = uniformpath(TARGETS['Python3'] + "/tool/test")
         skip = [ "/org/antlr/v4/test/rt/py3/" ]
-        javac(base, "out/test/" + t, version="1.6", cp=thiscp, args=args, skip=skip)
+        javac(base, "out/test/" + t, version="1.6", cp=thisjarwithjunit, args=args, skip=skip)
         skip = []
     elif t=='JavaScript':
         # don't test browsers automatically, this is overkilling and unreliable
         browsers = ["safari","chrome","firefox","explorer"]
         skip = [ uniformpath(srcdir + "/org/antlr/v4/test/rt/js/" + b) for b in browsers ]
-    javac(srcdir, "out/test/" + t, version="1.6", cp=thiscp, args=args, skip=skip)
+    javac(srcdir, trgdir="out/test/" + t, version="1.6", cp=thisjarwithjunit, args=args, skip=skip)
     # copy resource files required for testing
     files = allfiles(srcdir)
     for src in files:
@@ -225,7 +262,7 @@ def test(t, cp, juprops, args):
     junit("out/test/" + t, cp=thiscp, verbose=False, args=juprops)
 
 def install():
-    require(mkjar)
+    require(_mkjar)
     require(mksrc)
     require(mkdoc)
     mvn_install("dist/antlr4-" + VERSION + "-complete.jar",
@@ -243,7 +280,7 @@ def install():
 
 
 def deploy():
-    require(mkjar)
+    require(_mkjar)
     require(mksrc)
     require(mkdoc)
     binjar = uniformpath("dist/antlr4-%s-complete.jar" % VERSION)
@@ -339,7 +376,7 @@ def mkdoc():
 
 def all():
     clean(True)
-    mkjar()
+    _mkjar()
     tests()
     mkdoc()
     mksrc()

tool/test/org/antlr/v4/test/rt/gen/Generator.java

@@ -1,6 +1,5 @@
 package org.antlr.v4.test.rt.gen;
 
-import org.antlr.v4.test.rt.java.BaseTest;
 import org.stringtemplate.v4.ST;
 import org.stringtemplate.v4.STGroup;
 import org.stringtemplate.v4.STGroupFile;
@@ -9,97 +8,14 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.OutputStream;
-import java.net.URI;
-import java.net.URL;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 
 public class Generator {
-
+	public static String antlrRoot = "."; // assume root is current working dir
-	public static void main(String[] args) throws Exception {
-		Map<String, File> configs = readConfigs();
-		File source = configs.get("Source");
-		for(Map.Entry<String, File> item : configs.entrySet()) {
-			if("Source".equals(item.getKey()))
-				continue;
-			Generator gen = new Generator(item.getKey(), source, item.getValue());
-			gen.generateTests();
-		}
-	}
-
-	private static Map<String, File> readConfigs() throws Exception {
-		Map<String, File> configs = new HashMap<String, File>();
-		configs.put("Source", readGrammarDir()); // source of test templates
-		configs.put("Java", readJavaDir()); // generated Java tests
-		configs.put("CSharp", readCSharpDir()); // generated CSharp tests
-		configs.put("Python2", readPython2Dir()); // generated Python2 tests
-		configs.put("Python3", readPython3Dir()); // generated Python3 tests
-		configs.put("NodeJS", readNodeJSDir()); // generated NodeJS tests
-		configs.put("Safari", readSafariDir()); // generated Safari tests
-		configs.put("Firefox", readFirefoxDir()); // generated Firefox tests
-		configs.put("Chrome", readChromeDir()); // generated Chrome tests
-		configs.put("Explorer", readExplorerDir()); // generated Explorer tests
-		return configs;
-	}
-
-	private static File readJavaDir() throws Exception {
-		String className = BaseTest.class.getName().replace(".", "/");
-		className = className.substring(0, className.lastIndexOf("/") + 1);
-		URL url = ClassLoader.getSystemResource(className);
-		String uri = url.toURI().toString().replace("target/test-classes", "test");
-		return new File(new URI(uri));
-	}
-
-	private static File readCSharpDir() {
-		return new File("../antlr4-csharp/tool/test/org/antlr/v4/test/rt/csharp");
-	}
-
-	private static File readPython2Dir() {
-		return new File("../antlr4-python2/tool/test/org/antlr/v4/test/rt/py2");
-	}
-
-	private static File readPython3Dir() {
-		return new File("../antlr4-python3/tool/test/org/antlr/v4/test/rt/py3");
-	}
-
-	private static File readNodeJSDir() {
-		return new File("../antlr4-javascript/tool/test/org/antlr/v4/test/rt/js/node");
-	}
-
-	private static File readSafariDir() {
-		return new File("../antlr4-javascript/tool/test/org/antlr/v4/test/rt/js/safari");
-	}
-
-	private static File readFirefoxDir() {
-		return new File("../antlr4-javascript/tool/test/org/antlr/v4/test/rt/js/firefox");
-	}
-
-	private static File readChromeDir() {
-		return new File("../antlr4-javascript/tool/test/org/antlr/v4/test/rt/js/chrome");
-	}
-
-	private static File readExplorerDir() {
-		return new File("../antlr4-javascript/tool/test/org/antlr/v4/test/rt/js/explorer");
-	}
-
-	private static File readGrammarDir() throws Exception {
-		File parent = readThisDir();
-		return new File(parent, "grammars");
-	}
-
-	private static File readThisDir() throws Exception {
-		String className = Generator.class.getName().replace(".", "/");
-		className = className.substring(0, className.lastIndexOf("/") + 1);
-		URL url = ClassLoader.getSystemResource(className);
-		return new File(url.toURI());
-	}
-
-	public static String escape(String s) {
-		return s==null ? null : s.replace("\\","\\\\").replace("\r", "\\r").replace("\n", "\\n").replace("\"","\\\"");
-	}
 
 	String target;
 	File input;
@@ -112,7 +28,81 @@ public class Generator {
 		this.output = output;
 	}
 
+	public static void main(String[] args) throws Exception {
+		if ( args.length==1 ) {
+			antlrRoot = args[0];
+		}
+		Map<String, File> configs = setup();
+		File source = configs.get("Source");
+		for(Map.Entry<String, File> item : configs.entrySet()) {
+			if( !"Source".equals(item.getKey()) ) {
+				Generator gen = new Generator(item.getKey(), source, item.getValue());
+				gen.generateTests();
+			}
+		}
+	}
+
+	private static Map<String, File> setup() throws Exception {
+		Map<String, File> configs = new LinkedHashMap<String, File>();
+		configs.put("Source", testTemplatesRootDir()); // source of test templates
+		configs.put("Java", javaGenDir());
+		configs.put("CSharp", csharpGenDir());
+		configs.put("Python2", python2GenDir());
+		configs.put("Python3", python3GenDir());
+		configs.put("NodeJS", nodeJSGenDir());
+		configs.put("Safari", safariGenDir());
+		configs.put("Firefox", firefoxGenDir());
+		configs.put("Chrome", chromeGenDir());
+		configs.put("Explorer", iexplorerGenDir());
+		return configs;
+	}
+
+	private static File javaGenDir() throws Exception {
+		return new File(antlrRoot+"/tool/test/org/antlr/v4/test/rt/java");
+	}
+
+	private static File csharpGenDir() {
+		return new File(antlrRoot+"/../antlr4-csharp/tool/test/org/antlr/v4/test/rt/csharp");
+	}
+
+	private static File python2GenDir() {
+		return new File(antlrRoot+"/../antlr4-python2/tool/test/org/antlr/v4/test/rt/py2");
+	}
+
+	private static File python3GenDir() {
+		return new File(antlrRoot+"/../antlr4-python3/tool/test/org/antlr/v4/test/rt/py3");
+	}
+
+	private static File nodeJSGenDir() {
+		return new File(antlrRoot+"/../antlr4-javascript/tool/test/org/antlr/v4/test/rt/js/node");
+	}
+
+	private static File safariGenDir() {
+		return new File(antlrRoot+"/../antlr4-javascript/tool/test/org/antlr/v4/test/rt/js/safari");
+	}
+
+	private static File firefoxGenDir() {
+		return new File(antlrRoot+"/../antlr4-javascript/tool/test/org/antlr/v4/test/rt/js/firefox");
+	}
+
+	private static File chromeGenDir() {
+		return new File(antlrRoot+"/../antlr4-javascript/tool/test/org/antlr/v4/test/rt/js/chrome");
+	}
+
+	private static File iexplorerGenDir() {
+		return new File(antlrRoot+"/../antlr4-javascript/tool/test/org/antlr/v4/test/rt/js/explorer");
+	}
+
+	private static File testTemplatesRootDir() throws Exception {
+		return new File(antlrRoot+"/tool/test/org/antlr/v4/test/rt/gen/grammars");
+	}
+
+	public static String escape(String s) {
+		return s==null ? null : s.replace("\\","\\\\").replace("\r", "\\r").replace("\n", "\\n").replace("\"","\\\"");
+	}
+
 	private void generateTests() throws Exception {
+		System.out.println("Generating runtime tests for "+target);
 		this.group = readTemplates();
 		Collection<JUnitTestFile> tests = buildTests();
 		for(JUnitTestFile test : tests) {

tool/test/org/antlr/v4/test/rt/gen/grammars/ParserErrors/SingleSetInsertionConsumption.st

@@ -1,3 +1,3 @@
 grammar <grammarName>;
-set: ('b'|'c') ;
+myset: ('b'|'c') ;
-a: 'a' set 'd' {System.out.println($set.stop);} ;
+a: 'a' myset 'd' {<writeln("$myset.stop")>} ;

tool/test/org/antlr/v4/test/rt/gen/grammars/ParserErrors/SingleTokenDeletionConsumption.st

@@ -1,3 +1,3 @@
 grammar <grammarName>;
-set: ('b'|'c') ;
+myset: ('b'|'c') ;
-a: 'a' set 'd' {System.out.println($set.stop);} ;
+a: 'a' myset 'd' {<writeln("$myset.stop")>} ;

tool/test/org/antlr/v4/test/rt/java/TestParserErrors.java

@@ -39,8 +39,8 @@ public class TestParserErrors extends BaseTest {
 	@Test
 	public void testSingleTokenDeletionConsumption() throws Exception {
 		String grammar = "grammar T;\n" +
-	                  "set: ('b'|'c') ;\n" +
+	                  "myset: ('b'|'c') ;\n" +
-	                  "a: 'a' set 'd' {System.out.println($set.stop);} ;";
+	                  "a: 'a' myset 'd' {System.out.println($myset.stop);} ;";
 		String found = execParser("T.g4", grammar, "TParser", "TLexer", "a", "aabd", false);
 		assertEquals("[@2,2:2='b',<1>,1:2]\n", found);
 		assertEquals("line 1:1 extraneous input 'a' expecting {'b', 'c'}\n", this.stderrDuringParse);
@@ -80,8 +80,8 @@ public class TestParserErrors extends BaseTest {
 	@Test
 	public void testSingleSetInsertionConsumption() throws Exception {
 		String grammar = "grammar T;\n" +
-	                  "set: ('b'|'c') ;\n" +
+	                  "myset: ('b'|'c') ;\n" +
-	                  "a: 'a' set 'd' {System.out.println($set.stop);} ;";
+	                  "a: 'a' myset 'd' {System.out.println($myset.stop);} ;";
 		String found = execParser("T.g4", grammar, "TParser", "TLexer", "a", "ad", false);
 		assertEquals("[@0,0:0='a',<3>,1:0]\n", found);
 		assertEquals("line 1:1 missing {'b', 'c'} at 'd'\n", this.stderrDuringParse);