Merge pull request #995 from ericvergnaud/fix-maven-test-scope

Fix maven test scope

.travis.yml

@@ -1,4 +1,4 @@
-sudo: false
+sudo: true
 language: java
 script:
   - mvn install
@@ -6,3 +6,13 @@ jdk:
   - openjdk6
   - oraclejdk7
   - oraclejdk8
+before_install: 
+  - sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
+  - sudo add-apt-repository ppa:fkrull/deadsnakes -y
+  - sudo add-apt-repository ppa:rwky/nodejs -y
+  - sudo apt-get update -qq 
+  - sudo apt-get install -qq python3.4
+  - sudo apt-get install -qq nodejs
+  - echo "deb http://download.mono-project.com/repo/debian wheezy/snapshots/3.12.1 main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list
+  - sudo apt-get install -qq mono-complete
+  

runtime-testsuite/pom.xml

@@ -67,15 +67,21 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
+        <version>2.12.4</version>
         <configuration>
           <includes>
+            <include>**/csharp/Test*.java</include>
             <include>**/java/Test*.java</include>
+            <include>**/javascript/node/Test*.java</include>
+            <include>**/python2/Test*.java</include> 
+            <include>**/python3/Test*.java</include>
           </includes>
         </configuration>
       </plugin>    
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-jar-plugin</artifactId>
+        <version>2.4</version>
         <executions>
           <execution>
             <goals>
@@ -106,8 +112,8 @@
                   <mainClass>org.antlr.v4.testgen.TestGenerator</mainClass>
                   <arguments>
                     <argument>-root</argument>
-                    <argument>${basedir}/resources</argument>
-                    <argument>-o</argument>
+                    <argument>${basedir}</argument>
+                    <argument>-outdir</argument>
                     <argument>${basedir}/test</argument>
                     <argument>-templates</argument>
                     <argument>${basedir}/resources/org/antlr/v4/test/runtime/templates</argument>

runtime-testsuite/src/org/antlr/v4/testgen/TestGenerator.java

@@ -34,7 +34,6 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.OutputStreamWriter;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -46,18 +45,8 @@ import org.stringtemplate.v4.STGroupFile;
 import org.stringtemplate.v4.gui.STViz;
 
 public class TestGenerator {
-	public final static String[] targets = {"CSharp", "Java", "Python2", "Python3", "JavaScript"};
-
-	// This project uses UTF-8, but the plugin might be used in another project
-	// which is not. Always load templates with UTF-8, but write using the
-	// specified encoding.
-	protected final String encoding;
-
-	protected final File runtimeTemplates;
-
-	protected final File outputDirectory;
-
-	protected final boolean visualize;
+	
+	public final static String[] targets = {"CSharp", "Java", "Python2", "Python3", "JavaScript/Node", "JavaScript/Safari", "JavaScript/Firefox", "JavaScript/Explorer", "JavaScript/Chrome"};
 
 	/** Execute from antlr4 root dir:
 	 * *
@@ -71,7 +60,8 @@ public class TestGenerator {
 		String rootDir = null;
 		String outDir = null;
 		String templatesRoot = null;
-		String targetSpecificTemplateFile = null;
+		String target = "ALL";
+		boolean browsers = false;
 		boolean viz = false;
 
 		int i = 0;
@@ -81,14 +71,20 @@ public class TestGenerator {
 				i++;
 				rootDir = args[i];
 			}
-			else if (arg.startsWith("-o")) {
+			else if (arg.startsWith("-outdir")) {
 				i++;
 				outDir = args[i];
 			}
 			else if (arg.startsWith("-templates")) {
 				i++;
-				targetSpecificTemplateFile = args[i];
-				templatesRoot = targetSpecificTemplateFile;
+				templatesRoot = args[i];
+			}
+			else if (arg.startsWith("-target")) {
+				i++;
+				target = args[i];
+			}
+			else if (arg.startsWith("-browsers")) {
+				browsers = true;
 			}
 			else if (arg.startsWith("-viz")) {
 				viz = true;
@@ -98,115 +94,102 @@ public class TestGenerator {
 		
 		System.out.println("rootDir = " + rootDir);
 		System.out.println("outputDir = " + outDir);
-		System.out.println("templates = " + targetSpecificTemplateFile);
+		System.out.println("templates = " + templatesRoot);
+		System.out.println("target = " + target);
+		System.out.println("browsers = " + browsers);
+		System.out.println("viz = " + viz);
 		
-		if ( rootDir!=null) {
-			genAllTargets(outDir, rootDir, templatesRoot, viz);
+		if(rootDir==null) {
+			System.out.println("rootDir is mandatory!" + rootDir);
 			return;
 		}
-
-		if ( outDir==null || targetSpecificTemplateFile==null ) {
-			System.err.println("You must give an output root dir and templates file");
-			return;
-		}
-
-		genTarget(outDir, targetSpecificTemplateFile, templatesRoot, viz);
-	}
-
-	public static void genAllTargets(String outDirRoot, final String rootDir, String templatesRoot, boolean viz) {
+		if(outDir==null)
+			outDir = rootDir + "/test";
+			
 		if(templatesRoot==null)
-			templatesRoot = rootDir + "/runtime-testsuite/resources/org/antlr/v4/test/runtime/templates";
-		for (String target : targets) {
-			String templatesPackage = rootDir + "/runtime-testsuite/resources/org/antlr/v4/test/runtime/" + target.toLowerCase();
-			List<String> targetTemplates = Arrays.asList(templatesPackage + "/" + target + ".test.stg");
-			if ( target.equals("JavaScript") ) {
-				targetTemplates = new ArrayList<String>();
-				String[] subTargets = { "Node", "Safari", "Firefox", "Explorer", "Chrome" };
-				for(String subTarget : subTargets) {
-					String subTargetTemplate = templatesPackage + "/" + subTarget.toLowerCase() + "/" + subTarget + ".test.stg";
-					targetTemplates.add(subTargetTemplate);
-				}
-			}
-			String outDir = rootDir + "/runtime-testsuite/test";
-			if ( outDirRoot!=null ) {
-				outDir = outDirRoot;
-			}
-			for(String targetTemplate : targetTemplates)
-				genTarget(outDir, targetTemplate, templatesRoot, viz);
-		}
+			templatesRoot = rootDir + "/resources/org/antlr/v4/test/runtime/templates";
+
+		if ( "ALL".equalsIgnoreCase(target)) {
+			genAllTargets(rootDir, outDir, templatesRoot, browsers, viz);
+		} else
+			genTarget(rootDir, outDir, target, templatesRoot, viz);
 	}
 
-	public static void genTarget(final String outDir, final String targetSpecificTemplateFile, final String templates, boolean viz) {
-		TestGenerator gen = new TestGenerator("UTF-8",
-											  new File(targetSpecificTemplateFile),
-											  new File(outDir),
-											  viz)
-		{
-			@Override
-			protected void info(String message) {
-				System.err.println(message);
-			}
-			@Override
-			public File getOutputDir(String templateFolder) {
-				String targetName = getTargetNameFromTemplatesFileName();
-				// compute package
-				String templatePath = runtimeTemplates.getPath();
-				int packageStart = templatePath.indexOf("org/antlr/v4/test/runtime");
-				int packageEnd = templatePath.indexOf("/" + targetName + ".test.stg");
-				String packageDir = templatePath.substring(packageStart, packageEnd);
-				return new File(outputDirectory, packageDir);
-			}
-			@Override
-			public String getTestTemplatesResourceDir() {
-			  return templates;
-			  //return "resources/org/antlr/v4/test/runtime/templates";
-			}
-		};
-
-		// Somehow the templates directory is getting picked up so let's block that
-		if(!targetSpecificTemplateFile.endsWith(".stg")) {
-		    return;
+	public static void genAllTargets(String rootDir, String outDirRoot, String templatesRoot, boolean browsers, boolean viz) {
+		for(String target : targets) {
+			if(!browsers && "JavaScript/Safari".equals(target))
+				return;
+			genTarget(rootDir, outDirRoot, target, templatesRoot, viz);
 		}
-		
-		gen.info("Generating target " + gen.getTargetNameFromTemplatesFileName());
+	}
+	
+	public static void genTarget(final String rootDir, final String outDir, final String fullTarget, final String templatesDir, boolean viz) {
+		String[] parts = fullTarget.split("/");
+		String target = parts[0];
+		String subTarget = parts.length>1 ? parts[1] : target;
+		String targetPackage = rootDir + "/resources/org/antlr/v4/test/runtime/" + fullTarget.toLowerCase();
+		String targetTemplate = targetPackage + "/" + subTarget + ".test.stg";
+		TestGenerator gen = new TestGenerator("UTF-8",
+					fullTarget,
+					rootDir,
+					new File(outDir),
+					new File(templatesDir),
+					new File(targetTemplate),
+					viz);
+		gen.info("Generating target " + gen.getTargetName());
 		gen.execute();
 	}
 
-	public TestGenerator(String encoding, File runtimeTemplates, File outputDirectory, boolean visualize) {
+	// This project uses UTF-8, but the plugin might be used in another project
+	// which is not. Always load templates with UTF-8, but write using the
+	// specified encoding.
+	protected final String encoding;
+	protected final String targetName;
+	protected final String rootDir;
+	protected final File outputDir;
+	protected final File testTemplates;
+	protected final File runtimeTemplate;
+	protected final boolean visualize;
+
+	public TestGenerator(String encoding, String targetName, String rootDir, File outputDir, File testTemplates, File runtimeTemplate, boolean visualize) {
 		this.encoding = encoding;
-		this.runtimeTemplates = runtimeTemplates;
-		this.outputDirectory = outputDirectory;
+		this.targetName = targetName;
+		this.rootDir = rootDir;
+		this.outputDir = outputDir;
+		this.testTemplates = testTemplates;
+		this.runtimeTemplate = runtimeTemplate;
 		this.visualize = visualize;
 	}
 
+	private String getTargetName() {
+		return targetName;
+	}
+
+
 	public void execute() {
-		STGroup targetGroup = new STGroupFile(runtimeTemplates.getPath());
+		STGroup targetGroup = new STGroupFile(runtimeTemplate.getPath());
 		targetGroup.registerModelAdaptor(STGroup.class, new STGroupModelAdaptor());
 		targetGroup.defineDictionary("escape", new JavaEscapeStringMap());
 		targetGroup.defineDictionary("lines", new LinesStringMap());
 		targetGroup.defineDictionary("strlen", new StrlenStringMap());
-
-		String rootFolder = getTestTemplatesResourceDir();
-		generateCodeForFoldersInIndex(targetGroup, rootFolder);
+		generateCodeForFoldersInIndex(targetGroup);
 	}
 
-	protected void generateCodeForFoldersInIndex(STGroup targetGroup, String rootFolder) {
-		STGroup index = new STGroupFile(rootFolder+"/Index.stg");
+	protected void generateCodeForFoldersInIndex(STGroup targetGroup) {
+		File targetFolder = getOutputDir(testTemplates+"");
+		STGroup index = new STGroupFile(testTemplates+"/Index.stg");
 		index.load(); // make sure the index group is loaded since we call rawGetDictionary
-
 		Map<String, Object> folders = index.rawGetDictionary("TestFolders");
 		if (folders != null) {
 			for (String key : folders.keySet()) {
-				final String testdir = rootFolder + "/" + key;
-				STGroup testIndex = new STGroupFile(testdir + "/Index.stg");
+				final String testDir = testTemplates + "/" + key;
+				STGroup testIndex = new STGroupFile(testDir + "/Index.stg");
 				testIndex.load();
 				Map<String, Object> templateNames = testIndex.rawGetDictionary("TestTemplates");
 				if ( templateNames != null && !templateNames.isEmpty() ) {
 					final ArrayList<String> sortedTemplateNames = new ArrayList<String>(templateNames.keySet());
 					Collections.sort(sortedTemplateNames);
-					generateTestFile(testIndex, targetGroup,
-									 testdir,
-									 sortedTemplateNames);
+					generateTestFile(testIndex, targetGroup, testDir, sortedTemplateNames, targetFolder);
 				}
 			}
 		}
@@ -214,16 +197,16 @@ public class TestGenerator {
 
 	protected void generateTestFile(STGroup index,
 									STGroup targetGroup,
-									String testdir,
-									Collection<String> testTemplates)
+									String testDir,
+									Collection<String> testTemplates,
+									File targetFolder)
 	{
-		File targetFolder = getOutputDir(testdir);
-		String testName = testdir.substring(testdir.lastIndexOf('/') + 1);
+		String testName = testDir.substring(testDir.lastIndexOf('/') + 1);
 		File targetFile = new File(targetFolder, "Test" + testName + ".java");
 		info("Generating file "+targetFile.getAbsolutePath());
 		List<ST> templates = new ArrayList<ST>();
 		for (String template : testTemplates) {
-			STGroup testGroup = new STGroupFile(testdir + "/" + template + STGroup.GROUP_FILE_EXTENSION);
+			STGroup testGroup = new STGroupFile(testDir + "/" + template + STGroup.GROUP_FILE_EXTENSION);
 			importLanguageTemplates(testGroup, targetGroup);
 			ST testType = testGroup.getInstanceOf("TestType");
 			if (testType == null) {
@@ -296,19 +279,18 @@ public class TestGenerator {
 		}
 	}
 
-	public String getTestTemplatesResourceDir() { return "org/antlr/v4/test/runtime/templates"; }
-
-	public String getTargetNameFromTemplatesFileName() {
-		// runtimeTemplates is like ~/antlr/code/antlr4/runtime-testsuite/resources/org/antlr/v4/test/runtime/java/Java.test.stg
-		int targetEnd = runtimeTemplates.getName().indexOf(".test.stg");
-		return runtimeTemplates.getName().substring(0, targetEnd);
-	}
-
 	public File getOutputDir(String templateFolder) {
-		return new File(outputDirectory, templateFolder.substring(0, templateFolder.indexOf("/templates")));
+		if(templateFolder.startsWith(rootDir))
+			templateFolder = templateFolder.substring(rootDir.length());
+		if(templateFolder.startsWith("/resources"))
+			templateFolder = templateFolder.substring("/resources".length());
+		templateFolder = templateFolder.substring(0, templateFolder.indexOf("/templates"));
+		templateFolder += "/" + targetName.toLowerCase();
+		return new File(outputDir, templateFolder);
 	}
 
 	protected void info(String message) {
+		// System.out.println("INFO: " + message);
 	}
 
 	protected void warn(String message) {

runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseTest.java

@@ -289,7 +289,10 @@ public abstract class BaseTest {
 		writeFile(tmpdir, "input", input);
 		writeLexerTestFile(lexerName, showDFA);
 		addSourceFiles("Test.cs");
-		compile();
+		if(!compile()) {
+			System.err.println("Failed to compile!");
+			return stderrDuringParse;
+		}
 		String output = execTest();
 		if ( stderrDuringParse!=null && stderrDuringParse.length()>0 ) {
 			System.err.println(stderrDuringParse);
@@ -416,8 +419,10 @@ public abstract class BaseTest {
 				"/p:Configuration=Release",
 				getTestProjectFile().getAbsolutePath()
 			};
-		System.err.println("Starting build "+Utils.join(args, " "));
-		Process process = Runtime.getRuntime().exec(args, null, new File(tmpdir));
+		System.err.println("Starting build "+ Utils.join(args, " "));
+		ProcessBuilder pb = new ProcessBuilder(args);
+		pb.directory(new File(tmpdir));
+		Process process = pb.start();
 		StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream());
 		StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream());
 		stdoutVacuum.start();
@@ -425,11 +430,13 @@ public abstract class BaseTest {
 		process.waitFor();
 		stdoutVacuum.join();
 		stderrVacuum.join();
-		if ( stderrVacuum.toString().length()>0 ) {
-			this.stderrDuringParse = stderrVacuum.toString();
-			System.err.println("buildProject stderrVacuum: "+ stderrVacuum);
+		// xbuild sends errors to output, so check exit code
+		boolean success = process.exitValue()==0;
+		if ( !success ) {
+			this.stderrDuringParse = stdoutVacuum.toString();
+			System.err.println("buildProject stderrVacuum: "+ this.stderrDuringParse);
 		}
-		return process.exitValue()==0;
+		return success;
 	}
 
 	private String locateMSBuild() {
@@ -472,7 +479,7 @@ public abstract class BaseTest {
 			String runtimeName = isWindows() ? "Antlr4.Runtime.vs2013.csproj" : "Antlr4.Runtime.mono.csproj";
 			final URL runtimeProj = loader.getResource("CSharp/runtime/CSharp/Antlr4.Runtime/"+runtimeName);
 			if ( runtimeProj==null ) {
-				throw new RuntimeException("C# runtime project file not found at:" + runtimeProj.getPath());
+				throw new RuntimeException("C# runtime project file not found!");
 			}
 			String runtimeProjPath = runtimeProj.getPath();
 			XPathExpression exp = XPathFactory.newInstance().newXPath()
@@ -529,12 +536,12 @@ public abstract class BaseTest {
 					new String[] { "mono", exec, new File(tmpdir, "input").getAbsolutePath() };
 			ProcessBuilder pb = new ProcessBuilder(args);
 			pb.directory(new File(tmpdir));
-			Process p = pb.start();
-			StreamVacuum stdoutVacuum = new StreamVacuum(p.getInputStream());
-			StreamVacuum stderrVacuum = new StreamVacuum(p.getErrorStream());
+			Process process = pb.start();
+			StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream());
+			StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream());
 			stdoutVacuum.start();
 			stderrVacuum.start();
-			p.waitFor();
+			process.waitFor();
 			stdoutVacuum.join();
 			stderrVacuum.join();
 			String output = stdoutVacuum.toString();

runtime-testsuite/test/org/antlr/v4/test/runtime/python3/BasePython3Test.java

@@ -41,7 +41,7 @@ public abstract class BasePython3Test extends BasePythonTest {
 
 	@Override
 	protected String getPythonExecutable() {
-		return "python3.4";
+		return "python3.4"; 
 	}
 
 	@Override

tool-testsuite/pom.xml

@@ -76,6 +76,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
+        <version>2.12.4</version>
         <configuration>
           <systemPropertyVariables>
             <antlr-python2-runtime>../../antlr4-python2/src</antlr-python2-runtime>

tool/resources/org/antlr/v4/tool/templates/codegen/CSharp/CSharp.stg

@@ -931,6 +931,7 @@ namespace <file.genPackage> {
 <endif>
 <namedActions.header>
 using System;
+using System.Text;
 using Antlr4.Runtime;
 using Antlr4.Runtime.Atn;
 using Antlr4.Runtime.Misc;
@@ -986,8 +987,14 @@ public partial class <csIdentifier.(lexer.name)> : <superClass; null="Lexer"> {
 
 
 SerializedATN(model) ::= <<
-public static readonly string _serializedATN =
-	"<model.serialized; wrap={"+<\n><\t>"}>";
+private static string _serializedATN = _serializeATN();
+private static string _serializeATN()
+{
+    StringBuilder sb = new StringBuilder();
+    sb.Append("<model.serialized; wrap={");<\n><\t>sb.Append("}>");
+    return sb.ToString();
+}
+
 public static readonly ATN _ATN =
 	new ATNDeserializer().Deserialize(_serializedATN.ToCharArray());
 >>