prepare release

all tests pass
added test for complement set

CreatingTargets.md

@@ -1,72 +0,0 @@
-# Work in progress
-
-This document is a work in progress, and may or may not resemble the final
-documentation for creating a new language target (runtime+templates) for ANTLR 4.
-
-# Creating a runtime port
-
-## Development directory structure
-
-The directory structure for developing a new ANTLR 4 target does not have a required
-form. The following directory structure may be used to provide the same form as the
-ANTLR 4 reference code base. This form is especially recommended for developers who
-plan to port both the ANTLR 4 Tool *and* Runtime to a new language.
-
-	/
-	/runtime/[target]/
-	/tool/src/org/antlr/v4/codegen/[target]Target.java
-	/tool/resources/org/antlr/v4/tool/templates/codegen/[target]/[target].stg
-
-### Tracking progress against the reference repository
-
-To assist in updating your target as changes are made in the reference (Java) repository,
-you can include the reference repository as a submodule of the Git repository for your
-target. The submodule will also allow other users to see which commit your target is
-synchronized with. I included this submodule at the following location.
-
-	/reference/antlr4
-
-The following command will add the submodule to your working repository.
-
-	git submodule add -b "master" "git://github.com/antlr/antlr4.git" "reference/antlr4"
-
-*Note:* the C# target uses a special branch of ANTLR 4 as a reference. The branch "sharpen"
-is based on the "optimized" branch, but modified to operate as an input to the Sharpen tool
-for automatically converting Java code to C#. The submodule for this target was added
-using the following command.
-
-	git submodule add -b "sharpen" "git://github.com/sharwell/antlr4.git" "reference/antlr4"
-
-## Release structure
-
-### Code generation support
-
-The target must provide a `.jar` file for the ANTLR 4 Tool to use for code generation.
-The key file to include is the following:
-
-	/META-INF/services/org.antlr.v4.codegen.Target
-
-This file should contain the fully qualified name of the class extending `Target` for
-your runtime. This will likely look like the following.
-
-	org.antlr.v4.codegen.[target]Target
-
-If you are providing multiple code generation targets in a single `.jar` file, you
-should include one line for each target. The C# code generation target provides the
-following entries.
-
-	org.antlr.v4.codegen.CSharp2Target
-	org.antlr.v4.codegen.CSharp3Target
-	org.antlr.v4.codegen.CSharp4Target
-
-The code generation templates themselves are loaded by the runtime via the `TODO()`
-method. The default implementation (inherited from the `Target` superclass) will
-attempt to load the template from the following path in the `.jar` file.
-
-	/org/antlr/v4/tool/templates/codegen/[target]/[target].stg
-
-### Runtime
-
-The runtime for a new target may be distributed in any form relevant to that target
-language.
-

runtime/CSharp/Antlr4.Runtime/Atn/LexerATNSimulator.cs

@@ -195,8 +195,6 @@ namespace Antlr4.Runtime.Atn
             {
                 // allow zero-length tokens
                 CaptureSimState(prevAccept, input, ds0);
-                // adjust index since the current input character was not yet consumed
-                prevAccept.index--;
             }
             int t = input.La(1);
             DFAState s = ds0;
@@ -230,6 +228,14 @@ namespace Antlr4.Runtime.Atn
                 {
                     break;
                 }
+				// If this is a consumable input element, make sure to consume before
+				// capturing the accept state so the input index, line, and char
+				// position accurately reflect the state of the interpreter at the
+				// end of the token.
+				if (t != IntStreamConstants.Eof) {
+					Consume(input);
+				}
+
 				if (target.IsAcceptState)
                 {
                     CaptureSimState(prevAccept, input, target);
@@ -238,11 +244,7 @@ namespace Antlr4.Runtime.Atn
                         break;
                     }
                 }
-                if (t != IntStreamConstants.Eof)
-                {
-                    Consume(input);
                 t = input.La(1);
-                }
                 s = target;
             }
             // flip; current DFA target becomes new src/from state
@@ -398,10 +400,6 @@ namespace Antlr4.Runtime.Atn
             input.Seek(index);
             this._line = line;
             this.charPositionInLine = charPos;
-            if (input.La(1) != IntStreamConstants.Eof)
-            {
-                Consume(input);
-            }
             if (lexerActionExecutor != null && recog != null)
             {
                 lexerActionExecutor.Execute(recog, input, startIndex);

runtime/CSharp/Antlr4.Runtime/Lexer.cs

@@ -355,14 +355,8 @@ outer_continue: ;
         public virtual IToken EmitEOF()
         {
             int cpos = Column;
-            // The character position for EOF is one beyond the position of
+			int line = Line;
-            // the previous token's last character
+            IToken eof = _factory.Create(_tokenFactorySourcePair, TokenConstants.Eof, null, TokenConstants.DefaultChannel, _input.Index, _input.Index - 1, line, cpos);
-            if (_token != null)
-            {
-                int n = _token.StopIndex - _token.StartIndex + 1;
-                cpos = _token.Column + n;
-            }
-            IToken eof = _factory.Create(_tokenFactorySourcePair, TokenConstants.Eof, null, TokenConstants.DefaultChannel, _input.Index, _input.Index - 1, Line, cpos);
             Emit(eof);
             return eof;
         }

runtime/CSharp/Antlr4.mono.userprefs

@@ -1,6 +1,11 @@
 <Properties>
   <MonoDevelop.Ide.Workspace ActiveConfiguration="Debug" />
-  <MonoDevelop.Ide.Workbench />
+  <MonoDevelop.Ide.Workbench ActiveDocument="Antlr4.Runtime/Atn/LexerATNSimulator.cs">
+    <Files>
+      <File FileName="Antlr4.Runtime/Lexer.cs" Line="1" Column="1" />
+      <File FileName="Antlr4.Runtime/Atn/LexerATNSimulator.cs" Line="39" Column="39" />
+    </Files>
+  </MonoDevelop.Ide.Workbench>
   <MonoDevelop.Ide.DebuggingService.Breakpoints>
     <BreakpointStore>
       <Breakpoint file="/Users/ericvergnaud/Development/antlr4/antlr/antlr4-csharp/runtime/CSharp/Antlr4.Test.mono/TParser.cs" line="158" column="1" />

tool/test/org/antlr/v4/test/rt/csharp/BaseTest.java

@@ -30,6 +30,7 @@
 package org.antlr.v4.test.rt.csharp;
 
 import static org.junit.Assert.*;
+
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileOutputStream;
@@ -90,7 +91,7 @@ public abstract class BaseTest {
 	 * directories for all tests which completed successfully, and preserving
 	 * the directories for tests which failed.</p>
 	 */
-	public static final boolean PRESERVE_TEST_DIR = Boolean.parseBoolean(System.getProperty("antlr.preserve-csharp-test-dir"));
+	public static final boolean PRESERVE_TEST_DIR = Boolean.parseBoolean(System.getProperty("antlr-preserve-csharp-test-dir"));
 
 	/**
 	 * The base test directory is the directory where generated files get placed
@@ -116,7 +117,7 @@ public abstract class BaseTest {
 	public static final boolean CREATE_PER_TEST_DIRECTORIES;
 
 	static {
-		String baseTestDir = System.getProperty("antlr.csharp-test-dir");
+		String baseTestDir = System.getProperty("antlr-csharp-test-dir");
 		boolean perTestDirectories = false;
 		if (baseTestDir == null || baseTestDir.isEmpty()) {
 			baseTestDir = System.getProperty("java.io.tmpdir");
@@ -403,21 +404,41 @@ public abstract class BaseTest {
 		}
 	}
 	
-	private File getProjectFile() {
+	private File getTestProjectFile() {
 		return new File(tmpdir, "Antlr4.Test.mono.csproj");
 	}
 	
 	private boolean buildProject() throws Exception {
+		String msbuild = locateMSBuild();
 		String[] args = {
-				"xbuild",
+				msbuild,
 				"/p:Configuration=Release",
-				getProjectFile().getAbsolutePath()
+				getTestProjectFile().getAbsolutePath()
 			};
 		Process process = Runtime.getRuntime().exec(args, null, new File(tmpdir));
 		process.waitFor();
 		return process.exitValue()==0;
 	}
 
+	private String locateMSBuild() {
+		return locateTool("xbuild");
+	}
+	
+	private String locateExec() {
+		return locateTool("mono");
+		// new File(tmpdir, "bin/Release/Test.exe").getAbsolutePath(),
+
+	}
+
+	private String locateTool(String tool) {
+		String[] roots = { "/usr/bin/", "/usr/local/bin/" };
+		for(String root : roots) {
+			if(new File(root + tool).exists())
+				return root + tool;
+		}
+		throw new RuntimeException("Could not locate " + tool);
+	}
+
 	public boolean createProject() {
 		try {
 			String pack = this.getClass().getPackage().getName().replace(".", "/") + "/";
@@ -433,7 +454,13 @@ public abstract class BaseTest {
 			input = Thread.currentThread().getContextClassLoader().getResourceAsStream(pack + "Antlr4.Test.mono.csproj");
 			Document prjXml = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(input);
 			// update runtime project reference
-			String runtimePath = System.getProperty("antlr.csharp-runtime-project");
+			String runtimePath = System.getProperty("antlr-csharp-runtime-project");
+			if(runtimePath==null)
+				runtimePath = "../../antlr4-csharp/runtime/CSharp/Antlr4.Runtime/Antlr4.Runtime.mono.csproj";
+			File projFile = new File(runtimePath);
+			if(!projFile.exists())
+				throw new RuntimeException("C# runtime project file not found at:" + projFile.getAbsolutePath());
+			runtimePath = projFile.getAbsolutePath();
 			XPathExpression exp = XPathFactory.newInstance().newXPath().compile("/Project/ItemGroup/ProjectReference[@Include='Antlr4.Runtime.mono.csproj']");
 			Element node = (Element)exp.evaluate(prjXml, XPathConstants.NODE);
 			node.setAttribute("Include", runtimePath.replace("/", "\\"));
@@ -454,7 +481,7 @@ public abstract class BaseTest {
 				group.appendChild(elem);
 			}
 			// save project
-			File prjFile = getProjectFile();
+			File prjFile = getTestProjectFile();
 			Transformer transformer = TransformerFactory.newInstance().newTransformer();
 			transformer.setOutputProperty(OutputKeys.INDENT, "yes");
 			transformer.transform(new DOMSource(prjXml), new StreamResult(prjFile));
@@ -466,8 +493,9 @@ public abstract class BaseTest {
 
 	public String execTest() {
 		try {
+			String exec = locateExec();
 			String[] args = new String[] {
-				"mono", 
+					exec, 
 				new File(tmpdir, "bin/Release/Test.exe").getAbsolutePath(),
 				new File(tmpdir, "input").getAbsolutePath()
 			};

tool/test/org/antlr/v4/test/rt/csharp/CSharp.test.stg

@@ -25,7 +25,9 @@ public void test<test.name>() throws Exception {
 	mkdir(tmpdir);
 	writeFile(tmpdir, "<grammar.grammarName>.g4", slave_<grammar.grammarName>);
 	};separator="\n", wrap, anchor>
-	String grammar = <test.grammar.lines:{ line | "<line>};separator="\\n\" +\n", wrap, anchor>";
+	StringBuilder sb = new StringBuilder();
+	<test.grammar.lines:{ line | sb.append("<line>\\n");};separator="\n", wrap, anchor>
+	String grammar = sb.toString();
 	<test.afterGrammar>
 	String found = execLexer("<test.grammar.grammarName>.g4", grammar, "<test.grammar.grammarName><if(!test.lexerOnly)>Lexer<endif>", "<test.input>", <test.showDFA>);
 	assertEquals(<test.outputLines:{ line | "<line>\\n"};separator=" + \n", wrap, anchor>, found);

tool/test/org/antlr/v4/test/rt/csharp/TestCompositeLexers.java

@@ -12,10 +12,12 @@ public class TestCompositeLexers extends BaseTest {
 		mkdir(tmpdir);
 		writeFile(tmpdir, "S.g4", slave_S);
 
-		String grammar = "lexer grammar M;\n" +
+		StringBuilder sb = new StringBuilder();
-	                  "import S;\n" +
+		sb.append("lexer grammar M;\n");
-	                  "B : 'b';\n" +
+		sb.append("import S;\n");
-	                  "WS : (' '|'\\n') -> skip ;";
+		sb.append("B : 'b';\n");
+		sb.append("WS : (' '|'\\n') -> skip ;\n");
+		String grammar = sb.toString();
 		String found = execLexer("M.g4", grammar, "M", "abc", false);
 		assertEquals("S.A\n" + 
 	              "[@0,0:0='a',<3>,1:0]\n" + 
@@ -33,10 +35,12 @@ public class TestCompositeLexers extends BaseTest {
 		mkdir(tmpdir);
 		writeFile(tmpdir, "S.g4", slave_S);
 
-		String grammar = "lexer grammar M;\n" +
+		StringBuilder sb = new StringBuilder();
-	                  "import S;\n" +
+		sb.append("lexer grammar M;\n");
-	                  "A : 'a' B {Console.WriteLine(\"M.A\");};\n" +
+		sb.append("import S;\n");
-	                  "WS : (' '|'\\n') -> skip ;";
+		sb.append("A : 'a' B {Console.WriteLine(\"M.A\");};\n");
+		sb.append("WS : (' '|'\\n') -> skip ;\n");
+		String grammar = sb.toString();
 		String found = execLexer("M.g4", grammar, "M", "ab", false);
 		assertEquals("M.A\n" + 
 	              "[@0,0:1='ab',<1>,1:0]\n" + 

tool/test/org/antlr/v4/test/rt/csharp/TestLexerErrors.java

@@ -6,8 +6,10 @@ public class TestLexerErrors extends BaseTest {
 
 	@Test
 	public void testInvalidCharAtStart() throws Exception {
-		String grammar = "lexer grammar L;\n" +
+		StringBuilder sb = new StringBuilder();
-	                  "A : 'a' 'b' ;";
+		sb.append("lexer grammar L;\n");
+		sb.append("A : 'a' 'b' ;\n");
+		String grammar = sb.toString();
 		String found = execLexer("L.g4", grammar, "L", "x", false);
 		assertEquals("[@0,1:0='<EOF>',<-1>,1:1]\n", found);
 		assertEquals("line 1:0 token recognition error at: 'x'\n", this.stderrDuringParse);
@@ -15,10 +17,12 @@ public class TestLexerErrors extends BaseTest {
 
 	@Test
 	public void testStringsEmbeddedInActions_1() throws Exception {
-		String grammar = "lexer grammar L;\n" +
+		StringBuilder sb = new StringBuilder();
-	                  "ACTION2 : '[' (STRING | ~'\"')*? ']';\n" +
+		sb.append("lexer grammar L;\n");
-	                  "STRING : '\"' ('\\\"' | .)*? '\"';\n" +
+		sb.append("ACTION2 : '[' (STRING | ~'\"')*? ']';\n");
-	                  "WS : [ \\t\\r\\n]+ -> skip;";
+		sb.append("STRING : '\"' ('\\\"' | .)*? '\"';\n");
+		sb.append("WS : [ \\t\\r\\n]+ -> skip;\n");
+		String grammar = sb.toString();
 		String found = execLexer("L.g4", grammar, "L", "[\"foo\"]", false);
 		assertEquals("[@0,0:6='[\"foo\"]',<1>,1:0]\n" + 
 	              "[@1,7:6='<EOF>',<-1>,1:7]\n", found);
@@ -27,10 +31,12 @@ public class TestLexerErrors extends BaseTest {
 
 	@Test
 	public void testStringsEmbeddedInActions_2() throws Exception {
-		String grammar = "lexer grammar L;\n" +
+		StringBuilder sb = new StringBuilder();
-	                  "ACTION2 : '[' (STRING | ~'\"')*? ']';\n" +
+		sb.append("lexer grammar L;\n");
-	                  "STRING : '\"' ('\\\"' | .)*? '\"';\n" +
+		sb.append("ACTION2 : '[' (STRING | ~'\"')*? ']';\n");
-	                  "WS : [ \\t\\r\\n]+ -> skip;";
+		sb.append("STRING : '\"' ('\\\"' | .)*? '\"';\n");
+		sb.append("WS : [ \\t\\r\\n]+ -> skip;\n");
+		String grammar = sb.toString();
 		String found = execLexer("L.g4", grammar, "L", "[\"foo]", false);
 		assertEquals("[@0,6:5='<EOF>',<-1>,1:6]\n", found);
 		assertEquals("line 1:0 token recognition error at: '[\"foo]'\n", this.stderrDuringParse);
@@ -38,9 +44,11 @@ public class TestLexerErrors extends BaseTest {
 
 	@Test
 	public void testEnforcedGreedyNestedBrances_1() throws Exception {
-		String grammar = "lexer grammar L;\n" +
+		StringBuilder sb = new StringBuilder();
-	                  "ACTION : '{' (ACTION | ~[{}])* '}';\n" +
+		sb.append("lexer grammar L;\n");
-	                  "WS : [ \\r\\n\\t]+ -> skip;";
+		sb.append("ACTION : '{' (ACTION | ~[{}])* '}';\n");
+		sb.append("WS : [ \\r\\n\\t]+ -> skip;\n");
+		String grammar = sb.toString();
 		String found = execLexer("L.g4", grammar, "L", "{ { } }", false);
 		assertEquals("[@0,0:6='{ { } }',<1>,1:0]\n" + 
 	              "[@1,7:6='<EOF>',<-1>,1:7]\n", found);
@@ -49,9 +57,11 @@ public class TestLexerErrors extends BaseTest {
 
 	@Test
 	public void testEnforcedGreedyNestedBrances_2() throws Exception {
-		String grammar = "lexer grammar L;\n" +
+		StringBuilder sb = new StringBuilder();
-	                  "ACTION : '{' (ACTION | ~[{}])* '}';\n" +
+		sb.append("lexer grammar L;\n");
-	                  "WS : [ \\r\\n\\t]+ -> skip;";
+		sb.append("ACTION : '{' (ACTION | ~[{}])* '}';\n");
+		sb.append("WS : [ \\r\\n\\t]+ -> skip;\n");
+		String grammar = sb.toString();
 		String found = execLexer("L.g4", grammar, "L", "{ { }", false);
 		assertEquals("[@0,5:4='<EOF>',<-1>,1:5]\n", found);
 		assertEquals("line 1:0 token recognition error at: '{ { }'\n", this.stderrDuringParse);
@@ -59,8 +69,10 @@ public class TestLexerErrors extends BaseTest {
 
 	@Test
 	public void testInvalidCharAtStartAfterDFACache() throws Exception {
-		String grammar = "lexer grammar L;\n" +
+		StringBuilder sb = new StringBuilder();
-	                  "A : 'a' 'b' ;";
+		sb.append("lexer grammar L;\n");
+		sb.append("A : 'a' 'b' ;\n");
+		String grammar = sb.toString();
 		String found = execLexer("L.g4", grammar, "L", "abx", false);
 		assertEquals("[@0,0:1='ab',<1>,1:0]\n" + 
 	              "[@1,3:2='<EOF>',<-1>,1:3]\n", found);
@@ -69,8 +81,10 @@ public class TestLexerErrors extends BaseTest {
 
 	@Test
 	public void testInvalidCharInToken() throws Exception {
-		String grammar = "lexer grammar L;\n" +
+		StringBuilder sb = new StringBuilder();
-	                  "A : 'a' 'b' ;";
+		sb.append("lexer grammar L;\n");
+		sb.append("A : 'a' 'b' ;\n");
+		String grammar = sb.toString();
 		String found = execLexer("L.g4", grammar, "L", "ax", false);
 		assertEquals("[@0,2:1='<EOF>',<-1>,1:2]\n", found);
 		assertEquals("line 1:0 token recognition error at: 'ax'\n", this.stderrDuringParse);
@@ -78,8 +92,10 @@ public class TestLexerErrors extends BaseTest {
 
 	@Test
 	public void testInvalidCharInTokenAfterDFACache() throws Exception {
-		String grammar = "lexer grammar L;\n" +
+		StringBuilder sb = new StringBuilder();
-	                  "A : 'a' 'b' ;";
+		sb.append("lexer grammar L;\n");
+		sb.append("A : 'a' 'b' ;\n");
+		String grammar = sb.toString();
 		String found = execLexer("L.g4", grammar, "L", "abax", false);
 		assertEquals("[@0,0:1='ab',<1>,1:0]\n" + 
 	              "[@1,4:3='<EOF>',<-1>,1:4]\n", found);
@@ -88,9 +104,11 @@ public class TestLexerErrors extends BaseTest {
 
 	@Test
 	public void testDFAToATNThatFailsBackToDFA() throws Exception {
-		String grammar = "lexer grammar L;\n" +
+		StringBuilder sb = new StringBuilder();
-	                  "A : 'ab' ;\n" +
+		sb.append("lexer grammar L;\n");
-	                  "B : 'abc' ;";
+		sb.append("A : 'ab' ;\n");
+		sb.append("B : 'abc' ;\n");
+		String grammar = sb.toString();
 		String found = execLexer("L.g4", grammar, "L", "ababx", false);
 		assertEquals("[@0,0:1='ab',<1>,1:0]\n" + 
 	              "[@1,2:3='ab',<1>,1:2]\n" + 
@@ -100,10 +118,12 @@ public class TestLexerErrors extends BaseTest {
 
 	@Test
 	public void testDFAToATNThatMatchesThenFailsInATN() throws Exception {
-		String grammar = "lexer grammar L;\n" +
+		StringBuilder sb = new StringBuilder();
-	                  "A : 'ab' ;\n" +
+		sb.append("lexer grammar L;\n");
-	                  "B : 'abc' ;\n" +
+		sb.append("A : 'ab' ;\n");
-	                  "C : 'abcd' ;";
+		sb.append("B : 'abc' ;\n");
+		sb.append("C : 'abcd' ;\n");
+		String grammar = sb.toString();
 		String found = execLexer("L.g4", grammar, "L", "ababcx", false);
 		assertEquals("[@0,0:1='ab',<1>,1:0]\n" + 
 	              "[@1,2:4='abc',<2>,1:2]\n" + 
@@ -113,8 +133,10 @@ public class TestLexerErrors extends BaseTest {
 
 	@Test
 	public void testErrorInMiddle() throws Exception {
-		String grammar = "lexer grammar L;\n" +
+		StringBuilder sb = new StringBuilder();
-	                  "A : 'abc' ;";
+		sb.append("lexer grammar L;\n");
+		sb.append("A : 'abc' ;\n");
+		String grammar = sb.toString();
 		String found = execLexer("L.g4", grammar, "L", "abx", false);
 		assertEquals("[@0,3:2='<EOF>',<-1>,1:3]\n", found);
 		assertEquals("line 1:0 token recognition error at: 'abx'\n", this.stderrDuringParse);
@@ -122,11 +144,13 @@ public class TestLexerErrors extends BaseTest {
 
 	@Test
 	public void testLexerExecDFA() throws Exception {
-		String grammar = "grammar L;\n" +
+		StringBuilder sb = new StringBuilder();
-	                  "start : ID ':' expr;\n" +
+		sb.append("grammar L;\n");
-	                  "expr : primary expr? {} | expr '->' ID;\n" +
+		sb.append("start : ID ':' expr;\n");
-	                  "primary : ID;\n" +
+		sb.append("expr : primary expr? {} | expr '->' ID;\n");
-	                  "ID : [a-z]+;";
+		sb.append("primary : ID;\n");
+		sb.append("ID : [a-z]+;\n");
+		String grammar = sb.toString();
 		String found = execLexer("L.g4", grammar, "LLexer", "x : x", false);
 		assertEquals("[@0,0:0='x',<3>,1:0]\n" + 
 	              "[@1,2:2=':',<1>,1:2]\n" + 

tool/test/org/antlr/v4/test/rt/csharp/TestParserExec.java

@@ -415,28 +415,5 @@ public class TestParserExec extends BaseTest {
 		assertNull(this.stderrDuringParse);
 	}
 
-	@Test
-	public void testAlternateQuotes() throws Exception {
-		String slave_ModeTagsLexer = "lexer grammar ModeTagsLexer;\n" +
-	                              "// Default mode rules (the SEA)\n" +
-	                              "OPEN  : '«'     -> mode(ISLAND) ;       // switch to ISLAND mode\n" +
-	                              "TEXT  : ~'«'+ ;                         // clump all text together\n" +
-	                              "mode ISLAND;\n" +
-	                              "CLOSE : '»'     -> mode(DEFAULT_MODE) ; // back to SEA mode\n" +
-	                              "SLASH : '/' ;\n" +
-	                              "ID    : [a-zA-Z]+ ;                     // match/send ID in tag to parser";
-		rawGenerateRecognizer("ModeTagsLexer.g4", slave_ModeTagsLexer, null, "ModeTagsLexer");
-
-		String grammar = "parser grammar ModeTagsParser;\n" +
-	                  "options { tokenVocab=ModeTagsLexer; } // use tokens from ModeTagsLexer.g4\n" +
-	                  "file_: (tag | TEXT)* ;\n" +
-	                  "tag : '«' ID '»'\n" +
-	                  "    | '«' '/' ID '»'\n" +
-	                  "    ;";
-		String found = execParser("ModeTagsParser.g4", grammar, "ModeTagsParser", "ModeTagsLexer", "file_", "", false);
-		assertEquals("", found);
-		assertNull(this.stderrDuringParse);
-	}
-
 
 }

tool/test/org/antlr/v4/test/rt/csharp/TestSemPredEvalLexer.java

@@ -6,11 +6,13 @@ public class TestSemPredEvalLexer extends BaseTest {
 
 	@Test
 	public void testDisableRule() throws Exception {
-		String grammar = "lexer grammar L;\n" +
+		StringBuilder sb = new StringBuilder();
-	                  "E1 : 'enum' { false }? ;\n" +
+		sb.append("lexer grammar L;\n");
-	                  "E2 : 'enum' { true }? ;  // winner not E1 or ID\n" +
+		sb.append("E1 : 'enum' { false }? ;\n");
-	                  "ID : 'a'..'z'+ ;\n" +
+		sb.append("E2 : 'enum' { true }? ;  // winner not E1 or ID\n");
-	                  "WS : (' '|'\\n') -> skip;";
+		sb.append("ID : 'a'..'z'+ ;\n");
+		sb.append("WS : (' '|'\\n') -> skip;\n");
+		String grammar = sb.toString();
 		String found = execLexer("L.g4", grammar, "L", "enum abc", true);
 		assertEquals("[@0,0:3='enum',<2>,1:0]\n" + 
 	              "[@1,5:7='abc',<3>,1:5]\n" + 
@@ -27,10 +29,12 @@ public class TestSemPredEvalLexer extends BaseTest {
 
 	@Test
 	public void testIDvsEnum() throws Exception {
-		String grammar = "lexer grammar L;\n" +
+		StringBuilder sb = new StringBuilder();
-	                  "ENUM : 'enum' { false }? ;\n" +
+		sb.append("lexer grammar L;\n");
-	                  "ID : 'a'..'z'+ ;\n" +
+		sb.append("ENUM : 'enum' { false }? ;\n");
-	                  "WS : (' '|'\\n') -> skip;";
+		sb.append("ID : 'a'..'z'+ ;\n");
+		sb.append("WS : (' '|'\\n') -> skip;\n");
+		String grammar = sb.toString();
 		String found = execLexer("L.g4", grammar, "L", "enum abc enum", true);
 		assertEquals("[@0,0:3='enum',<2>,1:0]\n" + 
 	              "[@1,5:7='abc',<2>,1:5]\n" + 
@@ -48,10 +52,12 @@ public class TestSemPredEvalLexer extends BaseTest {
 
 	@Test
 	public void testIDnotEnum() throws Exception {
-		String grammar = "lexer grammar L;\n" +
+		StringBuilder sb = new StringBuilder();
-	                  "ENUM : [a-z]+  { false }? ;\n" +
+		sb.append("lexer grammar L;\n");
-	                  "ID : [a-z]+  ;\n" +
+		sb.append("ENUM : [a-z]+  { false }? ;\n");
-	                  "WS : (' '|'\\n') -> skip;";
+		sb.append("ID : [a-z]+  ;\n");
+		sb.append("WS : (' '|'\\n') -> skip;\n");
+		String grammar = sb.toString();
 		String found = execLexer("L.g4", grammar, "L", "enum abc enum", true);
 		assertEquals("[@0,0:3='enum',<2>,1:0]\n" + 
 	              "[@1,5:7='abc',<2>,1:5]\n" + 
@@ -63,10 +69,12 @@ public class TestSemPredEvalLexer extends BaseTest {
 
 	@Test
 	public void testEnumNotID() throws Exception {
-		String grammar = "lexer grammar L;\n" +
+		StringBuilder sb = new StringBuilder();
-	                  "ENUM : [a-z]+  { this.Text.Equals(\"enum\") }? ;\n" +
+		sb.append("lexer grammar L;\n");
-	                  "ID : [a-z]+  ;\n" +
+		sb.append("ENUM : [a-z]+  { this.Text.Equals(\"enum\") }? ;\n");
-	                  "WS : (' '|'\\n') -> skip;";
+		sb.append("ID : [a-z]+  ;\n");
+		sb.append("WS : (' '|'\\n') -> skip;\n");
+		String grammar = sb.toString();
 		String found = execLexer("L.g4", grammar, "L", "enum abc enum", true);
 		assertEquals("[@0,0:3='enum',<1>,1:0]\n" + 
 	              "[@1,5:7='abc',<2>,1:5]\n" + 
@@ -78,12 +86,14 @@ public class TestSemPredEvalLexer extends BaseTest {
 
 	@Test
 	public void testIndent() throws Exception {
-		String grammar = "lexer grammar L;\n" +
+		StringBuilder sb = new StringBuilder();
-	                  "ID : [a-z]+  ;\n" +
+		sb.append("lexer grammar L;\n");
-	                  "INDENT : [ \\t]+ { this.TokenStartColumn==0 }?\n" +
+		sb.append("ID : [a-z]+  ;\n");
-	                  "         { Console.WriteLine(\"INDENT\"); }  ;\n" +
+		sb.append("INDENT : [ \\t]+ { this.TokenStartColumn==0 }?\n");
-	                  "NL : '\\n';\n" +
+		sb.append("         { Console.WriteLine(\"INDENT\"); }  ;\n");
-	                  "WS : [ \\t]+ ;";
+		sb.append("NL : '\\n';\n");
+		sb.append("WS : [ \\t]+ ;\n");
+		String grammar = sb.toString();
 		String found = execLexer("L.g4", grammar, "L", "abc\n  def  \n", true);
 		assertEquals("INDENT\n" + 
 	              "[@0,0:2='abc',<1>,1:0]\n" + 
@@ -92,7 +102,7 @@ public class TestSemPredEvalLexer extends BaseTest {
 	              "[@3,6:8='def',<1>,2:2]\n" + 
 	              "[@4,9:10='  ',<4>,2:5]\n" + 
 	              "[@5,11:11='\\n',<3>,2:7]\n" + 
-	              "[@6,12:11='<EOF>',<-1>,3:8]\n" + 
+	              "[@6,12:11='<EOF>',<-1>,3:0]\n" + 
 	              "s0-'\n" + 
 	              "'->:s2=>3\n" + 
 	              "s0-'a'->:s1=>1\n" + 
@@ -106,12 +116,14 @@ public class TestSemPredEvalLexer extends BaseTest {
 
 	@Test
 	public void testLexerInputPositionSensitivePredicates() throws Exception {
-		String grammar = "lexer grammar L;\n" +
+		StringBuilder sb = new StringBuilder();
-	                  "WORD1 : ID1+ { Console.WriteLine(this.Text); } ;\n" +
+		sb.append("lexer grammar L;\n");
-	                  "WORD2 : ID2+ { Console.WriteLine(this.Text); } ;\n" +
+		sb.append("WORD1 : ID1+ { Console.WriteLine(this.Text); } ;\n");
-	                  "fragment ID1 : { this.Column < 2 }? [a-zA-Z];\n" +
+		sb.append("WORD2 : ID2+ { Console.WriteLine(this.Text); } ;\n");
-	                  "fragment ID2 : { this.Column >= 2 }? [a-zA-Z];\n" +
+		sb.append("fragment ID1 : { this.Column < 2 }? [a-zA-Z];\n");
-	                  "WS : (' '|'\\n') -> skip;";
+		sb.append("fragment ID2 : { this.Column >= 2 }? [a-zA-Z];\n");
+		sb.append("WS : (' '|'\\n') -> skip;\n");
+		String grammar = sb.toString();
 		String found = execLexer("L.g4", grammar, "L", "a cde\nabcde\n", true);
 		assertEquals("a\n" + 
 	              "cde\n" + 
@@ -127,10 +139,12 @@ public class TestSemPredEvalLexer extends BaseTest {
 
 	@Test
 	public void testPredicatedKeywords() throws Exception {
-		String grammar = "lexer grammar L;\n" +
+		StringBuilder sb = new StringBuilder();
-	                  "ENUM : [a-z]+ { this.Text.Equals(\"enum\") }? { Console.WriteLine(\"enum!\"); } ;\n" +
+		sb.append("lexer grammar L;\n");
-	                  "ID   : [a-z]+ { Console.WriteLine(\"ID \" + this.Text); } ;\n" +
+		sb.append("ENUM : [a-z]+ { this.Text.Equals(\"enum\") }? { Console.WriteLine(\"enum!\"); } ;\n");
-	                  "WS   : [ \\n] -> skip ;";
+		sb.append("ID   : [a-z]+ { Console.WriteLine(\"ID \" + this.Text); } ;\n");
+		sb.append("WS   : [ \\n] -> skip ;\n");
+		String grammar = sb.toString();
 		String found = execLexer("L.g4", grammar, "L", "enum enu a", false);
 		assertEquals("enum!\n" + 
 	              "ID enu\n" + 

tool/test/org/antlr/v4/test/rt/csharp/TestSemPredEvalParser.java

@@ -411,7 +411,7 @@ public class TestSemPredEvalParser extends BaseTest {
 	public void testPredFromAltTestedInLoopBack_1() throws Exception {
 		String found = testPredFromAltTestedInLoopBack("s\n\n\nx\n");
 		assertEquals("(file_ (para (paraContent s) \\n \\n) (para (paraContent \\n x \\n)) <EOF>)\n", found);
-		assertEquals("line 5:2 mismatched input '<EOF>' expecting '\n'\n", this.stderrDuringParse);
+		assertEquals("line 5:0 mismatched input '<EOF>' expecting '\n'\n", this.stderrDuringParse);
 	}
 
 	@Test

tool/test/org/antlr/v4/test/rt/csharp/TestSets.java

@@ -222,5 +222,15 @@ public class TestSets extends BaseTest {
 		assertNull(this.stderrDuringParse);
 	}
 
+	@Test
+	public void testComplementSet() throws Exception {
+		String grammar = "grammar T;\n" +
+	                  "parse : ~NEW_LINE;\n" +
+	                  "NEW_LINE: '\\r'? '\\n';";
+		String found = execParser("T.g4", grammar, "TParser", "TLexer", "parse", "a", false);
+		assertEquals("", found);
+		assertEquals("line 1:0 token recognition error at: 'a'\nline 1:1 missing {} at '<EOF>'\n", this.stderrDuringParse);
+	}
+
 
 }