forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

prepare release 

all tests pass
added test for complement set
This commit is contained in:
ericvergnaud 2014-11-23 02:02:22 +08:00
parent bfb819e045
commit 0f557737da
13 changed files with 4520 additions and 4460 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
CreatingTargets.md
View File

@ -1,72 +0,0 @@
# Work in progress
This document is a work in progress, and may or may not resemble the final
documentation for creating a new language target (runtime+templates) for ANTLR 4.
# Creating a runtime port
## Development directory structure
The directory structure for developing a new ANTLR 4 target does not have a required
form. The following directory structure may be used to provide the same form as the
ANTLR 4 reference code base. This form is especially recommended for developers who
plan to port both the ANTLR 4 Tool *and* Runtime to a new language.
/
/runtime/[target]/
/tool/src/org/antlr/v4/codegen/[target]Target.java
/tool/resources/org/antlr/v4/tool/templates/codegen/[target]/[target].stg
### Tracking progress against the reference repository
To assist in updating your target as changes are made in the reference (Java) repository,
you can include the reference repository as a submodule of the Git repository for your
target. The submodule will also allow other users to see which commit your target is
synchronized with. I included this submodule at the following location.
/reference/antlr4
The following command will add the submodule to your working repository.
git submodule add -b "master" "git://github.com/antlr/antlr4.git" "reference/antlr4"
*Note:* the C# target uses a special branch of ANTLR 4 as a reference. The branch "sharpen"
is based on the "optimized" branch, but modified to operate as an input to the Sharpen tool
for automatically converting Java code to C#. The submodule for this target was added
using the following command.
git submodule add -b "sharpen" "git://github.com/sharwell/antlr4.git" "reference/antlr4"
## Release structure
### Code generation support
The target must provide a `.jar` file for the ANTLR 4 Tool to use for code generation.
The key file to include is the following:
/META-INF/services/org.antlr.v4.codegen.Target
This file should contain the fully qualified name of the class extending `Target` for
your runtime. This will likely look like the following.
org.antlr.v4.codegen.[target]Target
If you are providing multiple code generation targets in a single `.jar` file, you
should include one line for each target. The C# code generation target provides the
following entries.
org.antlr.v4.codegen.CSharp2Target
org.antlr.v4.codegen.CSharp3Target
org.antlr.v4.codegen.CSharp4Target
The code generation templates themselves are loaded by the runtime via the `TODO()`
method. The default implementation (inherited from the `Target` superclass) will
attempt to load the template from the following path in the `.jar` file.
/org/antlr/v4/tool/templates/codegen/[target]/[target].stg
### Runtime
The runtime for a new target may be distributed in any form relevant to that target
language.

22
runtime/CSharp/Antlr4.Runtime/Atn/LexerATNSimulator.cs
View File

@ -195,8 +195,6 @@ namespace Antlr4.Runtime.Atn
{
// allow zero-length tokens
CaptureSimState(prevAccept, input, ds0);
// adjust index since the current input character was not yet consumed
prevAccept.index--;
}
int t = input.La(1);
DFAState s = ds0;
@ -230,7 +228,15 @@ namespace Antlr4.Runtime.Atn
{
break;
}
if (target.IsAcceptState)
// If this is a consumable input element, make sure to consume before
// capturing the accept state so the input index, line, and char
// position accurately reflect the state of the interpreter at the
// end of the token.
if (t != IntStreamConstants.Eof) {
Consume(input);
}
if (target.IsAcceptState)
{
CaptureSimState(prevAccept, input, target);
if (t == IntStreamConstants.Eof)
@ -238,11 +244,7 @@ namespace Antlr4.Runtime.Atn
break;
}
}
if (t != IntStreamConstants.Eof)
{
Consume(input);
t = input.La(1);
}
t = input.La(1);
s = target;
}
// flip; current DFA target becomes new src/from state
@ -398,10 +400,6 @@ namespace Antlr4.Runtime.Atn
input.Seek(index);
this._line = line;
this.charPositionInLine = charPos;
if (input.La(1) != IntStreamConstants.Eof)
{
Consume(input);
}
if (lexerActionExecutor != null && recog != null)
{
lexerActionExecutor.Execute(recog, input, startIndex);

10
runtime/CSharp/Antlr4.Runtime/Lexer.cs
View File

@ -355,14 +355,8 @@ outer_continue: ;
public virtual IToken EmitEOF()
{
int cpos = Column;
// The character position for EOF is one beyond the position of
// the previous token's last character
if (_token != null)
{
int n = _token.StopIndex - _token.StartIndex + 1;
cpos = _token.Column + n;
}
IToken eof = _factory.Create(_tokenFactorySourcePair, TokenConstants.Eof, null, TokenConstants.DefaultChannel, _input.Index, _input.Index - 1, Line, cpos);
int line = Line;
IToken eof = _factory.Create(_tokenFactorySourcePair, TokenConstants.Eof, null, TokenConstants.DefaultChannel, _input.Index, _input.Index - 1, line, cpos);
Emit(eof);
return eof;
}

7
runtime/CSharp/Antlr4.mono.userprefs
View File

@ -1,6 +1,11 @@
<Properties>
<MonoDevelop.Ide.Workspace ActiveConfiguration="Debug" />
<MonoDevelop.Ide.Workbench />
<MonoDevelop.Ide.Workbench ActiveDocument="Antlr4.Runtime/Atn/LexerATNSimulator.cs">
<Files>
<File FileName="Antlr4.Runtime/Lexer.cs" Line="1" Column="1" />
<File FileName="Antlr4.Runtime/Atn/LexerATNSimulator.cs" Line="39" Column="39" />
</Files>
</MonoDevelop.Ide.Workbench>
<MonoDevelop.Ide.DebuggingService.Breakpoints>
<BreakpointStore>
<Breakpoint file="/Users/ericvergnaud/Development/antlr4/antlr/antlr4-csharp/runtime/CSharp/Antlr4.Test.mono/TParser.cs" line="158" column="1" />

44
tool/test/org/antlr/v4/test/rt/csharp/BaseTest.java
View File

@ -30,6 +30,7 @@
package org.antlr.v4.test.rt.csharp;
import static org.junit.Assert.*;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
@ -90,7 +91,7 @@ public abstract class BaseTest {
* directories for all tests which completed successfully, and preserving
* the directories for tests which failed.</p>
*/
public static final boolean PRESERVE_TEST_DIR = Boolean.parseBoolean(System.getProperty("antlr.preserve-csharp-test-dir"));
public static final boolean PRESERVE_TEST_DIR = Boolean.parseBoolean(System.getProperty("antlr-preserve-csharp-test-dir"));
/**
* The base test directory is the directory where generated files get placed
@ -116,7 +117,7 @@ public abstract class BaseTest {
public static final boolean CREATE_PER_TEST_DIRECTORIES;
static {
String baseTestDir = System.getProperty("antlr.csharp-test-dir");
String baseTestDir = System.getProperty("antlr-csharp-test-dir");
boolean perTestDirectories = false;
if (baseTestDir == null || baseTestDir.isEmpty()) {
baseTestDir = System.getProperty("java.io.tmpdir");
@ -403,21 +404,41 @@ public abstract class BaseTest {
}
}
private File getProjectFile() {
private File getTestProjectFile() {
return new File(tmpdir, "Antlr4.Test.mono.csproj");
}
private boolean buildProject() throws Exception {
String msbuild = locateMSBuild();
String[] args = {
"xbuild",
msbuild,
"/p:Configuration=Release",
getProjectFile().getAbsolutePath()
getTestProjectFile().getAbsolutePath()
};
Process process = Runtime.getRuntime().exec(args, null, new File(tmpdir));
process.waitFor();
return process.exitValue()==0;
}
private String locateMSBuild() {
return locateTool("xbuild");
}
private String locateExec() {
return locateTool("mono");
// new File(tmpdir, "bin/Release/Test.exe").getAbsolutePath(),
}
private String locateTool(String tool) {
String[] roots = { "/usr/bin/", "/usr/local/bin/" };
for(String root : roots) {
if(new File(root + tool).exists())
return root + tool;
}
throw new RuntimeException("Could not locate " + tool);
}
public boolean createProject() {
try {
String pack = this.getClass().getPackage().getName().replace(".", "/") + "/";
@ -433,7 +454,13 @@ public abstract class BaseTest {
input = Thread.currentThread().getContextClassLoader().getResourceAsStream(pack + "Antlr4.Test.mono.csproj");
Document prjXml = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(input);
// update runtime project reference
String runtimePath = System.getProperty("antlr.csharp-runtime-project");
String runtimePath = System.getProperty("antlr-csharp-runtime-project");
if(runtimePath==null)
runtimePath = "../../antlr4-csharp/runtime/CSharp/Antlr4.Runtime/Antlr4.Runtime.mono.csproj";
File projFile = new File(runtimePath);
if(!projFile.exists())
throw new RuntimeException("C# runtime project file not found at:" + projFile.getAbsolutePath());
runtimePath = projFile.getAbsolutePath();
XPathExpression exp = XPathFactory.newInstance().newXPath().compile("/Project/ItemGroup/ProjectReference[@Include='Antlr4.Runtime.mono.csproj']");
Element node = (Element)exp.evaluate(prjXml, XPathConstants.NODE);
node.setAttribute("Include", runtimePath.replace("/", "\\"));
@ -454,7 +481,7 @@ public abstract class BaseTest {
group.appendChild(elem);
}
// save project
File prjFile = getProjectFile();
File prjFile = getTestProjectFile();
Transformer transformer = TransformerFactory.newInstance().newTransformer();
transformer.setOutputProperty(OutputKeys.INDENT, "yes");
transformer.transform(new DOMSource(prjXml), new StreamResult(prjFile));
@ -466,8 +493,9 @@ public abstract class BaseTest {
public String execTest() {
try {
String exec = locateExec();
String[] args = new String[] {
"mono",
exec,
new File(tmpdir, "bin/Release/Test.exe").getAbsolutePath(),
new File(tmpdir, "input").getAbsolutePath()
};

4
tool/test/org/antlr/v4/test/rt/csharp/CSharp.test.stg
View File

@ -25,7 +25,9 @@ public void test<test.name>() throws Exception {
mkdir(tmpdir);
writeFile(tmpdir, "<grammar.grammarName>.g4", slave_<grammar.grammarName>);
};separator="\n", wrap, anchor>
String grammar = <test.grammar.lines:{ line | "<line>};separator="\\n\" +\n", wrap, anchor>";
StringBuilder sb = new StringBuilder();
<test.grammar.lines:{ line | sb.append("<line>\\n");};separator="\n", wrap, anchor>
String grammar = sb.toString();
<test.afterGrammar>
String found = execLexer("<test.grammar.grammarName>.g4", grammar, "<test.grammar.grammarName><if(!test.lexerOnly)>Lexer<endif>", "<test.input>", <test.showDFA>);
assertEquals(<test.outputLines:{ line | "<line>\\n"};separator=" + \n", wrap, anchor>, found);

20
tool/test/org/antlr/v4/test/rt/csharp/TestCompositeLexers.java
View File

@ -12,10 +12,12 @@ public class TestCompositeLexers extends BaseTest {
mkdir(tmpdir);
writeFile(tmpdir, "S.g4", slave_S);
String grammar = "lexer grammar M;\n" +
"import S;\n" +
"B : 'b';\n" +
"WS : (' '|'\\n') -> skip ;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar M;\n");
sb.append("import S;\n");
sb.append("B : 'b';\n");
sb.append("WS : (' '|'\\n') -> skip ;\n");
String grammar = sb.toString();
String found = execLexer("M.g4", grammar, "M", "abc", false);
assertEquals("S.A\n" +
"[@0,0:0='a',<3>,1:0]\n" +
@ -33,10 +35,12 @@ public class TestCompositeLexers extends BaseTest {
mkdir(tmpdir);
writeFile(tmpdir, "S.g4", slave_S);
String grammar = "lexer grammar M;\n" +
"import S;\n" +
"A : 'a' B {Console.WriteLine(\"M.A\");};\n" +
"WS : (' '|'\\n') -> skip ;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar M;\n");
sb.append("import S;\n");
sb.append("A : 'a' B {Console.WriteLine(\"M.A\");};\n");
sb.append("WS : (' '|'\\n') -> skip ;\n");
String grammar = sb.toString();
String found = execLexer("M.g4", grammar, "M", "ab", false);
assertEquals("M.A\n" +
"[@0,0:1='ab',<1>,1:0]\n" +

96
tool/test/org/antlr/v4/test/rt/csharp/TestLexerErrors.java
View File

@ -6,8 +6,10 @@ public class TestLexerErrors extends BaseTest {
@Test
public void testInvalidCharAtStart() throws Exception {
String grammar = "lexer grammar L;\n" +
"A : 'a' 'b' ;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("A : 'a' 'b' ;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "x", false);
assertEquals("[@0,1:0='<EOF>',<-1>,1:1]\n", found);
assertEquals("line 1:0 token recognition error at: 'x'\n", this.stderrDuringParse);
@ -15,10 +17,12 @@ public class TestLexerErrors extends BaseTest {
@Test
public void testStringsEmbeddedInActions_1() throws Exception {
String grammar = "lexer grammar L;\n" +
"ACTION2 : '[' (STRING | ~'\"')*? ']';\n" +
"STRING : '\"' ('\\\"' | .)*? '\"';\n" +
"WS : [ \\t\\r\\n]+ -> skip;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("ACTION2 : '[' (STRING | ~'\"')*? ']';\n");
sb.append("STRING : '\"' ('\\\"' | .)*? '\"';\n");
sb.append("WS : [ \\t\\r\\n]+ -> skip;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "[\"foo\"]", false);
assertEquals("[@0,0:6='[\"foo\"]',<1>,1:0]\n" +
"[@1,7:6='<EOF>',<-1>,1:7]\n", found);
@ -27,10 +31,12 @@ public class TestLexerErrors extends BaseTest {
@Test
public void testStringsEmbeddedInActions_2() throws Exception {
String grammar = "lexer grammar L;\n" +
"ACTION2 : '[' (STRING | ~'\"')*? ']';\n" +
"STRING : '\"' ('\\\"' | .)*? '\"';\n" +
"WS : [ \\t\\r\\n]+ -> skip;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("ACTION2 : '[' (STRING | ~'\"')*? ']';\n");
sb.append("STRING : '\"' ('\\\"' | .)*? '\"';\n");
sb.append("WS : [ \\t\\r\\n]+ -> skip;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "[\"foo]", false);
assertEquals("[@0,6:5='<EOF>',<-1>,1:6]\n", found);
assertEquals("line 1:0 token recognition error at: '[\"foo]'\n", this.stderrDuringParse);
@ -38,9 +44,11 @@ public class TestLexerErrors extends BaseTest {
@Test
public void testEnforcedGreedyNestedBrances_1() throws Exception {
String grammar = "lexer grammar L;\n" +
"ACTION : '{' (ACTION | ~[{}])* '}';\n" +
"WS : [ \\r\\n\\t]+ -> skip;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("ACTION : '{' (ACTION | ~[{}])* '}';\n");
sb.append("WS : [ \\r\\n\\t]+ -> skip;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "{ { } }", false);
assertEquals("[@0,0:6='{ { } }',<1>,1:0]\n" +
"[@1,7:6='<EOF>',<-1>,1:7]\n", found);
@ -49,9 +57,11 @@ public class TestLexerErrors extends BaseTest {
@Test
public void testEnforcedGreedyNestedBrances_2() throws Exception {
String grammar = "lexer grammar L;\n" +
"ACTION : '{' (ACTION | ~[{}])* '}';\n" +
"WS : [ \\r\\n\\t]+ -> skip;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("ACTION : '{' (ACTION | ~[{}])* '}';\n");
sb.append("WS : [ \\r\\n\\t]+ -> skip;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "{ { }", false);
assertEquals("[@0,5:4='<EOF>',<-1>,1:5]\n", found);
assertEquals("line 1:0 token recognition error at: '{ { }'\n", this.stderrDuringParse);
@ -59,8 +69,10 @@ public class TestLexerErrors extends BaseTest {
@Test
public void testInvalidCharAtStartAfterDFACache() throws Exception {
String grammar = "lexer grammar L;\n" +
"A : 'a' 'b' ;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("A : 'a' 'b' ;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "abx", false);
assertEquals("[@0,0:1='ab',<1>,1:0]\n" +
"[@1,3:2='<EOF>',<-1>,1:3]\n", found);
@ -69,8 +81,10 @@ public class TestLexerErrors extends BaseTest {
@Test
public void testInvalidCharInToken() throws Exception {
String grammar = "lexer grammar L;\n" +
"A : 'a' 'b' ;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("A : 'a' 'b' ;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "ax", false);
assertEquals("[@0,2:1='<EOF>',<-1>,1:2]\n", found);
assertEquals("line 1:0 token recognition error at: 'ax'\n", this.stderrDuringParse);
@ -78,8 +92,10 @@ public class TestLexerErrors extends BaseTest {
@Test
public void testInvalidCharInTokenAfterDFACache() throws Exception {
String grammar = "lexer grammar L;\n" +
"A : 'a' 'b' ;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("A : 'a' 'b' ;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "abax", false);
assertEquals("[@0,0:1='ab',<1>,1:0]\n" +
"[@1,4:3='<EOF>',<-1>,1:4]\n", found);
@ -88,9 +104,11 @@ public class TestLexerErrors extends BaseTest {
@Test
public void testDFAToATNThatFailsBackToDFA() throws Exception {
String grammar = "lexer grammar L;\n" +
"A : 'ab' ;\n" +
"B : 'abc' ;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("A : 'ab' ;\n");
sb.append("B : 'abc' ;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "ababx", false);
assertEquals("[@0,0:1='ab',<1>,1:0]\n" +
"[@1,2:3='ab',<1>,1:2]\n" +
@ -100,10 +118,12 @@ public class TestLexerErrors extends BaseTest {
@Test
public void testDFAToATNThatMatchesThenFailsInATN() throws Exception {
String grammar = "lexer grammar L;\n" +
"A : 'ab' ;\n" +
"B : 'abc' ;\n" +
"C : 'abcd' ;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("A : 'ab' ;\n");
sb.append("B : 'abc' ;\n");
sb.append("C : 'abcd' ;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "ababcx", false);
assertEquals("[@0,0:1='ab',<1>,1:0]\n" +
"[@1,2:4='abc',<2>,1:2]\n" +
@ -113,8 +133,10 @@ public class TestLexerErrors extends BaseTest {
@Test
public void testErrorInMiddle() throws Exception {
String grammar = "lexer grammar L;\n" +
"A : 'abc' ;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("A : 'abc' ;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "abx", false);
assertEquals("[@0,3:2='<EOF>',<-1>,1:3]\n", found);
assertEquals("line 1:0 token recognition error at: 'abx'\n", this.stderrDuringParse);
@ -122,11 +144,13 @@ public class TestLexerErrors extends BaseTest {
@Test
public void testLexerExecDFA() throws Exception {
String grammar = "grammar L;\n" +
"start : ID ':' expr;\n" +
"expr : primary expr? {} | expr '->' ID;\n" +
"primary : ID;\n" +
"ID : [a-z]+;";
StringBuilder sb = new StringBuilder();
sb.append("grammar L;\n");
sb.append("start : ID ':' expr;\n");
sb.append("expr : primary expr? {} | expr '->' ID;\n");
sb.append("primary : ID;\n");
sb.append("ID : [a-z]+;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "LLexer", "x : x", false);
assertEquals("[@0,0:0='x',<3>,1:0]\n" +
"[@1,2:2=':',<1>,1:2]\n" +

8588
tool/test/org/antlr/v4/test/rt/csharp/TestLexerExec.java
View File

File diff suppressed because it is too large Load Diff

23
tool/test/org/antlr/v4/test/rt/csharp/TestParserExec.java
View File

@ -415,28 +415,5 @@ public class TestParserExec extends BaseTest {
assertNull(this.stderrDuringParse);
}
@Test
public void testAlternateQuotes() throws Exception {
String slave_ModeTagsLexer = "lexer grammar ModeTagsLexer;\n" +
"// Default mode rules (the SEA)\n" +
"OPEN : '«' -> mode(ISLAND) ; // switch to ISLAND mode\n" +
"TEXT : ~'«'+ ; // clump all text together\n" +
"mode ISLAND;\n" +
"CLOSE : '»' -> mode(DEFAULT_MODE) ; // back to SEA mode\n" +
"SLASH : '/' ;\n" +
"ID : [a-zA-Z]+ ; // match/send ID in tag to parser";
rawGenerateRecognizer("ModeTagsLexer.g4", slave_ModeTagsLexer, null, "ModeTagsLexer");
String grammar = "parser grammar ModeTagsParser;\n" +
"options { tokenVocab=ModeTagsLexer; } // use tokens from ModeTagsLexer.g4\n" +
"file_: (tag | TEXT)* ;\n" +
"tag : '«' ID '»'\n" +
" | '«' '/' ID '»'\n" +
" ;";
String found = execParser("ModeTagsParser.g4", grammar, "ModeTagsParser", "ModeTagsLexer", "file_", "", false);
assertEquals("", found);
assertNull(this.stderrDuringParse);
}
}

82
tool/test/org/antlr/v4/test/rt/csharp/TestSemPredEvalLexer.java
View File

@ -6,11 +6,13 @@ public class TestSemPredEvalLexer extends BaseTest {
@Test
public void testDisableRule() throws Exception {
String grammar = "lexer grammar L;\n" +
"E1 : 'enum' { false }? ;\n" +
"E2 : 'enum' { true }? ; // winner not E1 or ID\n" +
"ID : 'a'..'z'+ ;\n" +
"WS : (' '|'\\n') -> skip;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("E1 : 'enum' { false }? ;\n");
sb.append("E2 : 'enum' { true }? ; // winner not E1 or ID\n");
sb.append("ID : 'a'..'z'+ ;\n");
sb.append("WS : (' '|'\\n') -> skip;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "enum abc", true);
assertEquals("[@0,0:3='enum',<2>,1:0]\n" +
"[@1,5:7='abc',<3>,1:5]\n" +
@ -27,10 +29,12 @@ public class TestSemPredEvalLexer extends BaseTest {
@Test
public void testIDvsEnum() throws Exception {
String grammar = "lexer grammar L;\n" +
"ENUM : 'enum' { false }? ;\n" +
"ID : 'a'..'z'+ ;\n" +
"WS : (' '|'\\n') -> skip;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("ENUM : 'enum' { false }? ;\n");
sb.append("ID : 'a'..'z'+ ;\n");
sb.append("WS : (' '|'\\n') -> skip;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "enum abc enum", true);
assertEquals("[@0,0:3='enum',<2>,1:0]\n" +
"[@1,5:7='abc',<2>,1:5]\n" +
@ -48,10 +52,12 @@ public class TestSemPredEvalLexer extends BaseTest {
@Test
public void testIDnotEnum() throws Exception {
String grammar = "lexer grammar L;\n" +
"ENUM : [a-z]+ { false }? ;\n" +
"ID : [a-z]+ ;\n" +
"WS : (' '|'\\n') -> skip;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("ENUM : [a-z]+ { false }? ;\n");
sb.append("ID : [a-z]+ ;\n");
sb.append("WS : (' '|'\\n') -> skip;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "enum abc enum", true);
assertEquals("[@0,0:3='enum',<2>,1:0]\n" +
"[@1,5:7='abc',<2>,1:5]\n" +
@ -63,10 +69,12 @@ public class TestSemPredEvalLexer extends BaseTest {
@Test
public void testEnumNotID() throws Exception {
String grammar = "lexer grammar L;\n" +
"ENUM : [a-z]+ { this.Text.Equals(\"enum\") }? ;\n" +
"ID : [a-z]+ ;\n" +
"WS : (' '|'\\n') -> skip;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("ENUM : [a-z]+ { this.Text.Equals(\"enum\") }? ;\n");
sb.append("ID : [a-z]+ ;\n");
sb.append("WS : (' '|'\\n') -> skip;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "enum abc enum", true);
assertEquals("[@0,0:3='enum',<1>,1:0]\n" +
"[@1,5:7='abc',<2>,1:5]\n" +
@ -78,12 +86,14 @@ public class TestSemPredEvalLexer extends BaseTest {
@Test
public void testIndent() throws Exception {
String grammar = "lexer grammar L;\n" +
"ID : [a-z]+ ;\n" +
"INDENT : [ \\t]+ { this.TokenStartColumn==0 }?\n" +
" { Console.WriteLine(\"INDENT\"); } ;\n" +
"NL : '\\n';\n" +
"WS : [ \\t]+ ;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("ID : [a-z]+ ;\n");
sb.append("INDENT : [ \\t]+ { this.TokenStartColumn==0 }?\n");
sb.append(" { Console.WriteLine(\"INDENT\"); } ;\n");
sb.append("NL : '\\n';\n");
sb.append("WS : [ \\t]+ ;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "abc\n def \n", true);
assertEquals("INDENT\n" +
"[@0,0:2='abc',<1>,1:0]\n" +
@ -92,7 +102,7 @@ public class TestSemPredEvalLexer extends BaseTest {
"[@3,6:8='def',<1>,2:2]\n" +
"[@4,9:10=' ',<4>,2:5]\n" +
"[@5,11:11='\\n',<3>,2:7]\n" +
"[@6,12:11='<EOF>',<-1>,3:8]\n" +
"[@6,12:11='<EOF>',<-1>,3:0]\n" +
"s0-'\n" +
"'->:s2=>3\n" +
"s0-'a'->:s1=>1\n" +
@ -106,12 +116,14 @@ public class TestSemPredEvalLexer extends BaseTest {
@Test
public void testLexerInputPositionSensitivePredicates() throws Exception {
String grammar = "lexer grammar L;\n" +
"WORD1 : ID1+ { Console.WriteLine(this.Text); } ;\n" +
"WORD2 : ID2+ { Console.WriteLine(this.Text); } ;\n" +
"fragment ID1 : { this.Column < 2 }? [a-zA-Z];\n" +
"fragment ID2 : { this.Column >= 2 }? [a-zA-Z];\n" +
"WS : (' '|'\\n') -> skip;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("WORD1 : ID1+ { Console.WriteLine(this.Text); } ;\n");
sb.append("WORD2 : ID2+ { Console.WriteLine(this.Text); } ;\n");
sb.append("fragment ID1 : { this.Column < 2 }? [a-zA-Z];\n");
sb.append("fragment ID2 : { this.Column >= 2 }? [a-zA-Z];\n");
sb.append("WS : (' '|'\\n') -> skip;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "a cde\nabcde\n", true);
assertEquals("a\n" +
"cde\n" +
@ -127,10 +139,12 @@ public class TestSemPredEvalLexer extends BaseTest {
@Test
public void testPredicatedKeywords() throws Exception {
String grammar = "lexer grammar L;\n" +
"ENUM : [a-z]+ { this.Text.Equals(\"enum\") }? { Console.WriteLine(\"enum!\"); } ;\n" +
"ID : [a-z]+ { Console.WriteLine(\"ID \" + this.Text); } ;\n" +
"WS : [ \\n] -> skip ;";
StringBuilder sb = new StringBuilder();
sb.append("lexer grammar L;\n");
sb.append("ENUM : [a-z]+ { this.Text.Equals(\"enum\") }? { Console.WriteLine(\"enum!\"); } ;\n");
sb.append("ID : [a-z]+ { Console.WriteLine(\"ID \" + this.Text); } ;\n");
sb.append("WS : [ \\n] -> skip ;\n");
String grammar = sb.toString();
String found = execLexer("L.g4", grammar, "L", "enum enu a", false);
assertEquals("enum!\n" +
"ID enu\n" +

2
tool/test/org/antlr/v4/test/rt/csharp/TestSemPredEvalParser.java
View File

@ -411,7 +411,7 @@ public class TestSemPredEvalParser extends BaseTest {
public void testPredFromAltTestedInLoopBack_1() throws Exception {
String found = testPredFromAltTestedInLoopBack("s\n\n\nx\n");
assertEquals("(file_ (para (paraContent s) \\n \\n) (para (paraContent \\n x \\n)) <EOF>)\n", found);
assertEquals("line 5:2 mismatched input '<EOF>' expecting '\n'\n", this.stderrDuringParse);
assertEquals("line 5:0 mismatched input '<EOF>' expecting '\n'\n", this.stderrDuringParse);
}
@Test

10
tool/test/org/antlr/v4/test/rt/csharp/TestSets.java
View File

@ -222,5 +222,15 @@ public class TestSets extends BaseTest {
assertNull(this.stderrDuringParse);
}
@Test
public void testComplementSet() throws Exception {
String grammar = "grammar T;\n" +
"parse : ~NEW_LINE;\n" +
"NEW_LINE: '\\r'? '\\n';";
String found = execParser("T.g4", grammar, "TParser", "TLexer", "parse", "a", false);
assertEquals("", found);
assertEquals("line 1:0 token recognition error at: 'a'\nline 1:1 missing {} at '<EOF>'\n", this.stderrDuringParse);
}
}