Verify behavior of lexer tests, and correct errors

antlr4-testgen-maven-plugin/resources/org/antlr4/runtime/test/templates/LexerExec/CharSet.stg

@@ -4,14 +4,14 @@ Grammar ::= [
 	"L": {<grammar("L")>}
 ]
 
-Input() ::= "34<\n> 34"
+Input() ::= "34<\u000D><\n> 34"
 
 Output() ::= <<
 I
 I
 [@0,0:1='34',\<1>,1:0]
-[@1,4:5='34',\<1>,2:1]
-[@2,6:5='\<EOF>',\<-1>,2:3]<\n>
+[@1,5:6='34',\<1>,2:1]
+[@2,7:6='\<EOF>',\<-1>,2:3]<\n>
 >>
 
 Errors() ::= ""

antlr4-testgen-maven-plugin/resources/org/antlr4/runtime/test/templates/LexerExec/CharSetPlus.stg

@@ -4,14 +4,14 @@ Grammar ::= [
 	"L": {<grammar("L")>}
 ]
 
-Input() ::= "34<\n> 34"
+Input() ::= "34<\u000D><\n> 34"
 
 Output() ::= <<
 I
 I
 [@0,0:1='34',\<1>,1:0]
-[@1,4:5='34',\<1>,2:1]
-[@2,6:5='\<EOF>',\<-1>,2:3]<\n>
+[@1,5:6='34',\<1>,2:1]
+[@2,7:6='\<EOF>',\<-1>,2:3]<\n>
 >>
 
 Errors() ::= ""

antlr4-testgen-maven-plugin/resources/org/antlr4/runtime/test/templates/LexerExec/CharSetRange.stg

@@ -4,7 +4,7 @@ Grammar ::= [
 	"L": {<grammar("L")>}
 ]
 
-Input() ::= "34<\n> 34 a2 abc <\n>   "
+Input() ::= "34<\u000D> 34 a2 abc <\n>   "
 
 Output() ::= <<
 I
@@ -12,10 +12,10 @@ I
 ID
 ID
 [@0,0:1='34',\<1>,1:0]
-[@1,4:5='34',\<1>,2:1]
-[@2,7:8='a2',\<2>,2:4]
-[@3,10:12='abc',\<2>,2:7]
-[@4,18:17='\<EOF>',\<-1>,3:3]<\n>
+[@1,4:5='34',\<1>,1:4]
+[@2,7:8='a2',\<2>,1:7]
+[@3,10:12='abc',\<2>,1:10]
+[@4,18:17='\<EOF>',\<-1>,2:3]<\n>
 >>
 
 Errors() ::= ""

antlr4-testgen-maven-plugin/resources/org/antlr4/runtime/test/templates/LexerExec/CharSetWithMissingEndRange.stg

@@ -4,12 +4,12 @@ Grammar ::= [
 	"L": {<grammar("L")>}
 ]
 
-Input() ::= "00<\n>"
+Input() ::= "00<\u000D><\n>"
 
 Output() ::= <<
 I
 [@0,0:1='00',\<1>,1:0]
-[@1,3:2='\<EOF>',\<-1>,2:0]<\n>
+[@1,4:3='\<EOF>',\<-1>,2:0]<\n>
 >>
 
 Errors() ::= ""

antlr4-testgen-maven-plugin/resources/org/antlr4/runtime/test/templates/LexerExec/CharSetWithQuote2.stg

@@ -18,6 +18,6 @@ Errors() ::= ""
 
 grammar(grammarName) ::= <<
 lexer grammar <grammarName>;
-A : ["\\ab]+ {<writeln("\"A\"")>} ;
+A : ["\\\\ab]+ {<writeln("\"A\"")>} ;
 WS : [ \n\t]+ -> skip ;
 >>

antlr4-testgen-maven-plugin/resources/org/antlr4/runtime/test/templates/LexerExec/EOFByItself.stg

@@ -6,6 +6,7 @@ Grammar ::= [
 
 Input() ::= ""
 
+// must get DONE EOF
 Output() ::= <<
 [@0,0:-1='\<EOF>',\<1>,1:0]
 [@1,0:-1='\<EOF>',\<-1>,1:0]<\n>

antlr4-testgen-maven-plugin/resources/org/antlr4/runtime/test/templates/LexerExec/LargeLexer.stg

@@ -1,3 +1,8 @@
+/*
+ * This is a regression test for antlr/antlr4#76 "Serialized ATN strings
+ * should be split when longer than 2^16 bytes (class file limitation)"
+ * https://github.com/antlr/antlr4/issues/76
+ */
 TestType() ::= "Lexer"
 
 Grammar ::= [

antlr4-testgen-maven-plugin/resources/org/antlr4/runtime/test/templates/LexerExec/Parentheses.stg

@@ -1,3 +1,8 @@
+/*
+ * This is a regression test for antlr/antlr4#224: "Parentheses without
+ * quantifier in lexer rules have unclear effect".
+ * https://github.com/antlr/antlr4/issues/224
+ */
 TestType() ::= "Lexer"
 
 Grammar ::= [

antlr4-testgen-maven-plugin/resources/org/antlr4/runtime/test/templates/LexerExec/RecursiveLexerRuleRefWithWildcardPlus_1.stg

@@ -6,6 +6,7 @@ Grammar ::= [
 	"L": {<grammar("L")>}
 ]
 
+// stuff on end of comment matches another rule
 Input() ::= <<
 /* ick */
 /* /* */

antlr4-testgen-maven-plugin/resources/org/antlr4/runtime/test/templates/LexerExec/RecursiveLexerRuleRefWithWildcardPlus_2.stg

@@ -6,6 +6,7 @@ Grammar ::= [
 	"L": {<grammar("L")>}
 ]
 
+// stuff on end of comment doesn't match another rule
 Input() ::= <<
 /* ick */x
 /* /* */x

antlr4-testgen-maven-plugin/resources/org/antlr4/runtime/test/templates/LexerExec/RecursiveLexerRuleRefWithWildcardStar_1.stg

@@ -6,6 +6,7 @@ Grammar ::= [
 	"L": {<grammar("L")>}
 ]
 
+// stuff on end of comment matches another rule
 Input() ::= <<
 /* ick */
 /* /* */

antlr4-testgen-maven-plugin/resources/org/antlr4/runtime/test/templates/LexerExec/RecursiveLexerRuleRefWithWildcardStar_2.stg

@@ -6,6 +6,7 @@ Grammar ::= [
 	"L": {<grammar("L")>}
 ]
 
+// stuff on end of comment doesn't match another rule
 Input() ::= <<
 /* ick */x
 /* /* */x

antlr4-testgen-maven-plugin/resources/org/antlr4/runtime/test/templates/LexerExec/RefToRuleDoesNotSetTokenNorEmitAnother.stg

@@ -12,7 +12,7 @@ Output() ::= <<
 [@0,0:1='34',\<2>,1:0]
 [@1,3:5='-21',\<1>,1:3]
 [@2,7:7='3',\<2>,1:7]
-[@3,8:7='\<EOF>',\<-1>,1:8]<\n>
+[@3,8:7='\<EOF>',\<-1>,1:8]<\n><! EOF has no length so range is 8:7 not 8:8 !>
 >>
 
 Errors() ::= ""

antlr4-testgen-maven-plugin/resources/org/antlr4/runtime/test/templates/LexerExec/ZeroLengthToken.stg

@@ -2,6 +2,8 @@
  * This is a regression test for antlr/antlr4#687 "Empty zero-length tokens
  * cannot have lexer commands" and antlr/antlr4#688 "Lexer cannot match
  * zero-length tokens"
+ * https://github.com/antlr/antlr4/issues/687
+ * https://github.com/antlr/antlr4/issues/688
  */
 
 TestType() ::= "Lexer"