Merge pull request #893 from sharwell/test-generation

Completed test generation review

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/CompositeLexers/LexerDelegatorRuleOverridesDelegate.stg

@@ -21,12 +21,12 @@ Errors() ::= ""
 masterGrammar(grammarName, slaveGrammarName) ::= <<
 lexer grammar <grammarName>;
 import <slaveGrammarName>;
-A : 'a' B {<writeln("\"M.A\"")>};
+A : 'a' B {<writeln("\"M.A\"")>} ;
 WS : (' '|'\n') -> skip ;
 >>
 
 slaveGrammar(grammarName) ::= <<
 lexer grammar <grammarName>;
-A : 'a' {<writeln("\"S.A\"")>};
+A : 'a' {<writeln("\"S.A\"")>} ;
-B : 'b' {<writeln("\"S.B\"")>};
+B : 'b' {<writeln("\"S.B\"")>} ;
 >>

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/CompositeParsers/DelegatesSeeSameTokenType.stg

@@ -60,7 +60,7 @@ x : A {<writeln("\"S.x\"")>};
 >>
 
 slaveGrammarT(grammarName) ::= <<
-parser grammar S;
+parser grammar T;
 tokens { C, B, A } // reverse order
 y : A {<writeln("\"T.y\"")>};
 >>

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/CompositeParsers/DelegatorAccessesDelegateMembers.stg

@@ -21,13 +21,13 @@ Errors() ::= ""
 masterGrammar(grammarName, slaveGrammarName) ::= <<
 grammar M; // uses no rules from the import
 import S;
-s : 'b'{<Invoke_foo()>}; // gS is import pointer
+s : 'b' {<Invoke_foo()>} ; // gS is import pointer
 WS : (' '|'\n') -> skip ;
 >>
 
 slaveGrammar(grammarName) ::= <<
 parser grammar S;
-@members {
+@parser::members {
 <Declare_foo()>
 }
 a : B;

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/CompositeParsers/DelegatorInvokesDelegateRuleWithArgs.stg

@@ -28,5 +28,5 @@ WS : (' '|'\n') -> skip ;
 
 slaveGrammar(grammarName) ::= <<
 parser grammar S;
-a[int x] returns [int y] : B {<write("\"S.a\"")>;$y=1000;};
+a[int x] returns [int y] : B {<write("\"S.a\"")> $y=1000;} ;
 >>

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/CompositeParsers/DelegatorInvokesDelegateRuleWithReturnStruct.stg

@@ -28,5 +28,5 @@ WS : (' '|'\n') -> skip ;
 
 slaveGrammar(grammarName) ::= <<
 parser grammar S;
-a : B {<write("\"S.a\"")>};
+a : B {<write("\"S.a\"")>} ;
 >>

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/CompositeParsers/DelegatorInvokesFirstVersionOfDelegateRule.stg

@@ -29,11 +29,11 @@ WS : (' '|'\n') -> skip ;
 
 slaveGrammarS(grammarName) ::= <<
 parser grammar S;
-a : B {<writeln("\"S.a\"")>};
+a : b {<writeln("\"S.a\"")>};
 b : B;
 >>
 
 slaveGrammarT(grammarName) ::= <<
 parser grammar T;
-a : B {<writeln("\"T.a\"")>};
+a : B {<writeln("\"T.a\"")>};<! hidden by S.a !>
 >>

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/CompositeParsers/DelegatorRuleOverridesDelegates.stg

@@ -34,7 +34,7 @@ b : 'b' ;
 >>
 
 slaveGrammarT(grammarName) ::= <<
-parser grammar S;
+parser grammar T;
 tokens { A }
 b : 'b' {<writeln("\"T.b\"")>};
 >>

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/CompositeParsers/DelegatorRuleOverridesLookaheadInDelegate.stg

@@ -13,16 +13,17 @@ Rule() ::= "prog"
 Input() ::= "float x = 3;"
 
 Output() ::= <<
-Decl: floatx=3;<\n>
+JavaDecl: floatx=3;<\n>
 >>
 
 Errors() ::= ""
 
+// for float to work in decl, type must be overridden
 masterGrammar(grammarName, slaveGrammarName) ::= <<
 grammar M;
 import S;
 prog : decl ;
-type_ : 'int' | 'float' ;
+type : 'int' | 'float' ;
 ID  : 'a'..'z'+ ;
 INT : '0'..'9'+ ;
 WS : (' '|'\n') -> skip;
@@ -30,8 +31,8 @@ WS : (' '|'\n') -> skip;
 
 slaveGrammar(grammarName) ::= <<
 parser grammar S;
-type_ : 'int' ;
+type : 'int' ;
-decl : type_ ID ';'
+decl : type ID ';'
-	| type_ ID init ';' {<write("\"Decl: \" + $text")>};
+	| type ID init ';' {<write("\"JavaDecl: \" + $text")>};
 init : '=' INT;
 >>

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/CompositeParsers/ImportLexerWithOnlyFragmentRules.stg

@@ -1,11 +1,17 @@
+/*
+ * This is a regression test for antlr/antlr4#248 "Including grammar with only
+ * fragments breaks generated lexer".
+ * https://github.com/antlr/antlr4/issues/248
+ */
+
 TestType() ::= "CompositeParser"
 
 Grammar ::= [
-	"M": {<masterGrammar("M", "S")>}
+	"Test": {<masterGrammar("Test", "Unicode")>}
 ]
 
 SlaveGrammars ::= [
-	"S": {<slaveGrammar("S")>}
+	"Unicode": {<slaveGrammar("Unicode")>}
 ]
 
 Rule() ::= "program"
@@ -18,17 +24,22 @@ Output() ::= <<
 Errors() ::= ""
 
 masterGrammar(grammarName, slaveGrammarName) ::= <<
-grammar M;
+grammar <grammarName>;
-import S;
+import <slaveGrammarName; separator=", ">;
+
 program : 'test' 'test';
+
 WS : (UNICODE_CLASS_Zs)+ -> skip;
+
 >>
 
 slaveGrammar(grammarName) ::= <<
-lexer grammar S;
+lexer grammar <grammarName>;
+
 fragment
 UNICODE_CLASS_Zs    : '\u0020' | '\u00A0' | '\u1680' | '\u180E'
                     | '\u2000'..'\u200A'
                     | '\u202F' | '\u205F' | '\u3000'
                     ;
+
 >>

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/CompositeParsers/ImportedGrammarWithEmptyOptions.stg

@@ -20,13 +20,13 @@ Errors() ::= ""
 masterGrammar(grammarName, slaveGrammarName) ::= <<
 grammar M;
 import S;
-s : a;
+s : a ;
-B : 'b';
+B : 'b' ;
 WS : (' '|'\n') -> skip ;
 >>
 
 slaveGrammar(grammarName) ::= <<
 parser grammar S;
 options {}
-a : B;
+a : B ;
 >>

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/CompositeParsers/ImportedRuleWithAction.stg

@@ -25,7 +25,8 @@ B : 'b';
 WS : (' '|'\n') -> skip ;
 >>
 
+// wasn't terminating. @after was injected into M as if it were @members
 slaveGrammar(grammarName) ::= <<
 parser grammar S;
-a @after {} : B;
+a @after {int x;} : B;
 >>

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/CompositeParsers/KeywordVSIDOrder.stg

@@ -1,3 +1,7 @@
+// rules in lexer are imported at END so rules in master override
+// *and* get priority over imported rules. So importing ID doesn't
+// mess up keywords in master grammar
+
 TestType() ::= "CompositeParser"
 
 Grammar ::= [

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/FullContextParsing/FullContextIF_THEN_ELSEParse_2.stg

@@ -7,6 +7,13 @@ Decision 1:
 s0-'else'->:s1^=>1<\n>
 >>
 
+// Technically, this input sequence is not ambiguous because else
+// uniquely predicts going into the optional subrule. else cannot
+// be matched by exiting stat since that would only match '}' or
+// the start of a stat. But, we are using the theory that
+// SLL(1)=LL(1) and so we are avoiding full context parsing
+// by declaring all else clause parsing to be ambiguous.
+
 Errors() ::= <<
 line 1:19 reportAttemptingFullContext d=1 (stat), input='else'
 line 1:19 reportContextSensitivity d=1 (stat), input='else'<\n>

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/FullContextParsing/LoopsSimulateTailRecursion.stg

@@ -1,3 +1,7 @@
+/*
+ * Tests predictions for the following case involving closures.
+ * http://www.antlr.org/wiki/display/~admin/2011/12/29/Flaw+in+ANTLR+v3+LL(*)+analysis+algorithm
+ */
 TestType() ::= "Parser"
 
 Options ::= [

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/AmbigLR.stg

@@ -1,7 +1,7 @@
 TestType() ::= "Parser"
 
 Options ::= [
-	"Debug": false
+	"Debug": true
 ]
 
 Grammar ::= [

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/DirectCallToLeftRecursiveRule.stg

@@ -1,4 +1,4 @@
-/**
+/*
  * This is a regression test for "Support direct calls to left-recursive
  * rules".
  * https://github.com/antlr/antlr4/issues/161

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/JavaExpressions.stg

@@ -21,14 +21,14 @@ e   :   '(' e ')'
     |   'super'
     |   INT
     |   ID
-    |   type_ '.' 'class'
+    |   type '.' 'class'
     |   e '.' ID
     |   e '.' 'this'
     |   e '.' 'super' '(' expressionList? ')'
     |   e '.' 'new' ID '(' expressionList? ')'
-	 |	 'new' type_ ( '(' expressionList? ')' | ('[' e ']')+)
+	 |	 'new' type ( '(' expressionList? ')' | ('[' e ']')+)
     |   e '[' e ']'
-    |   '(' type_ ')' e
+    |   '(' type ')' e
     |   e ('++' | '--')
     |   e '(' expressionList? ')'
     |   ('+'|'-'|'++'|'--') e
@@ -59,7 +59,7 @@ e   :   '(' e ')'
           |'\<\<='
           |'%=') e
     ;
-type_: ID 
+type: ID 
     | ID '[' ']'
     | 'int'
 	 | 'int' '[' ']' 

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/JavaExpressions_12.stg

@@ -3,7 +3,7 @@ import "JavaExpressions.stg"
 Input() ::= "new T[((n-1) * x) + 1]"
 
 Output() ::= <<
-(s (e new (type_ T) [ (e (e ( (e (e ( (e (e n) - (e 1)) )) * (e x)) )) + (e 1)) ]) \<EOF>)<\n>
+(s (e new (type T) [ (e (e ( (e (e ( (e (e n) - (e 1)) )) * (e x)) )) + (e 1)) ]) \<EOF>)<\n>
 >>
 
 Errors() ::= ""

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/JavaExpressions_7.stg

@@ -3,7 +3,7 @@ import "JavaExpressions.stg"
 Input() ::= "(T)x"
 
 Output() ::= <<
-(s (e ( (type_ T) ) (e x)) \<EOF>)<\n>
+(s (e ( (type T) ) (e x)) \<EOF>)<\n>
 >>
 
 Errors() ::= ""

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/JavaExpressions_8.stg

@@ -3,7 +3,7 @@ import "JavaExpressions.stg"
 Input() ::= "new A().b"
 
 Output() ::= <<
-(s (e (e new (type_ A) ( )) . b) \<EOF>)<\n>
+(s (e (e new (type A) ( )) . b) \<EOF>)<\n>
 >>
 
 Errors() ::= ""

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/JavaExpressions_9.stg

@@ -3,7 +3,7 @@ import "JavaExpressions.stg"
 Input() ::= "(T)t.f()"
 
 Output() ::= <<
-(s (e (e ( (type_ T) ) (e (e t) . f)) ( )) \<EOF>)<\n>
+(s (e (e ( (type T) ) (e (e t) . f)) ( )) \<EOF>)<\n>
 >>
 
 Errors() ::= ""

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/MultipleActions.stg

@@ -1,4 +1,4 @@
-/**
+/*
  * This is a regression test for antlr/antlr4#625 "Duplicate action breaks
  * operator precedence"
  * https://github.com/antlr/antlr4/issues/625

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/MultipleActionsPredicatesOptions.stg

@@ -1,4 +1,4 @@
-/**
+/*
  * This is a regression test for antlr/antlr4#625 "Duplicate action breaks
  * operator precedence"
  * https://github.com/antlr/antlr4/issues/625

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/MultipleAlternativesWithCommonLabel.stg

@@ -1,4 +1,4 @@
-/**
+/*
  * This is a regression test for antlr/antlr4#433 "Not all context accessor
  * methods are generated when an alternative rule label is used for multiple
  * alternatives".
@@ -25,8 +25,8 @@ e returns [int v]
   | e '+' e     {$v = <Cast("BinaryContext","$ctx")>.e(0).v + <Cast("BinaryContext","$ctx")>.e(1).v;}  # binary
   | INT         {$v = $INT.int;}                   # anInt
   | '(' e ')'   {$v = $e.v;}                       # parens
-  | left=e INC  {<Cast("UnaryContext","$ctx"):Concat(".INC() != null"):Assert()>$v = $left.v + 1;}      # unary
+  | left=e INC  {<Cast("UnaryContext","$ctx"):Concat(".INC() != null"):Assert()> $v = $left.v + 1;}      # unary
-  | left=e DEC  {<Cast("UnaryContext","$ctx"):Concat(".DEC() != null"):Assert()>$v = $left.v - 1;}      # unary
+  | left=e DEC  {<Cast("UnaryContext","$ctx"):Concat(".DEC() != null"):Assert()> $v = $left.v - 1;}      # unary
   | ID          {<AssignLocal("$v","3")>}                                                     # anID
   ; 
 ID : 'a'..'z'+ ;

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/ReturnValueAndActionsList1.stg

@@ -1,10 +1,10 @@
-/**
+/*
- * This is a regression test for antlr/antlr4#677 "labels not working in
+ * This is a regression test for antlr/antlr4#677 "labels not working in grammar
- * grammar file".
+ * file".
  * https://github.com/antlr/antlr4/issues/677
  *
- * <p>This test treats {@code ,} and {@code >>} as part of a single compound
+ * This test treats `,` and `>>` as part of a single compound operator (similar
- * operator (similar to a ternary operator).</p>
+ * to a ternary operator).
  */
 
 TestType() ::= "Parser"

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/ReturnValueAndActionsList2.stg

@@ -1,9 +1,9 @@
-/**
+/*
- * This is a regression test for antlr/antlr4#677 "labels not working in
+ * This is a regression test for antlr/antlr4#677 "labels not working in grammar
- * grammar file".
+ * file".
  * https://github.com/antlr/antlr4/issues/677
  *
- * <p>This test treats the {@code ,} and {@code >>} operators separately.</p>
+ * This test treats the `,` and `>>` operators separately.
  */
 
 TestType() ::= "Parser"

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LeftRecursion/WhitespaceInfluence.stg

@@ -1,6 +1,6 @@
-/**
+/*
- * This is a regression test for #239 "recoursive parser using implicit
+ * This is a regression test for #239 "recoursive parser using implicit tokens
- * tokens ignore white space lexer rule".
+ * ignore white space lexer rule".
  * https://github.com/antlr/antlr4/issues/239
  */
 
@@ -63,7 +63,7 @@ ID              : LETTER ALPHANUM*;
 NUMBER          : DIGIT+ ('.' DIGIT+)? (('e'|'E')('+'|'-')? DIGIT+)?;
 DATE            : '\'' DIGIT DIGIT DIGIT DIGIT '-' DIGIT DIGIT '-' DIGIT DIGIT (' ' DIGIT DIGIT ':' DIGIT DIGIT ':' DIGIT DIGIT ('.' DIGIT+)?)? '\'';
 SQ_STRING       : '\'' ('\'\'' | ~'\'')* '\'';
-DQ_STRING       : '\"' ('\\\"' | ~'\"')* '\"';
+DQ_STRING       : '"' ('\\\\"' | ~'"')* '"';
 WS              : [ \t\n\r]+ -> skip ;
 COMMENTS        : ('/*' .*? '*/' | '//' ~'\n'* '\n' ) -> skip;
 >>

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LexerExec/CharSet.stg

@@ -1,17 +1,21 @@
 TestType() ::= "Lexer"
 
+Options ::= [
+	"Ignore": "Currently unable to generate the required CRLF input sequence."
+]
+
 Grammar ::= [
 	"L": {<grammar("L")>}
 ]
 
-Input() ::= "34<\n> 34"
+Input() ::= "34<\u000D><\n> 34"
 
 Output() ::= <<
 I
 I
 [@0,0:1='34',\<1>,1:0]
-[@1,4:5='34',\<1>,2:1]
+[@1,5:6='34',\<1>,2:1]
-[@2,6:5='\<EOF>',\<-1>,2:3]<\n>
+[@2,7:6='\<EOF>',\<-1>,2:3]<\n>
 >>
 
 Errors() ::= ""

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LexerExec/CharSetPlus.stg

@@ -1,17 +1,21 @@
 TestType() ::= "Lexer"
 
+Options ::= [
+	"Ignore": "Currently unable to generate the required CRLF input sequence."
+]
+
 Grammar ::= [
 	"L": {<grammar("L")>}
 ]
 
-Input() ::= "34<\n> 34"
+Input() ::= "34<\u000D><\n> 34"
 
 Output() ::= <<
 I
 I
 [@0,0:1='34',\<1>,1:0]
-[@1,4:5='34',\<1>,2:1]
+[@1,5:6='34',\<1>,2:1]
-[@2,6:5='\<EOF>',\<-1>,2:3]<\n>
+[@2,7:6='\<EOF>',\<-1>,2:3]<\n>
 >>
 
 Errors() ::= ""

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LexerExec/CharSetRange.stg

@@ -1,10 +1,14 @@
 TestType() ::= "Lexer"
 
+Options ::= [
+	"Ignore": "Currently unable to generate the required CRLF input sequence."
+]
+
 Grammar ::= [
 	"L": {<grammar("L")>}
 ]
 
-Input() ::= "34<\n> 34 a2 abc <\n>   "
+Input() ::= "34<\u000D> 34 a2 abc <\n>   "
 
 Output() ::= <<
 I
@@ -12,10 +16,10 @@ I
 ID
 ID
 [@0,0:1='34',\<1>,1:0]
-[@1,4:5='34',\<1>,2:1]
+[@1,4:5='34',\<1>,1:4]
-[@2,7:8='a2',\<2>,2:4]
+[@2,7:8='a2',\<2>,1:7]
-[@3,10:12='abc',\<2>,2:7]
+[@3,10:12='abc',\<2>,1:10]
-[@4,18:17='\<EOF>',\<-1>,3:3]<\n>
+[@4,18:17='\<EOF>',\<-1>,2:3]<\n>
 >>
 
 Errors() ::= ""

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LexerExec/CharSetWithMissingEndRange.stg

@@ -1,15 +1,19 @@
 TestType() ::= "Lexer"
 
+Options ::= [
+	"Ignore": "Currently unable to generate the required CRLF input sequence."
+]
+
 Grammar ::= [
 	"L": {<grammar("L")>}
 ]
 
-Input() ::= "00<\n>"
+Input() ::= "00<\u000D><\n>"
 
 Output() ::= <<
 I
 [@0,0:1='00',\<1>,1:0]
-[@1,3:2='\<EOF>',\<-1>,2:0]<\n>
+[@1,4:3='\<EOF>',\<-1>,2:0]<\n>
 >>
 
 Errors() ::= ""

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LexerExec/CharSetWithQuote2.stg

@@ -18,6 +18,6 @@ Errors() ::= ""
 
 grammar(grammarName) ::= <<
 lexer grammar <grammarName>;
-A : ["\\ab]+ {<writeln("\"A\"")>} ;
+A : ["\\\\ab]+ {<writeln("\"A\"")>} ;
 WS : [ \n\t]+ -> skip ;
 >>

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LexerExec/EOFByItself.stg

@@ -6,6 +6,7 @@ Grammar ::= [
 
 Input() ::= ""
 
+// must get DONE EOF
 Output() ::= <<
 [@0,0:-1='\<EOF>',\<1>,1:0]
 [@1,0:-1='\<EOF>',\<-1>,1:0]<\n>

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LexerExec/LargeLexer.stg

@@ -1,3 +1,8 @@
+/*
+ * This is a regression test for antlr/antlr4#76 "Serialized ATN strings
+ * should be split when longer than 2^16 bytes (class file limitation)"
+ * https://github.com/antlr/antlr4/issues/76
+ */
 TestType() ::= "Lexer"
 
 Grammar ::= [

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LexerExec/Parentheses.stg

@@ -1,3 +1,8 @@
+/*
+ * This is a regression test for antlr/antlr4#224: "Parentheses without
+ * quantifier in lexer rules have unclear effect".
+ * https://github.com/antlr/antlr4/issues/224
+ */
 TestType() ::= "Lexer"
 
 Grammar ::= [

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LexerExec/RecursiveLexerRuleRefWithWildcardPlus_1.stg

@@ -6,6 +6,7 @@ Grammar ::= [
 	"L": {<grammar("L")>}
 ]
 
+// stuff on end of comment matches another rule
 Input() ::= <<
 /* ick */
 /* /* */

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LexerExec/RecursiveLexerRuleRefWithWildcardPlus_2.stg

@@ -6,6 +6,7 @@ Grammar ::= [
 	"L": {<grammar("L")>}
 ]
 
+// stuff on end of comment doesn't match another rule
 Input() ::= <<
 /* ick */x
 /* /* */x

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LexerExec/RecursiveLexerRuleRefWithWildcardStar_1.stg

@@ -6,6 +6,7 @@ Grammar ::= [
 	"L": {<grammar("L")>}
 ]
 
+// stuff on end of comment matches another rule
 Input() ::= <<
 /* ick */
 /* /* */

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LexerExec/RecursiveLexerRuleRefWithWildcardStar_2.stg

@@ -6,6 +6,7 @@ Grammar ::= [
 	"L": {<grammar("L")>}
 ]
 
+// stuff on end of comment doesn't match another rule
 Input() ::= <<
 /* ick */x
 /* /* */x

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LexerExec/RefToRuleDoesNotSetTokenNorEmitAnother.stg

@@ -12,7 +12,7 @@ Output() ::= <<
 [@0,0:1='34',\<2>,1:0]
 [@1,3:5='-21',\<1>,1:3]
 [@2,7:7='3',\<2>,1:7]
-[@3,8:7='\<EOF>',\<-1>,1:8]<\n>
+[@3,8:7='\<EOF>',\<-1>,1:8]<\n><! EOF has no length so range is 8:7 not 8:8 !>
 >>
 
 Errors() ::= ""

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/LexerExec/ZeroLengthToken.stg

@@ -2,6 +2,8 @@
  * This is a regression test for antlr/antlr4#687 "Empty zero-length tokens
  * cannot have lexer commands" and antlr/antlr4#688 "Lexer cannot match
  * zero-length tokens"
+ * https://github.com/antlr/antlr4/issues/687
+ * https://github.com/antlr/antlr4/issues/688
  */
 
 TestType() ::= "Lexer"

tool/test/org/antlr/v4/test/runtime/java/AbstractParserTestMethod.java

@@ -1,12 +0,0 @@
-package org.antlr.v4.test.runtime.java;
-
-public class AbstractParserTestMethod extends JUnitTestMethod {
-
-	public String startRule;
-
-	public AbstractParserTestMethod(String name, String grammarName, String startRule) {
-		super(name, grammarName, null, null, null, null);
-		this.startRule = startRule;
-	}
-
-}

tool/test/org/antlr/v4/test/runtime/java/CompositeLexerTestMethod.java

@@ -1,35 +0,0 @@
-package org.antlr.v4.test.runtime.java;
-
-import org.stringtemplate.v4.STGroup;
-
-import java.io.File;
-
-public class CompositeLexerTestMethod extends LexerTestMethod {
-
-	public Grammar[] slaveGrammars;
-
-	public CompositeLexerTestMethod(String name, String grammarName,
-			String input, String expectedOutput,
-			String expectedErrors, String ... slaves) {
-		super(name, grammarName, input, expectedOutput, expectedErrors, null);
-		this.slaveGrammars = new Grammar[slaves.length];
-		for(int i=0;i<slaves.length;i++)
-			this.slaveGrammars[i] = new Grammar(name + "_" + slaves[i], slaves[i]);
-
-	}
-
-	@Override
-	public void loadGrammars(File grammarDir, String testFileName) throws Exception {
-		for(Grammar slave : slaveGrammars)
-			slave.load(new File(grammarDir, testFileName));
-		super.loadGrammars(grammarDir, testFileName);
-	}
-
-	@Override
-	public void generateGrammars(STGroup group) {
-		for(Grammar slave : slaveGrammars)
-			slave.generate(group);
-		super.generateGrammars(group);
-	}
-
-}

tool/test/org/antlr/v4/test/runtime/java/CompositeParserTestMethod.java

@@ -1,36 +0,0 @@
-package org.antlr.v4.test.runtime.java;
-
-import org.stringtemplate.v4.STGroup;
-
-import java.io.File;
-
-public class CompositeParserTestMethod extends ParserTestMethod {
-
-	public Grammar[] slaveGrammars;
-	public boolean slaveIsLexer = false;
-
-	public CompositeParserTestMethod(String name, String grammarName,
-			String startRule, String input, String expectedOutput,
-			String expectedErrors, String ... slaves) {
-		super(name, grammarName, startRule, input, expectedOutput, expectedErrors);
-		this.slaveGrammars = new Grammar[slaves.length];
-		for(int i=0;i<slaves.length;i++)
-			this.slaveGrammars[i] = new Grammar(name + "_" + slaves[i], slaves[i]);
-
-	}
-
-	@Override
-	public void loadGrammars(File grammarDir, String testFileName) throws Exception {
-		for(Grammar slave : slaveGrammars)
-			slave.load(new File(grammarDir, testFileName));
-		super.loadGrammars(grammarDir, testFileName);
-	}
-
-	@Override
-	public void generateGrammars(STGroup group) {
-		for(Grammar slave : slaveGrammars)
-			slave.generate(group);
-		super.generateGrammars(group);
-	}
-
-}

tool/test/org/antlr/v4/test/runtime/java/ConcreteParserTestMethod.java

@@ -1,25 +0,0 @@
-package org.antlr.v4.test.runtime.java;
-
-import org.stringtemplate.v4.STGroup;
-
-import java.io.File;
-
-public class ConcreteParserTestMethod extends JUnitTestMethod {
-
-	public String baseName;
-
-	public ConcreteParserTestMethod(String name, String input, String expectedOutput,
-			String expectedErrors, Integer index) {
-		super(name, null, input, expectedOutput, expectedErrors, index);
-		this.baseName = name;
-	}
-
-	@Override
-	public void loadGrammars(File grammarDir, String testFileName) throws Exception {
-	}
-
-	@Override
-	public void generateGrammars(STGroup group) {
-	}
-
-}

tool/test/org/antlr/v4/test/runtime/java/Grammar.java

@@ -1,51 +0,0 @@
-package org.antlr.v4.test.runtime.java;
-
-import org.stringtemplate.v4.ST;
-import org.stringtemplate.v4.STGroup;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.InputStream;
-
-public class Grammar {
-
-	public String fileName;
-	public String grammarName;
-	public String[] lines;
-	public ST template;
-
-	public Grammar(String fileName, String grammarName) {
-		this.fileName = fileName;
-		this.grammarName = grammarName;
-	}
-
-	public void load(File grammarDir) throws Exception {
-		template = loadGrammar(grammarDir, fileName);
-	}
-
-	protected ST loadGrammar(File grammarDir, String grammarFileName) throws Exception {
-		File file = new File(grammarDir, grammarFileName + ".st");
-		InputStream input = new FileInputStream(file);
-		try {
-			byte[] data = new byte[(int)file.length()];
-			int next = 0;
-			while(input.available()>0) {
-				int read = input.read(data, next, data.length - next);
-				next += read;
-			}
-			String s = new String(data);
-			return new ST(s);
-		} finally {
-			input.close();
-		}
-	}
-
-	public void generate(STGroup group) {
-		template.add("grammarName", grammarName);
-		template.groupThatCreatedThisInstance = group; // so templates get interpreted
-		lines = template.render().split("\n");
-		for(int i=0;i<lines.length;i++)
-			lines[i] = Generator.escape(lines[i]);
-	}
-
-}

tool/test/org/antlr/v4/test/runtime/java/JUnitTestFile.java

@@ -1,101 +0,0 @@
-package org.antlr.v4.test.runtime.java;
-
-import org.stringtemplate.v4.ST;
-import org.stringtemplate.v4.STGroup;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.List;
-
-public class JUnitTestFile {
-
-	List<JUnitTestMethod> unitTests = new ArrayList<JUnitTestMethod>();
-	public String name;
-	public List<String> tests = new ArrayList<String>();
-	public boolean importErrorQueue = false;
-	public boolean importGrammar = false;
-
-	public JUnitTestFile(String name) {
-		this.name = name;
-	}
-
-	public String getName() {
-		return name;
-	}
-
-	public ParserTestMethod addParserTest(File grammarDir, String name, String grammarName, String methodName,
-			String input, String expectedOutput, String expectedErrors) throws Exception {
-		ParserTestMethod tm = new ParserTestMethod(name, grammarName, methodName, input, expectedOutput, expectedErrors);
-		tm.loadGrammars(grammarDir, this.name);
-		unitTests.add(tm);
-		return tm;
-	}
-
-	public AbstractParserTestMethod addParserTests(File grammarDir, String name, String grammarName, String methodName,
-			String ... inputsAndOuputs) throws Exception {
-		AbstractParserTestMethod tm = new AbstractParserTestMethod(name, grammarName, methodName);
-		tm.loadGrammars(grammarDir, this.name);
-		unitTests.add(tm);
-		for(int i=0; i<inputsAndOuputs.length; i+=2) {
-			ConcreteParserTestMethod cm = new ConcreteParserTestMethod(name,
-					inputsAndOuputs[i], inputsAndOuputs[i+1], null,
-					1 + (i/2));
-			unitTests.add(cm);
-		}
-		return tm;
-	}
-
-	public AbstractParserTestMethod addParserTestsWithErrors(File grammarDir, String name, String grammarName, String methodName,
-			String ... inputsOuputsAndErrors) throws Exception {
-		AbstractParserTestMethod tm = new AbstractParserTestMethod(name, grammarName, methodName);
-		tm.loadGrammars(grammarDir, this.name);
-		unitTests.add(tm);
-		for(int i=0; i<inputsOuputsAndErrors.length; i+=3) {
-			ConcreteParserTestMethod cm = new ConcreteParserTestMethod(name,
-					inputsOuputsAndErrors[i], inputsOuputsAndErrors[i+1], inputsOuputsAndErrors[i+2],
-					1 + (i/3));
-			unitTests.add(cm);
-		}
-		return tm;
-	}
-
-	public CompositeParserTestMethod addCompositeParserTest(File grammarDir, String name, String grammarName, String methodName,
-			String input, String expectedOutput, String expectedErrors, String ... slaves) throws Exception {
-		CompositeParserTestMethod tm = new CompositeParserTestMethod(name, grammarName, methodName, input, expectedOutput, expectedErrors, slaves);
-		tm.loadGrammars(grammarDir, this.name);
-		unitTests.add(tm);
-		return tm;
-	}
-
-	public LexerTestMethod addLexerTest(File grammarDir, String name, String grammarName,
-			String input, String expectedOutput, String expectedErrors) throws Exception {
-		return addLexerTest(grammarDir, name, grammarName, input, expectedOutput, expectedErrors, null);
-	}
-
-	public LexerTestMethod addLexerTest(File grammarDir, String name, String grammarName,
-			String input, String expectedOutput, String expectedErrors, Integer index) throws Exception {
-		LexerTestMethod tm = new LexerTestMethod(name, grammarName, input, expectedOutput, expectedErrors, index);
-		tm.loadGrammars(grammarDir, this.name);
-		unitTests.add(tm);
-		return tm;
-	}
-
-	public CompositeLexerTestMethod addCompositeLexerTest(File grammarDir, String name, String grammarName,
-			String input, String expectedOutput, String expectedErrors, String ... slaves) throws Exception {
-		CompositeLexerTestMethod tm = new CompositeLexerTestMethod(name, grammarName, input, expectedOutput, expectedErrors, slaves);
-		tm.loadGrammars(grammarDir, this.name);
-		unitTests.add(tm);
-		return tm;
-	}
-
-	public void generateUnitTests(STGroup group) {
-		for(JUnitTestMethod tm : unitTests) {
-			tm.generateGrammars(group);
-			String name = tm.getClass().getSimpleName();
-			ST template = group.getInstanceOf(name);
-			template.add("test", tm);
-			tests.add(template.render());
-		}
-	}
-
-}

tool/test/org/antlr/v4/test/runtime/java/JUnitTestMethod.java

@@ -1,34 +0,0 @@
-package org.antlr.v4.test.runtime.java;
-
-import org.stringtemplate.v4.STGroup;
-
-import java.io.File;
-
-public abstract class JUnitTestMethod {
-
-	public String name;
-	public Grammar grammar;
-	public String afterGrammar;
-	public String input;
-	public String expectedOutput;
-	public String expectedErrors;
-	public boolean debug = false;
-
-	protected JUnitTestMethod(String name, String grammarName, String input,
-			String expectedOutput, String expectedErrors, Integer index) {
-		this.name = name + (index==null ? "" : "_" + index);
-		this.grammar = new Grammar(name, grammarName);
-		this.input = Generator.escape(input);
-		this.expectedOutput = Generator.escape(expectedOutput);
-		this.expectedErrors = Generator.escape(expectedErrors);
-	}
-
-	public void loadGrammars(File grammarDir, String testFileName) throws Exception {
-		grammar.load(new File(grammarDir, testFileName));
-	}
-
-	public void generateGrammars(STGroup group) {
-		grammar.generate(group);
-	}
-
-}

tool/test/org/antlr/v4/test/runtime/java/Java.test.stg

@@ -2,6 +2,7 @@ TestFile(file) ::= <<
 package org.antlr.v4.test.runtime.java;
 
 import org.antlr.v4.test.tool.BaseTest;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import static org.junit.Assert.*;
@@ -21,7 +22,7 @@ public class Test<file.name> extends BaseTest {
 >>
 
 LexerTestMethod(test) ::= <<
-@Test
+<testAnnotations(test)>
 public void test<test.name>() throws Exception {
 	mkdir(tmpdir);
 
@@ -52,8 +53,42 @@ CompositeLexerTestMethod(test) ::= <<
 <LexerTestMethod(test)>
 >>
 
-ParserTestMethod(test) ::= <<
+testAnnotations(test) ::= <%
 @Test
+<if(test.Options.("Ignore"))>
+<\n>@Ignore(<writeStringLiteral(test.Options.("Ignore"))>)
+<elseif(IgnoredTests.(({<file.name>.<test.name>})))>
+<\n>@Ignore(<writeStringLiteral(IgnoredTests.(({<file.name>.<test.name>})))>)
+<endif>
+%>
+
+writeBoolean(o) ::= "<if(o && !isEmpty.(o))>true<else>false<endif>"
+
+buildStringLiteral(text, variable) ::= <<
+StringBuilder <variable>Builder = new StringBuilder(<strlen.(text)>);
+<lines.(text):{line|<variable>Builder.append("<escape.(line)>");}; separator="\n">
+String <variable> = <variable>Builder.toString();
+>>
+
+writeStringLiteral(text) ::= <%
+<if(isEmpty.(text))>
+""
+<else>
+<writeLines(lines.(text))>
+<endif>
+%>
+
+writeLines(textLines) ::= <%
+<if(rest(textLines))>
+<textLines:{line|
+<\n>		"<escape.(line)>}; separator="\" +">"
+<else>
+"<escape.(first(textLines))>"
+<endif>
+%>
+
+ParserTestMethod(test) ::= <<
+<testAnnotations(test)>
 public void test<test.name>() throws Exception {
 	mkdir(tmpdir);
 
@@ -83,12 +118,15 @@ public void test<test.name>() throws Exception {
 
 >>
 
+string(text) ::= <<
+"<escape.(text)>"
+>>
+
 CompositeParserTestMethod(test) ::= <<
 <ParserTestMethod(test)>
 >>
 
 AbstractParserTestMethod(test) ::= <<
-/* this file and method are generated, any edit will be overwritten by the next generation */
 String test<test.name>(String input) throws Exception {
 	String grammar = <test.grammar.lines:{ line | "<line>};separator="\\n\" +\n", wrap, anchor>";
 	return execParser("<test.grammar.grammarName>.g4", grammar, "<test.grammar.grammarName>Parser", "<test.grammar.grammarName>Lexer", "<test.startRule>", input, <test.debug>);
@@ -97,8 +135,7 @@ String test<test.name>(String input) throws Exception {
 >>
 
 ConcreteParserTestMethod(test) ::= <<
-/* this file and method are generated, any edit will be overwritten by the next generation */
+<testAnnotations(test)>
-@Test
 public void test<test.name>() throws Exception {
 	String found = test<test.baseName>("<test.input>");
 	assertEquals("<test.expectedOutput>", found);
@@ -111,35 +148,6 @@ public void test<test.name>() throws Exception {
 
 >>
 
-string(text) ::= <<
-"<escape.(text)>"
->>
-
-writeBoolean(o) ::= "<if(o && !isEmpty.(o))>true<else>false<endif>"
-
-buildStringLiteral(text, variable) ::= <<
-StringBuilder <variable>Builder = new StringBuilder(<strlen.(text)>);
-<lines.(text):{line|<variable>Builder.append("<escape.(line)>");}; separator="\n">
-String <variable> = <variable>Builder.toString();
->>
-
-writeStringLiteral(text) ::= <%
-<if(isEmpty.(text))>
-""
-<else>
-<writeLines(lines.(text))>
-<endif>
-%>
-
-writeLines(textLines) ::= <%
-<if(rest(textLines))>
-<textLines:{line|
-<\n>		"<escape.(line)>}; separator="\" +">"
-<else>
-"<escape.(first(textLines))>"
-<endif>
-%>
-
 writeln(s) ::= <<System.out.println(<s>);>>
 
 write(s) ::= <<System.out.print(<s>);>>
@@ -220,14 +228,6 @@ RuleInvocationStack() ::= "getRuleInvocationStack()"
 
 LL_EXACT_AMBIG_DETECTION() ::= <<_interp.setPredictionMode(PredictionMode.LL_EXACT_AMBIG_DETECTION);>>
 
-ParserPropertyMember() ::= <<
-@members {
-boolean Property() {
-	return true;
-}
-}
->>
-
 PositionAdjustingLexer() ::= <<
 
 @Override
@@ -381,9 +381,11 @@ void foo() {
 }
 >>
 
-Declare_foo() ::= <<public void foo() {System.out.println("foo");}>>
+Declare_foo() ::= <<
+	public void foo() {System.out.println("foo");}
+>>
 
-Invoke_foo() ::= "this.foo();"
+Invoke_foo() ::= "foo();"
 
 Declare_pred() ::= <<boolean pred(boolean v) {
 	System.out.println("eval="+v);
@@ -393,6 +395,10 @@ Declare_pred() ::= <<boolean pred(boolean v) {
 
 Invoke_pred(v) ::= <<this.pred(<v>)>>
 
+IgnoredTests ::= [
+	default: false
+]
+
 isEmpty ::= [
 	"": true,
 	default: false

tool/test/org/antlr/v4/test/runtime/java/LexerTestMethod.java

@@ -1,17 +0,0 @@
-package org.antlr.v4.test.runtime.java;
-
-public class LexerTestMethod extends JUnitTestMethod {
-
-	public String[] outputLines;
-	public boolean lexerOnly = true;
-	public boolean showDFA = false;
-
-	public LexerTestMethod(String name, String grammarName, String input,
-			String expectedOutput, String expectedErrors, Integer index) {
-		super(name, grammarName, input, expectedOutput, expectedErrors, index);
-		outputLines = expectedOutput.split("\n");
-		for(int i=0;i<outputLines.length;i++)
-			outputLines[i] = Generator.escape(outputLines[i]);
-	}
-
-}

tool/test/org/antlr/v4/test/runtime/java/ParserTestMethod.java

@@ -1,13 +0,0 @@
-package org.antlr.v4.test.runtime.java;
-
-public class ParserTestMethod extends JUnitTestMethod {
-
-	public String startRule;
-
-	public ParserTestMethod(String name, String grammarName, String startRule,
-			String input, String expectedOutput, String expectedErrors) {
-		super(name, grammarName, input, expectedOutput, expectedErrors, null);
-		this.startRule = startRule;
-	}
-
-}

tool/test/org/antlr/v4/test/tool/TestCompositeGrammars.java

@@ -149,8 +149,6 @@ public class TestCompositeGrammars extends BaseTest {
 		assertEquals(0, equeue.size());
 	}
 
-
-
 	@Test public void testImportedTokenVocabIgnoredWithWarning() throws Exception {
 		ErrorQueue equeue = new ErrorQueue();
 		String slave =
@@ -404,7 +402,6 @@ public class TestCompositeGrammars extends BaseTest {
 		assertEquals(expecting, equeue.errors.size());
 	}
 
-
 	/**
 	 * This is a regression test for antlr/antlr4#670 "exception when importing
 	 * grammar".  I think this one always worked but I found that a different