Merge pull request #723 from sharwell/fix-465

Move DOC_COMMENT tokens to the comments channel

tool/src/org/antlr/v4/parse/ANTLRLexer.g

@@ -150,6 +150,8 @@ import org.antlr.v4.tool.*;
 
 
 @members {
+	public static final int COMMENTS_CHANNEL = 2;
+
     public CommonTokenStream tokens; // track stream we push to; need for context info
     public boolean isLexerRule = false;
 
@@ -261,15 +263,11 @@ COMMENT
            }
        )
        {
-         // Unless we had a documentation comment, then we do not wish to
-         // pass the comments in to the parser. If you are writing a formatter
-         // then you will want to preserve the comments off channel, but could
-         // just skip and save token space if not.
+         // We do not wish to pass the comments in to the parser. If you are
+         // writing a formatter then you will want to preserve the comments off
+         // channel, but could just skip and save token space if not.
          //
-         if ($type != DOC_COMMENT) {
-
-             $channel=2;  // Comments are on channel 2
-         }
+         $channel=COMMENTS_CHANNEL;
        }
     ;
 

tool/src/org/antlr/v4/parse/ANTLRParser.g

@@ -150,13 +150,7 @@ if ( options!=null ) {
 	Grammar.setNodeOptions($tree, options);
 }
 }
-    :
-      // The grammar itself can have a documenation comment, which is the
-      // first terminal in the file.
-      //
-      DOC_COMMENT?
-
-      // Next we should see the type and name of the grammar file that
+    : // First we should see the type and name of the grammar file that
       // we are about to parse.
       //
       grammarType id SEMI
@@ -195,7 +189,6 @@ if ( options!=null ) {
 
       -> ^(grammarType       // The grammar type is our root AST node
              id              // We need to identify the grammar of course
-             DOC_COMMENT?    // We may or may not have a global documentation comment for the file
              prequelConstruct* // The set of declarations we accumulated
              rules           // And of course, we need the set of rules we discovered
              modeSpec*
@@ -371,10 +364,7 @@ parserRule
 		Grammar.setNodeOptions($tree, options);
 	}
 }
-    : // A rule may start with an optional documentation comment
-      DOC_COMMENT?
-
-	  // Next comes the rule name. Here we do not distinguish between
+    : // Start with the rule name. Here we do not distinguish between
 	  // parser or lexer rules, the semantic verification phase will
 	  // reject any rules that make no sense, such as lexer rules in
 	  // a pure parser or tree parser.
@@ -419,7 +409,7 @@ parserRule
 
       exceptionGroup
 
-      -> ^( RULE<RuleAST> RULE_REF DOC_COMMENT? ARG_ACTION<ActionAST>?
+      -> ^( RULE<RuleAST> RULE_REF ARG_ACTION<ActionAST>?
       		ruleReturns? throwsSpec? localsSpec? rulePrequels? ruleBlock exceptionGroup*
       	  )
     ;
@@ -529,9 +519,9 @@ lexerRule
 @after {
 	paraphrases.pop();
 }
-    : DOC_COMMENT? FRAGMENT?
+    : FRAGMENT?
 	  TOKEN_REF COLON lexerRuleBlock SEMI
-      -> ^( RULE<RuleAST> TOKEN_REF DOC_COMMENT?
+      -> ^( RULE<RuleAST> TOKEN_REF
       		^(RULEMODIFIERS FRAGMENT)? lexerRuleBlock
       	  )
 	;

tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g

@@ -343,7 +343,7 @@ grammarSpec
 @after {
 	exitGrammarSpec($start);
 }
-    :   ^(	GRAMMAR ID {grammarName=$ID.text;} DOC_COMMENT?
+    :   ^(	GRAMMAR ID {grammarName=$ID.text;}
     		{discoverGrammar((GrammarRootAST)$GRAMMAR, $ID);}
  		   	prequelConstructs
     		{finishPrequels($prequelConstructs.firstOne);}
@@ -519,7 +519,7 @@ lexerRule
 }
 	:	^(	RULE TOKEN_REF
 			{currentRuleName=$TOKEN_REF.text; currentRuleAST=$RULE;}
-			DOC_COMMENT? (^(RULEMODIFIERS m=FRAGMENT {mods.add($m);}))?
+			(^(RULEMODIFIERS m=FRAGMENT {mods.add($m);}))?
       		{discoverLexerRule((RuleAST)$RULE, $TOKEN_REF, mods, (GrammarAST)input.LT(1));}
       		lexerRuleBlock
       		{
@@ -540,7 +540,7 @@ rule
 	exitRule($start);
 }
 	:   ^(	RULE RULE_REF {currentRuleName=$RULE_REF.text; currentRuleAST=$RULE;}
-			DOC_COMMENT? (^(RULEMODIFIERS (m=ruleModifier{mods.add($m.start);})+))?
+			(^(RULEMODIFIERS (m=ruleModifier{mods.add($m.start);})+))?
 			ARG_ACTION?
       		ret=ruleReturns?
       		thr=throwsSpec?

tool/src/org/antlr/v4/parse/LeftRecursiveRuleWalker.g

@@ -67,7 +67,7 @@ rec_rule returns [boolean isLeftRec]
 	currentOuterAltNumber = 1;
 }
 	:	^(	r=RULE id=RULE_REF {ruleName=$id.getText();}
-			DOC_COMMENT? ruleModifier?
+			ruleModifier?
 //			(ARG_ACTION)? shouldn't allow args, right?
 			(^(RETURNS a=ARG_ACTION {setReturnValues($a);}))?
 //      		( ^(THROWS .+) )? don't allow