after much bullshit, got sets working for ast stuff.

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 8855]

runtime/Java/src/org/antlr/v4/runtime/atn/ATNInterpreter.java

@@ -144,7 +144,7 @@ public abstract class ATNInterpreter {
 			case Transition.FORCED_ACTION : return new ActionTransition(target, arg1, arg2);
 			case Transition.SET : return new SetTransition(sets.get(arg1), target);
 			case Transition.NOT_ATOM : return new NotAtomTransition(arg1, target);
-			case Transition.NOT_SET : return new NotSetTransition(sets.get(arg1), target);
+			case Transition.NOT_SET : return new NotSetTransition(null, sets.get(arg1), target);
 			case Transition.WILDCARD : return new WildcardTransition(target);
 		}
 		return null;

runtime/Java/src/org/antlr/v4/runtime/atn/LL1Analyzer.java

@@ -95,7 +95,8 @@ public class LL1Analyzer {
 			}
 			else {
 //				System.out.println("adding "+ t);
-				look.addAll(t.label());
+				IntervalSet set = t.label();
+				look.addAll(set);
 			}
 		}
 	}

runtime/Java/src/org/antlr/v4/runtime/atn/LexerInterpreter.java

@@ -262,8 +262,8 @@ public class LexerInterpreter extends ATNInterpreter {
 		else if ( trans instanceof SetTransition ) {
 			SetTransition st = (SetTransition)trans;
 			boolean not = trans instanceof NotSetTransition;
-			if ( !not && st.label.member(t) || not && !st.label.member(t) ) {
+			if ( !not && st.set.member(t) || not && !st.set.member(t) ) {
-				if ( debug ) System.out.println("match set "+st.label.toString());
+				if ( debug ) System.out.println("match set "+st.set.toString());
 				return st.target;
 			}
 		}

runtime/Java/src/org/antlr/v4/runtime/atn/NotSetTransition.java

@@ -32,14 +32,22 @@ package org.antlr.v4.runtime.atn;
 import org.antlr.v4.runtime.misc.IntervalSet;
 
 public class NotSetTransition extends SetTransition {
-	public NotSetTransition(IntervalSet label, ATNState target) {
+	// keep both set, notSet; we can only compute at construction time
-		super(label, target);
+	// since only then do we have grammar, which knows token set for complement.
+	public IntervalSet notSet;
+
+	public NotSetTransition(IntervalSet set, IntervalSet notSet, ATNState target) {
+		super(set, target);
+		this.notSet = notSet;
 	}
 
 	public NotSetTransition(ATNState target) {
 		super(target);
 	}
 
+	@Override
+	public IntervalSet label() { return notSet; }
+
 	@Override
 	public String toString() {
 		return '~'+super.toString();

runtime/Java/src/org/antlr/v4/runtime/atn/ParserInterpreter.java

@@ -386,15 +386,15 @@ public class ParserInterpreter extends ATNInterpreter {
 	public ATNState getReachableTarget(Transition trans, int ttype) {
 		if ( trans instanceof AtomTransition ) {
 			AtomTransition at = (AtomTransition)trans;
-			boolean not = trans instanceof NotAtomTransition;
+//			boolean not = trans instanceof NotAtomTransition;
-			if ( !not && at.label == ttype || not && at.label!=ttype ) {
+			if ( at.label == ttype ) {
 				return at.target;
 			}
 		}
 		else if ( trans instanceof SetTransition ) {
 			SetTransition st = (SetTransition)trans;
 			boolean not = trans instanceof NotSetTransition;
-			if ( !not && st.label.member(ttype) || not && !st.label.member(ttype) ) {
+			if ( !not && st.set.member(ttype) || not && !st.set.member(ttype) ) {
 				return st.target;
 			}
 		}
@@ -648,7 +648,8 @@ public class ParserInterpreter extends ATNInterpreter {
 			}
 			else if ( t instanceof SetTransition ) {
 				SetTransition st = (SetTransition)t;
-				trans = "Set "+st.label.toString();
+				boolean not = st instanceof NotSetTransition;
+				trans = (not?"~":"")+"Set "+st.set.toString();
 			}
 			System.err.println(c.toString(parser, true)+":"+trans);
 		}

runtime/Java/src/org/antlr/v4/runtime/atn/SetTransition.java

@@ -34,21 +34,21 @@ import org.antlr.v4.runtime.misc.IntervalSet;
 
 /** A transition containing a set of values */
 public class SetTransition extends Transition {
-	public IntervalSet label;
+	public IntervalSet set;
 
-	public SetTransition(IntervalSet label, ATNState target) {
+	public SetTransition(IntervalSet set, ATNState target) {
 		super(target);
-		if ( label==null ) label = IntervalSet.of(Token.INVALID_TYPE);
+		if ( set == null ) set = IntervalSet.of(Token.INVALID_TYPE);
-		this.label = label;
+		this.set = set;
 	}
 
 	public SetTransition(ATNState target) {
 		super(target);
 	}
 
-	public IntervalSet label() { return label; }
+	public IntervalSet label() { return set; }
 
 	public String toString() {
-		return label.toString();
+		return set.toString();
 	}
 }

tool/playground/T.g

@@ -1,10 +1,11 @@
 grammar T;
 options {output=AST;}
 
-a : (ID INT) ;
+a : ('+' | '-')^  ;
 b returns [int i] : ID;
 ID : 'a'..'z'+ ;
 INT : '0'..'9'+;
+PLUS : '+';
 WS : (' '|'\n') {$channel=HIDDEN;} ;
 
 /*

tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg

@@ -235,6 +235,8 @@ case <i>:
 }
 >>
 
+// TODO: we we need uniqueID? a single _alt might work
+
 StarBlock(choice, alts, sync) ::= <<
 int _alt<choice.uniqueID> = _interp.adaptivePredict(input,<choice.decision>,_ctx);
 while ( _alt<choice.uniqueID>!=<choice.exitAlt> ) {
@@ -285,6 +287,18 @@ _localctx.s = <m.stateNumber>;
 <if(m.labels)><m.labels:{l | <labelref(l)> = }>(Token)<endif>match(<m.name>);
 >>
 
+MatchSet(m, expr, capture) ::= "<CommonSetStuff(m, expr, capture, false)>"
+
+MatchNotSet(m, expr, capture) ::= "<CommonSetStuff(m, expr, capture, true)>"
+
+CommonSetStuff(m, expr, capture, invert) ::= <<
+_localctx.s = <m.stateNumber>;
+<if(m.labels)><m.labels:{l | <labelref(l)> = }><endif>input.LT(1);
+<capture>
+if ( <if(!invert)>!<endif>(<expr>) ) throw new MismatchedSetException(this, input);
+input.consume();
+>>
+
 Wildcard(w) ::= <<
 _localctx.s = <w.stateNumber>;
 <if(w.labels)><w.labels:{l | <labelref(l)> = }><endif>input.LT(1); input.consume();
@@ -354,6 +368,7 @@ ParserRuleContext() ::= "ParserRuleContext"
 RuleDynamicScopeStructName(ruleName) ::= "<ruleName>_stk"
 ImplicitTokenLabel(tokenName) ::= "_t<tokenName>"
 ImplicitRuleLabel(ruleName)   ::= "_r<ruleName>"
+ImplicitSetLabel(id)		  ::= "_tset<id>"
 ListLabelName(label)		  ::= "<label>_list"
 
 CaptureNextToken(d) ::= "<d.varName> = input.LT(1);"

tool/src/org/antlr/v4/Tool.java

@@ -206,8 +206,11 @@ public class Tool {
 			GrammarAST t = load(fileName);
 			if ( t instanceof GrammarASTErrorNode ) return; // came back as error node
 			if ( ((GrammarRootAST)t).hasErrors ) return;
-
 			GrammarRootAST ast = (GrammarRootAST)t;
+
+			GrammarTransformPipeline transform = new GrammarTransformPipeline(ast);
+			transform.process();
+
 			Grammar g = createGrammar(ast);
 			g.fileName = fileName;
 			process(g);

tool/src/org/antlr/v4/automata/ATNFactory.java

@@ -30,7 +30,6 @@
 package org.antlr.v4.automata;
 
 import org.antlr.v4.runtime.atn.*;
-import org.antlr.v4.runtime.misc.IntervalSet;
 import org.antlr.v4.tool.*;
 
 import java.util.List;
@@ -68,16 +67,13 @@ public interface ATNFactory {
 
 	Handle tokenRef(TerminalAST node);
 
-	/** From set build single edge graph o->o-set->o.  To conform to
+	Handle set(GrammarAST associatedAST, List<GrammarAST> terminals, boolean invert);
-     *  what an alt block looks like, must have extra state on left.
-     */
-	Handle set(IntervalSet set, GrammarAST associatedAST);
 
 	Handle tree(List<Handle> els);
 
 	Handle range(GrammarAST a, GrammarAST b);
 
-	Handle not(GrammarAST a);
+//	Handle not(GrammarAST a);
 
 	/** For a non-lexer, just build a simple token reference atom.
 	 *  For a lexer, a string is a sequence of char to match.  That is,
@@ -149,7 +145,7 @@ public interface ATNFactory {
      */
 	Handle block(BlockAST blockAST, GrammarAST ebnfRoot, List<Handle> alternativeGrips);
 
-	Handle notBlock(GrammarAST blockAST, List<GrammarAST> terminals);
+//	Handle notBlock(GrammarAST blockAST, Handle set);
 
 	/** From (A)? build either:
 	 *

tool/src/org/antlr/v4/automata/ATNPrinter.java

@@ -79,6 +79,11 @@ public class ATNPrinter {
 					ActionTransition a = (ActionTransition)t;
 					buf.append("-"+a.toString()+"->"+ getStateString(t.target)+'\n');
 				}
+				else if ( t instanceof SetTransition ) {
+					SetTransition st = (SetTransition)t;
+					boolean not = st instanceof NotSetTransition;
+					buf.append("-"+(not?"~":"")+st.toString()+"->"+ getStateString(t.target)+'\n');
+				}
 				else if ( t instanceof AtomTransition ) {
 					AtomTransition a = (AtomTransition)t;
 					String label = a.toString();

tool/src/org/antlr/v4/automata/ATNSerializer.java

@@ -87,7 +87,7 @@ public class ATNSerializer {
 				int edgeType = Transition.serializationTypes.get(t.getClass());
 				if ( edgeType == Transition.SET || edgeType == Transition.NOT_SET ) {
 					SetTransition st = (SetTransition)t;
-					sets.add(st.label);
+					sets.add(st.set);
 				}
 			}
 		}

tool/src/org/antlr/v4/automata/LexerATNFactory.java

@@ -29,8 +29,10 @@
 
 package org.antlr.v4.automata;
 
+import org.antlr.runtime.Token;
 import org.antlr.v4.misc.CharSupport;
 import org.antlr.v4.runtime.atn.*;
+import org.antlr.v4.runtime.misc.IntervalSet;
 import org.antlr.v4.tool.*;
 
 import java.util.List;
@@ -98,6 +100,27 @@ public class LexerATNFactory extends ParserATNFactory {
 		return new Handle(left, right);
 	}
 
+	@Override
+	public Handle set(GrammarAST associatedAST, List<GrammarAST> terminals, boolean invert) {
+		ATNState left = newState(associatedAST);
+		ATNState right = newState(associatedAST);
+		IntervalSet set = new IntervalSet();
+		for (GrammarAST t : terminals) {
+			int c = CharSupport.getCharValueFromGrammarCharLiteral(t.getText());
+			set.add(c);
+		}
+		if ( invert ) {
+			IntervalSet notSet = (IntervalSet)set.complement(Token.MIN_TOKEN_TYPE, g.getMaxTokenType());
+			left.transition = new NotSetTransition(set, notSet, right);
+		}
+		else {
+			left.transition = new SetTransition(set, right);
+		}
+		right.incidentTransition = left.transition;
+		associatedAST.atnState = left;
+		return new Handle(left, right);
+	}
+
 	/** For a lexer, a string is a sequence of char to match.  That is,
 	 *  "fog" is treated as 'f' 'o' 'g' not as a single transition in
 	 *  the DFA.  Machine== o-'f'->o-'o'->o-'g'->o and has n+1 states

tool/src/org/antlr/v4/automata/ParserATNFactory.java

@@ -110,11 +110,23 @@ public class ParserATNFactory implements ATNFactory {
 
 	/** From set build single edge graph o->o-set->o.  To conform to
      *  what an alt block looks like, must have extra state on left.
+	 *  This handles ~A also, converted to ~{A} set.
      */
-	public Handle set(IntervalSet set, GrammarAST associatedAST) {
+	public Handle set(GrammarAST associatedAST, List<GrammarAST> terminals, boolean invert) {
 		ATNState left = newState(associatedAST);
 		ATNState right = newState(associatedAST);
+		IntervalSet set = new IntervalSet();
+		for (GrammarAST t : terminals) {
+			int ttype = g.getTokenType(t.getText());
+			set.add(ttype);
+		}
+		if ( invert ) {
+			IntervalSet notSet = (IntervalSet)set.complement(Token.MIN_TOKEN_TYPE, g.getMaxTokenType());
+			left.transition = new NotSetTransition(set, notSet, right);
+		}
+		else {
 			left.transition = new SetTransition(set, right);
+		}
 		right.incidentTransition = left.transition;
 		associatedAST.atnState = left;
 		return new Handle(left, right);
@@ -128,6 +140,7 @@ public class ParserATNFactory implements ATNFactory {
 	public Handle range(GrammarAST a, GrammarAST b) { throw new UnsupportedOperationException(); }
 
 	/** ~atom only */
+	/*
 	public Handle not(GrammarAST node) {
 		ATNState left = newState(node);
 		ATNState right = newState(node);
@@ -137,6 +150,7 @@ public class ParserATNFactory implements ATNFactory {
 		node.atnState = left;
 		return new Handle(left, right);
 	}
+	*/
 
 	protected int getTokenType(GrammarAST atom) {
 		int ttype;
@@ -300,27 +314,12 @@ public class ParserATNFactory implements ATNFactory {
 		return h;
 	}
 
-	public Handle notBlock(GrammarAST notAST, List<GrammarAST> terminals) {
+//	public Handle notBlock(GrammarAST notAST, Handle set) {
-		// assume list of atoms
+//		SetTransition st = (SetTransition)set.left.transition;
-		IntervalSet notSet = new IntervalSet();
+//		set.left.transition = new NotSetTransition(st.label, set.right);
-		for (GrammarAST elemAST : terminals) {
+//		notAST.atnState = set.left;
-			if ( elemAST.getType()==ANTLRParser.RANGE ) {
+//		return set;
-				GrammarAST from = (GrammarAST)elemAST.getChild(0);
+//	}
-				GrammarAST to = (GrammarAST)elemAST.getChild(1);
-				notSet.add(getTokenType(from), getTokenType(to));
-			}
-			else {
-				notSet.add(getTokenType(elemAST));
-			}
-		}
-
-		ATNState left = newState(notAST);
-		ATNState right = newState(notAST);
-		left.transition = new NotSetTransition(notSet, right);
-		right.incidentTransition = left.transition;
-		notAST.atnState = left;
-		return new Handle(left, right);
-	}
 
 	public Handle alt(List<Handle> els) {
 		Handle prev = null;

tool/src/org/antlr/v4/codegen/BlankOutputModelFactory.java

@@ -63,6 +63,8 @@ public abstract class BlankOutputModelFactory implements OutputModelFactory {
 
 	public List<SrcOp> stringRef(GrammarAST ID, GrammarAST label, GrammarAST astOp) { return tokenRef(ID, label, null, astOp); }
 
+	public List<SrcOp> set(GrammarAST setAST, GrammarAST label, GrammarAST astOp, boolean invert) {	return null; }
+
 	// ACTIONS
 
 	public List<SrcOp> action(GrammarAST ast) { return null; }

tool/src/org/antlr/v4/codegen/CodeGeneratorExtension.java

@@ -66,6 +66,8 @@ public class CodeGeneratorExtension {
 
 	public List<SrcOp> tokenRef(List<SrcOp> ops) { return ops; }
 
+	public List<SrcOp> set(List<SrcOp> ops) { return ops; }
+
 	public List<SrcOp> stringRef(List<SrcOp> ops) { return ops; }
 
 	public List<SrcOp> wildcard(List<SrcOp> ops) { return ops; }
@@ -92,11 +94,15 @@ public class CodeGeneratorExtension {
 
 	public List<SrcOp> leafString(List<SrcOp> ops) { return ops; }
 
+	public List<SrcOp> rootSet(List<SrcOp> ops) { return ops; }
+
+	public List<SrcOp> leafSet(List<SrcOp> ops) { return ops; }
+
 	// BLOCKS
 
-	public List<SrcOp> getChoiceBlock(List<SrcOp> ops) { return ops; }
+	public Choice getChoiceBlock(Choice c) { return c; }
 
-	public List<SrcOp> getEBNFBlock(List<SrcOp> ops) { return ops; }
+	public Choice getEBNFBlock(Choice c) { return c; }
 
 	public boolean needsImplicitLabel(GrammarAST ID, LabeledOp op) { return false; }
 

tool/src/org/antlr/v4/codegen/OutputModelController.java

@@ -215,6 +215,22 @@ public class OutputModelController {
 		return ops;
 	}
 
+	/** (A|B|C) possibly with ebnfRoot and label */
+	public List<SrcOp> set(GrammarAST setAST, GrammarAST labelAST,
+						   GrammarAST astOp, boolean invert) {
+		List<SrcOp> ops = delegate.set(setAST, labelAST, astOp, invert);
+		for (CodeGeneratorExtension ext : extensions) {
+			ops = ext.set(ops);
+			if ( astOp!=null && astOp.getType()==ANTLRParser.ROOT ) {
+				ops = ext.rootSet(ops);
+			}
+			else if ( astOp==null ) {
+				ops = ext.leafSet(ops);
+			}
+		}
+		return ops;
+	}
+
 	public CodeBlockForAlt epsilon() {
 		CodeBlockForAlt blk = delegate.epsilon();
 		for (CodeGeneratorExtension ext : extensions) blk = ext.epsilon(blk);
@@ -259,15 +275,13 @@ public class OutputModelController {
 
 	public Choice getChoiceBlock(BlockAST blkAST, List<CodeBlockForAlt> alts, GrammarAST label) {
 		Choice c = delegate.getChoiceBlock(blkAST, alts, label);
-		List<SrcOp> ops = DefaultOutputModelFactory.list(c);
+		for (CodeGeneratorExtension ext : extensions) c = ext.getChoiceBlock(c);
-		for (CodeGeneratorExtension ext : extensions) ops = ext.getChoiceBlock(ops);
 		return c;
 	}
 
 	public Choice getEBNFBlock(GrammarAST ebnfRoot, List<CodeBlockForAlt> alts) {
 		Choice c = delegate.getEBNFBlock(ebnfRoot, alts);
-		List<SrcOp> ops = DefaultOutputModelFactory.list(c);
+		for (CodeGeneratorExtension ext : extensions) c = ext.getEBNFBlock(c);
-		for (CodeGeneratorExtension ext : extensions) ops = ext.getEBNFBlock(ops);
 		return c;
 	}
 

tool/src/org/antlr/v4/codegen/OutputModelFactory.java

@@ -70,6 +70,8 @@ public interface OutputModelFactory {
 
 	List<SrcOp> stringRef(GrammarAST ID, GrammarAST label, GrammarAST astOp);
 
+	List<SrcOp> set(GrammarAST setAST, GrammarAST label, GrammarAST astOp, boolean invert);
+
 	List<SrcOp> action(GrammarAST ast);
 
 	List<SrcOp> forcedAction(GrammarAST ast);

tool/src/org/antlr/v4/codegen/ParserASTExtension.java

@@ -45,13 +45,12 @@ public class ParserASTExtension extends CodeGeneratorExtension {
 	}
 
 	@Override
-	public List<SrcOp> getChoiceBlock(List<SrcOp> ops) {
+	public Choice getChoiceBlock(Choice choice) {
-		Choice choice = (Choice)Utils.find(ops, Choice.class);
 		Alternative alt = factory.getCurrentOuterMostAlt();
 		if ( alt.hasRewrite() && choice.label!=null ) {
 			trackExplicitLabel(choice.preamble, choice.label, choice);
 		}
-		return ops;
+		return choice;
 	}
 
 	@Override
@@ -168,11 +167,15 @@ public class ParserASTExtension extends CodeGeneratorExtension {
 		TokenDecl label = (TokenDecl)((LabeledOp)matchOp).getLabels().get(0);
 		// First declare tracking lists for elements, labels
 		// track the named element like _track_A
-		String elemListName = factory.getGenerator().target.getElementListName(matchOp.ast.getText());
+		String elemName = matchOp.ast.getText();
+		if ( matchOp.ast.getType()==ANTLRParser.SET ) {
+			elemName = String.valueOf(matchOp.ast.token.getTokenIndex());
+		}
+		String elemListName = factory.getGenerator().target.getElementListName(elemName);
 		blk.addLocalDecl(new ElementListDecl(factory, elemListName));
 		// Now, generate track instructions for element and any labels
 		// do element
-		String trackName = factory.getGenerator().target.getElementListName(matchOp.ast.getText());
+		String trackName = factory.getGenerator().target.getElementListName(elemName);
 		TrackTokenElement t = new TrackTokenElement(factory, matchOp.ast, trackName,
 													label);
 		ops.add(t);
@@ -180,6 +183,12 @@ public class ParserASTExtension extends CodeGeneratorExtension {
 		return ops;
 	}
 
+	@Override
+	public List<SrcOp> rootSet(List<SrcOp> ops) { return rootToken(ops); }
+
+	@Override
+	public List<SrcOp> leafSet(List<SrcOp> ops) { return leafToken(ops); }
+
 	@Override
 	public List<SrcOp> wildcard(List<SrcOp> ops) {
 		Wildcard wild = (Wildcard)Utils.find(ops, Wildcard.class);

tool/src/org/antlr/v4/codegen/ParserFactory.java

@@ -103,6 +103,28 @@ public class ParserFactory extends DefaultOutputModelFactory {
 		return list(matchOp, listLabelOp);
 	}
 
+	@Override
+	public List<SrcOp> set(GrammarAST setAST, GrammarAST labelAST,
+						   GrammarAST astOp, boolean invert)
+	{
+		LabeledOp matchOp;
+		if ( invert ) matchOp = new MatchNotSet(this, setAST);
+		else matchOp = new MatchSet(this, setAST);
+		if ( labelAST!=null ) {
+			String label = labelAST.getText();
+			TokenDecl d = new TokenDecl(this, label);
+			((MatchSet)matchOp).labels.add(d);
+			getCurrentRuleFunction().addContextDecl(d);
+			if ( labelAST.parent.getType() == ANTLRParser.PLUS_ASSIGN ) {
+				TokenListDecl l = new TokenListDecl(this, gen.target.getListLabel(label));
+				getCurrentRuleFunction().addContextDecl(l);
+			}
+		}
+		if ( controller.needsImplicitLabel(setAST, matchOp) ) defineImplicitLabel(setAST, matchOp);
+		AddToLabelList listLabelOp = getListLabelIfPresent(matchOp, labelAST);
+		return list(matchOp, listLabelOp);
+	}
+
 	@Override
 	public List<SrcOp> wildcard(GrammarAST ast, GrammarAST labelAST) {
 		Wildcard wild = new Wildcard(this, ast);
@@ -348,18 +370,24 @@ public class ParserFactory extends DefaultOutputModelFactory {
 		return op;
 	}
 
-	public void defineImplicitLabel(GrammarAST ID, LabeledOp op) {
+	public void defineImplicitLabel(GrammarAST ast, LabeledOp op) {
 		Decl d;
-		Rule r = g.getRule(ID.getText());
+		Rule r = g.getRule(ast.getText());
-		if ( r!=null ) {
+		if ( ast.getType()==ANTLRParser.SET ) {
-			String implLabel = gen.target.getImplicitRuleLabel(ID.getText());
+			String implLabel =
+				gen.target.getImplicitSetLabel(String.valueOf(ast.token.getTokenIndex()));
+			d = new TokenDecl(this, implLabel);
+			((TokenDecl)d).isImplicit = true;
+		}
+		else if ( r!=null ) {
+			String implLabel = gen.target.getImplicitRuleLabel(ast.getText());
 			String ctxName =
 				gen.target.getRuleFunctionContextStructName(r);
 			d = new RuleContextDecl(this, implLabel, ctxName);
 			((RuleContextDecl)d).isImplicit = true;
 		}
 		else {
-			String implLabel = gen.target.getImplicitTokenLabel(ID.getText());
+			String implLabel = gen.target.getImplicitTokenLabel(ast.getText());
 			d = new TokenDecl(this, implLabel);
 			((TokenDecl)d).isImplicit = true;
 		}

tool/src/org/antlr/v4/codegen/SourceGenTriggers.g

@@ -78,8 +78,8 @@ alternative returns [CodeBlockForAlt altCodeBlock, List<SrcOp> ops]
 
 element returns [List<? extends SrcOp> omos]
 	:	labeledElement					{$omos = $labeledElement.omos;}
-	|	atom[null]						{$omos = $atom.omos;}
+	|	atom[null,null,false]			{$omos = $atom.omos;}
-	|	ebnf							{$omos = $ebnf.omos;}
+	|	subrule							{$omos = $subrule.omos;}
 	|   ACTION							{$omos = controller.action($ACTION);}
 	|   FORCED_ACTION					{$omos = controller.forcedAction($FORCED_ACTION);}
 	|   SEMPRED							{$omos = controller.sempred($SEMPRED);}
@@ -88,9 +88,9 @@ element returns [List<? extends SrcOp> omos]
 	;
 
 labeledElement returns [List<? extends SrcOp> omos]
-	:	^(ASSIGN ID atom[$ID] )					{$omos = $atom.omos;}
+	:	^(ASSIGN ID atom[$ID,null,false] )			{$omos = $atom.omos;}
-	|	^(ASSIGN ID block[$ID,null,null])		{$omos = $block.omos;}
+	|	^(PLUS_ASSIGN ID atom[$ID,null,false])		{$omos = $atom.omos;}
-	|	^(PLUS_ASSIGN ID atom[$ID])				{$omos = $atom.omos;}
+	|	^(ASSIGN ID block[$ID,null,null] )			{$omos = $block.omos;}
 	|	^(PLUS_ASSIGN ID block[$ID,null,null])		{$omos = $block.omos;}
 	;
 
@@ -98,8 +98,8 @@ treeSpec returns [SrcOp omo]
     : ^(TREE_BEGIN  (e=element )+)
     ;
 
-ebnf returns [List<? extends SrcOp> omos]
+subrule returns [List<? extends SrcOp> omos]
-	:	^(astBlockSuffix block[null,null,null])
+	:	^(astBlockSuffix block[null,null,$astBlockSuffix.start]) {$omos = $block.omos;}
 	|	^(OPTIONAL block[null,$OPTIONAL,null])	{$omos = $block.omos;}
 	|	^(CLOSURE block[null,$CLOSURE,null])	{$omos = $block.omos;}
 	|	^(POSITIVE_CLOSURE block[null,$POSITIVE_CLOSURE,null])
@@ -113,29 +113,31 @@ astBlockSuffix
     | BANG
     ;
 
-// TODO: combine ROOT/BANG into one then just make new op ref'ing return value of atom/terminal...
+blockSet[GrammarAST label, GrammarAST astOp, boolean invert] returns [List<SrcOp> omos]
-// TODO: same for NOT
+    :	^(SET atom[null,null,false]+) {$omos = controller.set($SET, $label, $astOp, invert);}
-atom[GrammarAST label] returns [List<SrcOp> omos]
-	:	^(ROOT notSet[label, $ROOT])		{$omos = $notSet.omos;}
-	|	^(BANG notSet[label, $BANG])		{$omos = $notSet.omos;}
-	|	notSet[label, null]					{$omos = $notSet.omos;}
-	|	range[label]						{$omos = $range.omos;}
-	|	^(DOT ID terminal[label, null])
-	|	^(DOT ID ruleref[label, null])
-    |	^(WILDCARD .)						{$omos = controller.wildcard($WILDCARD, $label);}
-    |	WILDCARD							{$omos = controller.wildcard($WILDCARD, $label);}
-    |	^(ROOT terminal[label, $ROOT])		{$omos = $terminal.omos;}
-    |	^(BANG terminal[label, $BANG])		{$omos = $terminal.omos;}
-    |   terminal[label, null]				{$omos = $terminal.omos;}
-    |	^(ROOT ruleref[label, $ROOT])		{$omos = $ruleref.omos;}
-    |	^(BANG ruleref[label, $BANG])		{$omos = $ruleref.omos;}
-    |   ruleref[label, null]				{$omos = $ruleref.omos;}
     ;
 
-// TODO: send NOT to factory methods
+/*
-notSet[GrammarAST label, GrammarAST astOp] returns [List<SrcOp> omos]
+setElement
-    : ^(NOT terminal[label, astOp])
+	:	STRING_LITERAL
-    | ^(NOT block[label,null, astOp])
+	|	TOKEN_REF
+	|	^(RANGE STRING_LITERAL STRING_LITERAL)
+	;
+*/
+
+// TODO: combine ROOT/BANG into one then just make new op ref'ing return value of atom/terminal...
+// TODO: same for NOT
+atom[GrammarAST label, GrammarAST astOp, boolean invert] returns [List<SrcOp> omos]
+	:	^(op=(ROOT|BANG) a=atom[$label, $op, $invert] )	{$omos = $a.omos;}
+	|	^(NOT a=atom[$label, $astOp, true])		{$omos = $a.omos;}
+	|	range[label]							{$omos = $range.omos;}
+	|	^(DOT ID terminal[$label, null])
+	|	^(DOT ID ruleref[$label, null])
+    |	^(WILDCARD .)							{$omos = controller.wildcard($WILDCARD, $label);}
+    |	WILDCARD								{$omos = controller.wildcard($WILDCARD, $label);}
+    |   terminal[label, $astOp]					{$omos = $terminal.omos;}
+    |   ruleref[label, $astOp]					{$omos = $ruleref.omos;}
+	|	blockSet[$label, $astOp, invert]		{$omos = $blockSet.omos;}
 	;
 	
 ruleref[GrammarAST label, GrammarAST astOp] returns [List<SrcOp> omos]

tool/src/org/antlr/v4/codegen/Target.java

@@ -239,6 +239,13 @@ public class Target {
 		return st.render();
 	}
 
+	// x=(A|B)
+	public String getImplicitSetLabel(String id) {
+		ST st = gen.templates.getInstanceOf("ImplicitSetLabel");
+		st.add("id", id);
+		return st.render();
+	}
+
 	public String getImplicitRuleLabel(String ruleName) {
 		ST st = gen.templates.getInstanceOf("ImplicitRuleLabel");
 		st.add("ruleName", ruleName);

tool/src/org/antlr/v4/codegen/model/Choice.java

@@ -77,7 +77,7 @@ public abstract class Choice extends RuleElement {
 
 	public SrcOp addCodeForLookaheadTempVar(IntervalSet look) {
 		List<SrcOp> testOps = factory.getLL1Test(look, ast);
-		SrcOp expr = (SrcOp)Utils.find(testOps, TestSetInline.class);
+		SrcOp expr = (SrcOp) Utils.find(testOps, TestSetInline.class);
 		if ( expr instanceof TestSetInline) {
 			TestSetInline e = (TestSetInline)expr;
 			Decl d = new TokenTypeDecl(factory, e.varName);
@@ -87,5 +87,4 @@ public abstract class Choice extends RuleElement {
 		}
 		return expr;
 	}
-
 }

tool/src/org/antlr/v4/codegen/model/MatchNotSet.java

@@ -0,0 +1,39 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2011 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+
+ 1. Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+    derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.antlr.v4.codegen.model;
+
+import org.antlr.v4.codegen.OutputModelFactory;
+import org.antlr.v4.tool.GrammarAST;
+
+public class MatchNotSet extends MatchSet {
+	public MatchNotSet(OutputModelFactory factory, GrammarAST ast) {
+		super(factory, ast);
+	}
+}

tool/src/org/antlr/v4/codegen/model/MatchSet.java

@@ -0,0 +1,49 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2011 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+
+ 1. Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+    derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.antlr.v4.codegen.model;
+
+import org.antlr.v4.codegen.OutputModelFactory;
+import org.antlr.v4.codegen.model.decl.*;
+import org.antlr.v4.runtime.atn.SetTransition;
+import org.antlr.v4.tool.GrammarAST;
+
+public class MatchSet extends MatchToken {
+	@ModelElement public TestSetInline expr;
+	@ModelElement public CaptureNextTokenType capture;
+
+	public MatchSet(OutputModelFactory factory, GrammarAST ast) {
+		super(factory, ast);
+		SetTransition st = (SetTransition)ast.atnState.transition;
+		expr = new TestSetInline(factory, null, st.set);
+		Decl d = new TokenTypeDecl(factory, expr.varName);
+		factory.getCurrentRuleFunction().addLocalDecl(d);
+		capture = new CaptureNextTokenType(factory,expr.varName);
+	}
+}

tool/src/org/antlr/v4/codegen/model/MatchToken.java

@@ -48,5 +48,9 @@ public class MatchToken extends RuleElement implements LabeledOp {
 		name = gen.target.getTokenTypeAsTargetLabel(g, ttype);
 	}
 
+	public MatchToken(OutputModelFactory factory, GrammarAST ast) {
+		super(factory, ast);
+	}
+
 	public List<Decl> getLabels() { return labels; }
 }

tool/src/org/antlr/v4/codegen/model/RuleElement.java

@@ -40,4 +40,5 @@ public class RuleElement extends SrcOp {
 		super(factory, ast);
 		if ( ast.atnState!=null ) stateNumber = ast.atnState.stateNumber;
 	}
+
 }

tool/src/org/antlr/v4/codegen/model/SrcOp.java

@@ -35,7 +35,7 @@ import org.antlr.v4.tool.GrammarAST;
 /** */
 public abstract class SrcOp extends OutputModelObject {
 	/** Used to create unique var names etc... */
-//	public int uniqueID;
+	public int uniqueID; // TODO: do we need?
 
 	/** All operations know in which block they live:
 	 *
@@ -49,7 +49,7 @@ public abstract class SrcOp extends OutputModelObject {
 	public SrcOp(OutputModelFactory factory) { this(factory,null); }
 	public SrcOp(OutputModelFactory factory, GrammarAST ast) {
 		super(factory,ast);
-		//uniqueID = ast.token.getTokenIndex();
+		if ( ast!=null ) uniqueID = ast.token.getTokenIndex();
 		enclosingBlock = factory.getCurrentBlock();
 	}
 }

tool/src/org/antlr/v4/parse/ANTLRParser.g

@@ -69,6 +69,7 @@ tokens {
     POSITIVE_CLOSURE;
     SYNPRED;
     RANGE;
+    SET;
     CHAR_RANGE;
     EPSILON;
     ALT;
@@ -133,6 +134,10 @@ import org.antlr.v4.tool.*;
 
 @members {
 Stack paraphrases = new Stack();
+/** Affects tree construction; no SET collapsing if AST (ID|INT) would hide them from rewrite.
+ *  Could use for just AST ops, but we can't see -> until after building sets.
+boolean buildAST;
+ */
 }
 
 // The main entry point for parsing a V3 grammar from top to toe. This is
@@ -245,6 +250,13 @@ optionsSpec
 
 option
     :   id ASSIGN^ optionValue
+/*
+    	{
+    	if ( $id.text.equals("output") ) {
+    		if ( $optionValue.text.equals("AST") ) buildAST = true;
+    	}
+    	}
+    	*/
     ;
 
 // ------------
@@ -611,53 +623,24 @@ element
 		}
         reportError(re);
         recover(input,re);
-/*
-		input.rewind(m);
-		final List subset = input.get(input.index(), input.range());
-		System.out.println("failed to match as element: '"+subset);
-		CommonTokenStream ns = new CommonTokenStream(
-			new TokenSource() {
-				int i = 0;
-				public Token nextToken() {
-					if ( i>=subset.size() ) return Token.EOF_TOKEN;
-					return (Token)subset.get(i++);
-				}
-				public String getSourceName() { return null; }
-			});
-		ANTLRParser errorParser = new ANTLRParser(ns);
-		errorParser.setTreeAdaptor(this.adaptor);
-		errorParser.element_errors(re);
-        retval.tree = (GrammarAST)adaptor.errorNode(input, retval.start, input.LT(-1), re);
-        */
 	}
 
+labeledElement
+	:	id (ass=ASSIGN|ass=PLUS_ASSIGN)
+		(	atom						-> ^($ass id atom)
+		|	block (op=ROOT|op=BANG)?	-> {$op!=null}? ^($ass id ^($op block))
+										->				^($ass id block)
 /*
-element_errors[RecognitionException origError]
+		|	{buildAST}? blockSet
-options {backtrack=true;}
+			{
-@init {
+			RecognitionException e =
-int m = input.mark();
+				new v4ParserException("can't  '"+
-//state.backtracking++;
+									  input.LT(1).getText()+" "+input.LT(2).getText()+"'", input);
-}
+			reportError(missingSemi);
-@after {
-//state.backtracking--;
-}
-	:	(	DOC_COMMENT? ruleModifiers? id ARG_ACTION<ActionAST>? ruleReturns? rulePrequel* COLON
-		|	exceptionGroup
-		)
-		{reportError(missingSemi); recover(input,null);}
-	;
-	catch [RecognitionException ignore]	{
-		input.rewind(m);
-		input.consume(); // kill at least one token
-		reportError(origError);
-		BitSet followSet = computeErrorRecoverySet();
-		beginResync();
-		consumeUntil(input, followSet);
-		endResync();
 			}
 */
-
+		)
-labeledElement : id (ASSIGN^|PLUS_ASSIGN^) (atom|block) ;
+	;
 
 // Tree specifying alt
 // Tree grammars need to have alts that describe a tree structure they
@@ -682,20 +665,19 @@ ebnf
     : block
       // And now we see if we have any of the optional suffixs and rewrite
       // the AST for this rule accordingly
-      //
+      (	blockSuffix	-> ^(blockSuffix block)
-      (	blockSuffixe	-> ^(blockSuffixe block)
       |				-> block
       )
     ;
 
 // The standard EBNF suffixes with additional components that make
 // sense only to ANTLR, in the context of a grammar block.
-blockSuffixe
+blockSuffix
     : ebnfSuffix // Standard EBNF
 
 	  // ANTLR Specific Suffixes
     | ROOT
-    | IMPLIES   // We will change this to syn/sem pred in the next phase
+//    | IMPLIES   // We will change this to syn/sem pred in the next phase
     | BANG
     ;
 
@@ -720,7 +702,7 @@ atom:	// Qualified reference delegate.rule. This must be
     |   ruleref
     |	notSet   (ROOT^|BANG^)?
 	|   // Wildcard '.' means any character in a lexer, any
-		// token in parser and any token or node in a tree parser
+		// token in parser and any node or subtree in a tree parser
 		// Because the terminal rule is allowed to be the node
 		// specification for the start of a tree rule, we must
 		// later check that wildcard was not used for that.
@@ -731,24 +713,34 @@ atom:	// Qualified reference delegate.rule. This must be
 // --------------------
 // Inverted element set
 //
-// A set of characters (in a lexer) or terminal tokens, if a parser
+// A set of characters (in a lexer) or terminal tokens, if a parser,
 // that are then used to create the inverse set of them.
-//
 notSet
-    : NOT terminal	-> ^(NOT terminal)
+    : NOT setElement	-> ^(NOT ^(SET[$setElement.start,"SET"] setElement))
     | NOT blockSet		-> ^(NOT blockSet)
     ;
 
 blockSet
-    : LPAREN
+@init {
-    	setElement (OR setElement)*
+	Token t;
-      RPAREN
+	boolean ebnf = false;
-      -> ^(BLOCK<BlockAST>[$LPAREN,"BLOCK"] setElement+ )
+}
+    :	LPAREN setElement (OR setElement)* RPAREN
+/*		{
+		t = input.LT(1);
+		ebnf = t!=null && (t.getType()==QUESTION || t.getType()==STAR || t.getType()==PLUS);
+	    }
+	    */
+		-> ^(BLOCK<BlockAST>[$LPAREN,"BLOCK"] ^(ALT setElement)+ )
+/*
+		-> {ebnf}?	^(BLOCK<BlockAST>[$LPAREN,"BLOCK"] ^(ALT ^(SET[$LPAREN,"SET"] setElement+ )))
+		-> 			^(SET[$LPAREN,"SET"] setElement+ )
+*/
     ;
 
 setElement
-	:	range
+	:	TOKEN_REF<TerminalAST>
-	|	terminal
+	|	STRING_LITERAL<TerminalAST>
 	;
 
 // -------------
@@ -760,8 +752,6 @@ setElement
 //
 block
  	:	LPAREN
-         // A new blocked altlist may have a set of options set sepcifically
-         // for it.
         ( optionsSpec? ra+=ruleAction* COLON )?
         altList
 		RPAREN

tool/src/org/antlr/v4/parse/ASTVerifier.g

@@ -256,26 +256,34 @@ elements
 element
 	:	labeledElement
 	|	atom
-	|	ebnf
+	|	subrule
 	|   ACTION
     |   FORCED_ACTION
 	|   SEMPRED
 	|	GATED_SEMPRED
 	|	treeSpec
+	|	^(ROOT astOperand)
+	|	^(BANG astOperand)
+	|	^(NOT blockSet)
+	|	^(NOT block)
+	;
+
+astOperand
+	:	atom
+	|	^(NOT blockSet)
+	|	^(NOT block)
 	;
 
 labeledElement
-	:	^(ASSIGN ID atom)
+	:	^((ASSIGN|PLUS_ASSIGN) ID element)
-	|	^(ASSIGN ID block)
-	|	^(PLUS_ASSIGN ID atom)
-	|	^(PLUS_ASSIGN ID block)
 	;
 
 treeSpec
     : ^(TREE_BEGIN element+)
     ;
 
-ebnf:	^(blockSuffix block)
+subrule
+	:	^(blockSuffix block)
 	| 	block
     ;
 
@@ -292,33 +300,23 @@ ebnfSuffix
    	|	POSITIVE_CLOSURE
 	;
 
-atom:	^(ROOT notSet)
+atom:	range
-	|	^(BANG notSet)
-	|	notSet
-    |	^(ROOT terminal)
-    |	^(BANG terminal)
-	|	range
 	|	^(DOT ID terminal)
 	|	^(DOT ID ruleref)
     |	^(WILDCARD elementOptions)
     |	WILDCARD
     |   terminal
+    |	blockSet
     |   ruleref
     ;
 
-notSet
-    : ^(NOT setElement)
-    | ^(NOT blockSet)
-    ;
-
 blockSet
-    :	^(BLOCK setElement+)
+	:	^(SET setElement+)
 	;
 	
 setElement
 	:	STRING_LITERAL
 	|	TOKEN_REF
-	|	^(RANGE STRING_LITERAL STRING_LITERAL)
 	;
 
 block
@@ -326,9 +324,7 @@ block
     ;
 
 ruleref
-    :	^(ROOT ^(RULE_REF ARG_ACTION?))
+    :	^(RULE_REF ARG_ACTION?)
-    |	^(BANG ^(RULE_REF ARG_ACTION?))
-    |	^(RULE_REF ARG_ACTION?)
     ;
 
 range

tool/src/org/antlr/v4/parse/ATNBuilder.g

@@ -92,19 +92,25 @@ alternative returns [ATNFactory.Handle p]
 element returns [ATNFactory.Handle p]
 	:	labeledElement				{$p = $labeledElement.p;}
 	|	atom						{$p = $atom.p;}
-	|	ebnf						{$p = $ebnf.p;}
+	|	subrule						{$p = $subrule.p;}
 	|   ACTION						{$p = factory.action((ActionAST)$ACTION);}
 	|   FORCED_ACTION				{$p = factory.action((ActionAST)$FORCED_ACTION);}
 	|   SEMPRED						{$p = factory.sempred((PredAST)$SEMPRED);}
 	|	GATED_SEMPRED				{$p = factory.gated_sempred($GATED_SEMPRED);}
 	|	treeSpec					{$p = $treeSpec.p;}
+	|	^(ROOT a=astOperand)		{$p = $a.p;}
+	|	^(BANG a=astOperand)		{$p = $a.p;}
+    |	^(NOT b=blockSet[true])		{$p = $b.p;}
+	;
+	
+astOperand returns [ATNFactory.Handle p]
+	:	atom						{$p = $atom.p;}
+	|	^(NOT blockSet[true])		{$p = $blockSet.p;}
 	;
 
 labeledElement returns [ATNFactory.Handle p]
-	:	^(ASSIGN ID atom)			    {$p = factory.label($atom.p);}
+	:	^(ASSIGN ID element)	    {$p = factory.label($element.p);}
-	|	^(ASSIGN ID block[null])		{$p = factory.label($block.p);}
+	|	^(PLUS_ASSIGN ID element)   {$p = factory.listLabel($element.p);}
-	|	^(PLUS_ASSIGN ID atom)		    {$p = factory.listLabel($atom.p);}
-	|	^(PLUS_ASSIGN ID block[null])	{$p = factory.listLabel($block.p);}
 	;
 
 treeSpec returns [ATNFactory.Handle p]
@@ -112,7 +118,7 @@ treeSpec returns [ATNFactory.Handle p]
     : ^(TREE_BEGIN  (e=element {els.add($e.p);})+)	{$p = factory.tree(els);}
     ;
 
-ebnf returns [ATNFactory.Handle p]
+subrule returns [ATNFactory.Handle p]
 	:	^(astBlockSuffix block[null])		{$p = $block.p;}
 	|	^(OPTIONAL block[$start])			{$p = $block.p;}
 	|	^(CLOSURE block[$start])			{$p = $block.p;}
@@ -120,6 +126,11 @@ ebnf returns [ATNFactory.Handle p]
 	| 	block[null] 						{$p = $block.p;}
     ;
 
+blockSet[boolean invert] returns [ATNFactory.Handle p]
+@init {List<GrammarAST> alts = new ArrayList<GrammarAST>();}
+	:	^(SET (atom {alts.add($atom.start);})+) {$p = factory.set($start, alts, $invert);}
+	;
+
 astBlockSuffix
     : ROOT
     | IMPLIES
@@ -127,40 +138,18 @@ astBlockSuffix
     ;
 
 atom returns [ATNFactory.Handle p]
-	:	^(ROOT range)			{$p = $range.p;}
+	:	range					{$p = $range.p;}
-	|	^(BANG range)			{$p = $range.p;}
-	|	^(ROOT notSet)			{$p = $notSet.p;}
-	|	^(BANG notSet)			{$p = $notSet.p;}
-	|	notSet					{$p = $notSet.p;}
-	|	range					{$p = $range.p;}
 	|	^(DOT ID terminal)		{$p = $terminal.p;}
 	|	^(DOT ID ruleref)		{$p = $ruleref.p;}
     |	^(WILDCARD .)			{$p = factory.wildcard($start);}
     |	WILDCARD				{$p = factory.wildcard($start);}
+    |	blockSet[false]			{$p = $blockSet.p;}
     |   terminal				{$p = $terminal.p;}
     |   ruleref					{$p = $ruleref.p;}
     ;
 
-notSet returns [ATNFactory.Handle p]
-    : ^(NOT setElement)		{$p = factory.not($NOT);}
-    | ^(NOT blockSet)		{$p = factory.notBlock($NOT, $blockSet.alts);}
-    ;
-
-blockSet returns [List<GrammarAST> alts]
-@init {$alts = new ArrayList<GrammarAST>();}
-    :	^(BLOCK (t=setElement {$alts.add($t.start);})+)
-    ;
-
-setElement
-	:	STRING_LITERAL
-	|	TOKEN_REF
-	|	^(RANGE STRING_LITERAL STRING_LITERAL)
-	;
-
 ruleref returns [ATNFactory.Handle p]
-    :	^(ROOT ^(RULE_REF ARG_ACTION?))	{$p = factory.ruleRef($RULE_REF);}
+    :	^(RULE_REF ARG_ACTION?)			{$p = factory.ruleRef($RULE_REF);}
-    |	^(BANG ^(RULE_REF ARG_ACTION?))	{$p = factory.ruleRef($RULE_REF);}
-    |	^(RULE_REF ARG_ACTION?)			{$p = factory.ruleRef($RULE_REF);}
     ;
 
 range returns [ATNFactory.Handle p]
@@ -173,6 +162,4 @@ terminal returns [ATNFactory.Handle p]
     |	^(TOKEN_REF ARG_ACTION .)	{$p = factory.tokenRef((TerminalAST)$start);}
     |	^(TOKEN_REF .)				{$p = factory.tokenRef((TerminalAST)$start);}
     |	TOKEN_REF					{$p = factory.tokenRef((TerminalAST)$start);}
-    |	^(ROOT t=terminal)			{$p = $t.p;}
-    |	^(BANG t=terminal)			{$p = $t.p;}
     ;

tool/src/org/antlr/v4/parse/BlockSetTransformer.g

@@ -0,0 +1,68 @@
+tree grammar BlockSetTransformer;
+options {
+	language     = Java;
+	tokenVocab   = ANTLRParser;
+	ASTLabelType = GrammarAST;
+	output		 = AST;
+	filter		 = true;
+}
+
+@header {
+package org.antlr.v4.parse;
+import org.antlr.v4.misc.Utils;
+import org.antlr.v4.tool.*;
+import java.util.List;
+import java.util.ArrayList;
+import org.antlr.v4.runtime.misc.IntervalSet;
+}
+
+@members {
+public String currentRuleName;
+public GrammarAST currentAlt;
+}
+
+topdown
+    :	^(RULE ID {currentRuleName=$ID.text;} .+)
+    |	setAlt
+    |	ebnfBlockSet
+    |	blockSet
+	;
+
+setAlt
+	:	{inContext("RULE BLOCK")}? ( ALT | ALT_REWRITE )
+		{currentAlt = (AltAST)$start;}
+	;
+
+// (BLOCK (ALT (+ (BLOCK (ALT INT) (ALT ID)))))
+ebnfBlockSet
+	:	^(ebnfSuffix blockSet) -> ^(ebnfSuffix ^(BLOCK<BlockAST> ^(ALT blockSet)))
+	;
+	
+ebnfSuffix
+@after {$tree = (GrammarAST)adaptor.dupNode($start);}
+	:	OPTIONAL
+  	|	CLOSURE
+   	|	POSITIVE_CLOSURE
+	;
+	
+blockSet
+@init {
+	if ( currentAlt!=null && currentAlt.getType()==ANTLRParser.ALT_REWRITE ) {
+		IntervalSet s = new IntervalSet();
+		s.add(RULE_REF);
+		s.add(STRING_LITERAL);
+		s.add(TOKEN_REF);
+		List<GrammarAST> elems = currentAlt.getNodesWithType(s);
+		System.out.println("stuff in rewrite: "+elems);
+	}
+}
+	:	{Character.isLowerCase(currentRuleName.charAt(0)) &&
+		!inContext("ALT_REWRITE ...") && !inContext("RULE")}?
+		^(BLOCK ( ^(ALT setElement) )+) -> ^(SET[$BLOCK.token, "SET"] setElement+)
+	;
+	
+setElement
+@after {$tree = new TerminalAST($start);}
+	:	STRING_LITERAL
+	|	TOKEN_REF
+	;

tool/src/org/antlr/v4/semantics/SemanticPipeline.java

@@ -71,7 +71,9 @@ public class SemanticPipeline {
 		ASTVerifier walker = new ASTVerifier(nodes);
 		try {walker.grammarSpec();}
 		catch (RecognitionException re) {
-			ErrorManager.fatalInternalError("bad grammar AST structure", re);
+			ErrorManager.fatalInternalError("bad grammar AST structure: "+
+											g.ast.toStringTree(),
+											re);
 		}
 
 		// DO BASIC / EASY SEMANTIC CHECKS

tool/src/org/antlr/v4/tool/Grammar.java

@@ -190,6 +190,9 @@ public class Grammar implements AttributeResolver {
 			this.ast = (GrammarRootAST)r.getTree();
 			this.ast.hasErrors = p.getNumberOfSyntaxErrors()>0;
 			this.name = ((GrammarAST)ast.getChild(0)).getText();
+
+			GrammarTransformPipeline transform = new GrammarTransformPipeline(ast);
+			transform.process();
 		}
 		initTokenSymbolTables();
     }

tool/src/org/antlr/v4/tool/GrammarAST.java

@@ -56,6 +56,7 @@ public class GrammarAST extends CommonTree {
 		token.setInputStream(t.getInputStream());
 		token.setLine(t.getLine());
 		token.setCharPositionInLine(t.getCharPositionInLine());
+		token.setTokenIndex(t.getTokenIndex());
     }
 
 	public List<GrammarAST> getNodesWithType(int ttype) {

tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java

@@ -0,0 +1,54 @@
+/*
+ [The "BSD license"]
+ Copyright (c) 2011 Terence Parr
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+
+ 1. Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+    derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.antlr.v4.tool;
+
+import org.antlr.v4.parse.*;
+
+/** Handle left-recursion and block-set transforms */
+public class GrammarTransformPipeline {
+	public GrammarAST ast;
+
+	public GrammarTransformPipeline(GrammarAST ast) {
+		this.ast = ast;
+	}
+
+	public void process() {
+		if ( ast==null ) return;
+
+		org.antlr.runtime.tree.CommonTreeNodeStream nodes =
+			new org.antlr.runtime.tree.CommonTreeNodeStream(ast);
+		GrammarASTAdaptor adaptor = new GrammarASTAdaptor();
+		BlockSetTransformer transformer = new BlockSetTransformer(nodes);
+		transformer.setTreeAdaptor(adaptor);
+		System.out.println("before: "+ast.toStringTree());
+		transformer.downup(ast);
+		System.out.println("after: "+ast.toStringTree());
+	}
+}

tool/test/org/antlr/v4/test/TestASTOps.java

@@ -65,7 +65,7 @@ public class TestASTOps extends BaseTest {
 		String grammar =
 			"grammar foo;\n" +
 			"options {output=AST;}\n" +
-			"a : (ID|INT) ;\n" +
+			"a : (ID{;}|INT) ;\n" +
 			"ID : 'a'..'z'+ ;\n" +
 			"INT : '0'..'9'+;\n" +
 			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
@@ -496,6 +496,7 @@ public class TestASTOps extends BaseTest {
 			"blort : '+' ;\n" +
 			"ID : 'a'..'z'+ ;\n" +
 			"INT : '0'..'9'+;\n" +
+			"PLUS : '+';\n" +
 			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
 		String found = execParser("T.g", grammar, "TParser", "TLexer",
 								  "a", "3+4+5", debug);

tool/test/org/antlr/v4/test/TestASTStructure.gunit

@@ -123,6 +123,12 @@ rule:
 block:
 	"( ^(A B) | ^(b C) )"	-> (BLOCK (ALT ("^(" A B)) (ALT ("^(" b C)))
 
+ebnf:
+	"(A|B)"					-> (SET A B)
+	"(A|B)?"				-> ("?" (BLOCK (ALT (SET A B))))
+	"(A|B)*"				-> ("*" (BLOCK (ALT (SET A B))))
+	"(A|B)+"				-> ("+" (BLOCK (ALT (SET A B))))
+
 alternative:
 	"x+=ID* -> $x*" ->
 	    (ALT_REWRITE
@@ -189,6 +195,7 @@ alternative:
     (-> EPSILON))
 
 element:
+	"~A"		-> (~ (SET A))
 	"b+"		-> (+ (BLOCK (ALT b)))
 	"(b)+"		-> (+ (BLOCK (ALT b)))
 	"b?"  		-> (? (BLOCK (ALT b)))
@@ -203,10 +210,11 @@ element:
 	"x=ID?"		-> (? (BLOCK (ALT (= x ID))))
 	"x=ID*"		-> (* (BLOCK (ALT (= x ID))))
 	"x=b"		-> (= x b)
-	"x=(A|B)"	-> (= x (BLOCK (ALT A) (ALT B)))
+	"x=(A|B)"	-> (= x (SET A B))
-	"x=~(A|B)"	-> (= x (~ (BLOCK A B)))
+	"x=(A|B)^"	-> (= x (^ (SET A B)))
-	"x+=~(A|B)"	-> (+= x (~ (BLOCK A B)))
+	"x=~(A|B)"	-> (= x (~ (SET A B)))
-	"x+=~(A|B)+"-> (+ (BLOCK (ALT (+= x (~ (BLOCK A B))))))
+	"x+=~(A|B)"	-> (+= x (~ (SET A B)))
+	"x+=~(A|B)+"-> (+ (BLOCK (ALT (+= x (~ (BLOCK (ALT (SET A B))))))))
 	"x=b+"		-> (+ (BLOCK (ALT (= x b))))
 	"x+=ID*"	-> (* (BLOCK (ALT (+= x ID))))
 	"x+='int'*"	-> (* (BLOCK (ALT (+= x 'int'))))

tool/test/org/antlr/v4/test/TestASTStructure.java

@@ -99,327 +99,373 @@ public class TestASTStructure extends org.antlr.v4.gunit.gUnitBase {
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(BLOCK (ALT (^( A B)) (ALT (^( b C)))";
 		assertEquals("testing rule block", expecting, actual);
-	} @Test public void test_alternative1() throws Exception {
+	} @Test public void test_ebnf1() throws Exception {
 		// gunit test on line 127
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "x+=ID* -> $x*", 127);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("ebnf", "(A|B)", 127);
+		Object actual = ((Tree)rstruct.getTree()).toStringTree();
+		Object expecting = "(SET A B)";
+		assertEquals("testing rule ebnf", expecting, actual);
+	}
+
+	@Test public void test_ebnf2() throws Exception {
+		// gunit test on line 128
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("ebnf", "(A|B)?", 128);
+		Object actual = ((Tree)rstruct.getTree()).toStringTree();
+		Object expecting = "(? (BLOCK (ALT (SET A B))))";
+		assertEquals("testing rule ebnf", expecting, actual);
+	}
+
+	@Test public void test_ebnf3() throws Exception {
+		// gunit test on line 129
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("ebnf", "(A|B)*", 129);
+		Object actual = ((Tree)rstruct.getTree()).toStringTree();
+		Object expecting = "(* (BLOCK (ALT (SET A B))))";
+		assertEquals("testing rule ebnf", expecting, actual);
+	}
+
+	@Test public void test_ebnf4() throws Exception {
+		// gunit test on line 130
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("ebnf", "(A|B)+", 130);
+		Object actual = ((Tree)rstruct.getTree()).toStringTree();
+		Object expecting = "(+ (BLOCK (ALT (SET A B))))";
+		assertEquals("testing rule ebnf", expecting, actual);
+	} @Test public void test_alternative1() throws Exception {
+		// gunit test on line 133
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "x+=ID* -> $x*", 133);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(ALT_REWRITE (ALT (* (BLOCK (ALT (+= x ID))))) (-> (ALT (* (REWRITE_BLOCK (ALT x))))))";
 		assertEquals("testing rule alternative", expecting, actual);
 	}
 
 	@Test public void test_alternative2() throws Exception {
-		// gunit test on line 132
+		// gunit test on line 138
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "A -> ...", 132);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "A -> ...", 138);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(ALT_REWRITE (ALT A) (-> ...))";
 		assertEquals("testing rule alternative", expecting, actual);
 	}
 
 	@Test public void test_alternative3() throws Exception {
-		// gunit test on line 133
+		// gunit test on line 139
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "A -> ", 133);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "A -> ", 139);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(ALT_REWRITE (ALT A) (-> EPSILON))";
 		assertEquals("testing rule alternative", expecting, actual);
 	}
 
 	@Test public void test_alternative4() throws Exception {
-		// gunit test on line 135
+		// gunit test on line 141
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "A -> foo(a={x}, b={y})", 135);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "A -> foo(a={x}, b={y})", 141);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(ALT_REWRITE (ALT A) (-> (TEMPLATE foo (ARGLIST (= a {x}) (= b {y})))))";
 		assertEquals("testing rule alternative", expecting, actual);
 	}
 
 	@Test public void test_alternative5() throws Exception {
-		// gunit test on line 140
+		// gunit test on line 146
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "A -> template(a={x}, b={y}) <<ick>>", 140);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "A -> template(a={x}, b={y}) <<ick>>", 146);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(ALT_REWRITE (ALT A) (-> (TEMPLATE (ARGLIST (= a {x}) (= b {y})) <<ick>>)))";
 		assertEquals("testing rule alternative", expecting, actual);
 	}
 
 	@Test public void test_alternative6() throws Exception {
-		// gunit test on line 145
+		// gunit test on line 151
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "A -> ({name})()", 145);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "A -> ({name})()", 151);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(ALT_REWRITE (ALT A) (-> (TEMPLATE {name})))";
 		assertEquals("testing rule alternative", expecting, actual);
 	}
 
 	@Test public void test_alternative7() throws Exception {
-		// gunit test on line 147
+		// gunit test on line 153
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "A -> {expr}", 147);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "A -> {expr}", 153);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(ALT_REWRITE (ALT A) (-> (ALT {expr})))";
 		assertEquals("testing rule alternative", expecting, actual);
 	}
 
 	@Test public void test_alternative8() throws Exception {
-		// gunit test on line 149
+		// gunit test on line 155
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "\n    A -> {p1}? {e1}\n    -> {e2}\n    ->\n    ", 149);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "\n    A -> {p1}? {e1}\n    -> {e2}\n    ->\n    ", 155);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(ALT_REWRITE (ALT A) (-> {p1}? (ALT {e1})) (-> (ALT {e2})))";
 		assertEquals("testing rule alternative", expecting, actual);
 	}
 
 	@Test public void test_alternative9() throws Exception {
-		// gunit test on line 161
+		// gunit test on line 167
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "A -> A", 161);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "A -> A", 167);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(ALT_REWRITE (ALT A) (-> (ALT A)))";
 		assertEquals("testing rule alternative", expecting, actual);
 	}
 
 	@Test public void test_alternative10() throws Exception {
-		// gunit test on line 163
+		// gunit test on line 169
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "a -> a", 163);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "a -> a", 169);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(ALT_REWRITE (ALT a) (-> (ALT a)))";
 		assertEquals("testing rule alternative", expecting, actual);
 	}
 
 	@Test public void test_alternative11() throws Exception {
-		// gunit test on line 165
+		// gunit test on line 171
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "a A X? Y* -> A a ^(TOP X)? Y*", 165);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "a A X? Y* -> A a ^(TOP X)? Y*", 171);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(ALT_REWRITE (ALT a A (? (BLOCK (ALT X))) (* (BLOCK (ALT Y)))) (-> (ALT A a (? (REWRITE_BLOCK (ALT (^( TOP X)))) (* (REWRITE_BLOCK (ALT Y))))))";
 		assertEquals("testing rule alternative", expecting, actual);
 	}
 
 	@Test public void test_alternative12() throws Exception {
-		// gunit test on line 173
+		// gunit test on line 179
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "A -> A[33]", 173);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "A -> A[33]", 179);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(ALT_REWRITE (ALT A) (-> (ALT (A 33))))";
 		assertEquals("testing rule alternative", expecting, actual);
 	}
 
 	@Test public void test_alternative13() throws Exception {
-		// gunit test on line 175
+		// gunit test on line 181
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "A -> 'int' ^(A A)*", 175);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "A -> 'int' ^(A A)*", 181);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(ALT_REWRITE (ALT A) (-> (ALT 'int' (* (REWRITE_BLOCK (ALT (^( A A)))))))";
 		assertEquals("testing rule alternative", expecting, actual);
 	}
 
 	@Test public void test_alternative14() throws Exception {
-		// gunit test on line 180
+		// gunit test on line 186
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "\n    A -> {p1}? A\n      -> {p2}? B\n      ->\n    ", 180);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("alternative", "\n    A -> {p1}? A\n      -> {p2}? B\n      ->\n    ", 186);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(ALT_REWRITE (ALT A) (-> {p1}? (ALT A)) (-> {p2}? (ALT B)) (-> EPSILON))";
 		assertEquals("testing rule alternative", expecting, actual);
 	} @Test public void test_element1() throws Exception {
-		// gunit test on line 192
+		// gunit test on line 198
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "b+", 192);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "~A", 198);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
-		Object expecting = "(+ (BLOCK (ALT b)))";
+		Object expecting = "(~ (SET A))";
 		assertEquals("testing rule element", expecting, actual);
 	}
 
 	@Test public void test_element2() throws Exception {
-		// gunit test on line 193
+		// gunit test on line 199
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "(b)+", 193);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "b+", 199);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(+ (BLOCK (ALT b)))";
 		assertEquals("testing rule element", expecting, actual);
 	}
 
 	@Test public void test_element3() throws Exception {
-		// gunit test on line 194
+		// gunit test on line 200
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "b?", 194);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "(b)+", 200);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
-		Object expecting = "(? (BLOCK (ALT b)))";
+		Object expecting = "(+ (BLOCK (ALT b)))";
 		assertEquals("testing rule element", expecting, actual);
 	}
 
 	@Test public void test_element4() throws Exception {
-		// gunit test on line 195
+		// gunit test on line 201
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "(b)?", 195);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "b?", 201);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(? (BLOCK (ALT b)))";
 		assertEquals("testing rule element", expecting, actual);
 	}
 
 	@Test public void test_element5() throws Exception {
-		// gunit test on line 196
+		// gunit test on line 202
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "(b)*", 196);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "(b)?", 202);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
-		Object expecting = "(* (BLOCK (ALT b)))";
+		Object expecting = "(? (BLOCK (ALT b)))";
 		assertEquals("testing rule element", expecting, actual);
 	}
 
 	@Test public void test_element6() throws Exception {
-		// gunit test on line 197
+		// gunit test on line 203
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "b*", 197);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "(b)*", 203);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(* (BLOCK (ALT b)))";
 		assertEquals("testing rule element", expecting, actual);
 	}
 
 	@Test public void test_element7() throws Exception {
-		// gunit test on line 198
+		// gunit test on line 204
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "'while'*", 198);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "b*", 204);
+		Object actual = ((Tree)rstruct.getTree()).toStringTree();
+		Object expecting = "(* (BLOCK (ALT b)))";
+		assertEquals("testing rule element", expecting, actual);
+	}
+
+	@Test public void test_element8() throws Exception {
+		// gunit test on line 205
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "'while'*", 205);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(* (BLOCK (ALT 'while')))";
 		assertEquals("testing rule element", expecting, actual);
 	}
 
-	@Test public void test_element8() throws Exception {
+	@Test public void test_element9() throws Exception {
-		// gunit test on line 199
+		// gunit test on line 206
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "'a'+", 199);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "'a'+", 206);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(+ (BLOCK (ALT 'a')))";
 		assertEquals("testing rule element", expecting, actual);
 	}
 
-	@Test public void test_element9() throws Exception {
+	@Test public void test_element10() throws Exception {
-		// gunit test on line 200
+		// gunit test on line 207
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "a[3]", 200);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "a[3]", 207);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(a 3)";
 		assertEquals("testing rule element", expecting, actual);
 	}
 
-	@Test public void test_element10() throws Exception {
+	@Test public void test_element11() throws Exception {
-		// gunit test on line 201
+		// gunit test on line 208
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "'a'..'z'+", 201);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "'a'..'z'+", 208);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(+ (BLOCK (ALT (.. 'a' 'z'))))";
 		assertEquals("testing rule element", expecting, actual);
 	}
 
-	@Test public void test_element11() throws Exception {
+	@Test public void test_element12() throws Exception {
-		// gunit test on line 202
+		// gunit test on line 209
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x=ID", 202);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x=ID", 209);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(= x ID)";
 		assertEquals("testing rule element", expecting, actual);
 	}
 
-	@Test public void test_element12() throws Exception {
+	@Test public void test_element13() throws Exception {
-		// gunit test on line 203
+		// gunit test on line 210
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x=ID?", 203);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x=ID?", 210);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(? (BLOCK (ALT (= x ID))))";
 		assertEquals("testing rule element", expecting, actual);
 	}
 
-	@Test public void test_element13() throws Exception {
+	@Test public void test_element14() throws Exception {
-		// gunit test on line 204
+		// gunit test on line 211
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x=ID*", 204);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x=ID*", 211);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(* (BLOCK (ALT (= x ID))))";
 		assertEquals("testing rule element", expecting, actual);
 	}
 
-	@Test public void test_element14() throws Exception {
+	@Test public void test_element15() throws Exception {
-		// gunit test on line 205
+		// gunit test on line 212
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x=b", 205);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x=b", 212);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(= x b)";
 		assertEquals("testing rule element", expecting, actual);
 	}
 
-	@Test public void test_element15() throws Exception {
-		// gunit test on line 206
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x=(A|B)", 206);
-		Object actual = ((Tree)rstruct.getTree()).toStringTree();
-		Object expecting = "(= x (BLOCK (ALT A) (ALT B)))";
-		assertEquals("testing rule element", expecting, actual);
-	}
-
 	@Test public void test_element16() throws Exception {
-		// gunit test on line 207
+		// gunit test on line 213
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x=~(A|B)", 207);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x=(A|B)", 213);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
-		Object expecting = "(= x (~ (BLOCK A B)))";
+		Object expecting = "(= x (SET A B))";
 		assertEquals("testing rule element", expecting, actual);
 	}
 
 	@Test public void test_element17() throws Exception {
-		// gunit test on line 208
+		// gunit test on line 214
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x+=~(A|B)", 208);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x=(A|B)^", 214);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
-		Object expecting = "(+= x (~ (BLOCK A B)))";
+		Object expecting = "(= x (^ (SET A B)))";
 		assertEquals("testing rule element", expecting, actual);
 	}
 
 	@Test public void test_element18() throws Exception {
-		// gunit test on line 209
+		// gunit test on line 215
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x+=~(A|B)+", 209);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x=~(A|B)", 215);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
-		Object expecting = "(+ (BLOCK (ALT (+= x (~ (BLOCK A B))))))";
+		Object expecting = "(= x (~ (SET A B)))";
 		assertEquals("testing rule element", expecting, actual);
 	}
 
 	@Test public void test_element19() throws Exception {
-		// gunit test on line 210
+		// gunit test on line 216
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x=b+", 210);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x+=~(A|B)", 216);
+		Object actual = ((Tree)rstruct.getTree()).toStringTree();
+		Object expecting = "(+= x (~ (SET A B)))";
+		assertEquals("testing rule element", expecting, actual);
+	}
+
+	@Test public void test_element20() throws Exception {
+		// gunit test on line 217
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x+=~(A|B)+", 217);
+		Object actual = ((Tree)rstruct.getTree()).toStringTree();
+		Object expecting = "(+ (BLOCK (ALT (+= x (~ (BLOCK (ALT (SET A B))))))))";
+		assertEquals("testing rule element", expecting, actual);
+	}
+
+	@Test public void test_element21() throws Exception {
+		// gunit test on line 218
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x=b+", 218);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(+ (BLOCK (ALT (= x b))))";
 		assertEquals("testing rule element", expecting, actual);
 	}
 
-	@Test public void test_element20() throws Exception {
+	@Test public void test_element22() throws Exception {
-		// gunit test on line 211
+		// gunit test on line 219
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x+=ID*", 211);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x+=ID*", 219);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(* (BLOCK (ALT (+= x ID))))";
 		assertEquals("testing rule element", expecting, actual);
 	}
 
-	@Test public void test_element21() throws Exception {
+	@Test public void test_element23() throws Exception {
-		// gunit test on line 212
+		// gunit test on line 220
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x+='int'*", 212);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x+='int'*", 220);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(* (BLOCK (ALT (+= x 'int'))))";
 		assertEquals("testing rule element", expecting, actual);
 	}
 
-	@Test public void test_element22() throws Exception {
+	@Test public void test_element24() throws Exception {
-		// gunit test on line 213
+		// gunit test on line 221
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x+=b+", 213);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x+=b+", 221);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(+ (BLOCK (ALT (+= x b))))";
 		assertEquals("testing rule element", expecting, actual);
 	}
 
-	@Test public void test_element23() throws Exception {
+	@Test public void test_element25() throws Exception {
-		// gunit test on line 214
+		// gunit test on line 222
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "('*'^)*", 214);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "('*'^)*", 222);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(* (BLOCK (ALT (^ '*'))))";
 		assertEquals("testing rule element", expecting, actual);
 	}
 
-	@Test public void test_element24() throws Exception {
+	@Test public void test_element26() throws Exception {
-		// gunit test on line 215
+		// gunit test on line 223
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "({blort} 'x')*", 215);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "({blort} 'x')*", 223);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(* (BLOCK (ALT {blort} 'x')))";
 		assertEquals("testing rule element", expecting, actual);
 	}
 
-	@Test public void test_element25() throws Exception {
+	@Test public void test_element27() throws Exception {
-		// gunit test on line 216
+		// gunit test on line 224
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "A!", 216);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "A!", 224);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(! A)";
 		assertEquals("testing rule element", expecting, actual);
 	}
 
-	@Test public void test_element26() throws Exception {
+	@Test public void test_element28() throws Exception {
-		// gunit test on line 217
+		// gunit test on line 225
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "A^", 217);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "A^", 225);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(^ A)";
 		assertEquals("testing rule element", expecting, actual);
 	}
 
-	@Test public void test_element27() throws Exception {
+	@Test public void test_element29() throws Exception {
-		// gunit test on line 218
+		// gunit test on line 226
-		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x=A^", 218);
+		RuleReturnScope rstruct = (RuleReturnScope)execParser("element", "x=A^", 226);
 		Object actual = ((Tree)rstruct.getTree()).toStringTree();
 		Object expecting = "(= x (^ A))";
 		assertEquals("testing rule element", expecting, actual);

tool/test/org/antlr/v4/test/TestATNConstruction.java

@@ -54,6 +54,23 @@ public class TestATNConstruction extends BaseTest {
 		checkRule(g, "a", expecting);
 	}
 
+	@Test public void testSetAorB() throws Exception {
+		Grammar g = new Grammar(
+			"parser grammar P;\n"+
+			"a : A | B ;");
+		String expecting =
+			"RuleStart_a_0->BlockStart_6\n" +
+			"BlockStart_6->s2\n" +
+			"BlockStart_6->s4\n" +
+			"s2-A->s3\n" +
+			"s4-B->s5\n" +
+			"s3->BlockEnd_7\n" +
+			"s5->BlockEnd_7\n" +
+			"BlockEnd_7->RuleStop_a_1\n" +
+			"RuleStop_a_1-EOF->s8\n";
+		checkRule(g, "a", expecting);
+	}
+
 	@Test public void testRange() throws Exception {
 		LexerGrammar g = new LexerGrammar(
 			"lexer grammar P;\n"+
@@ -195,20 +212,37 @@ public class TestATNConstruction extends BaseTest {
 	}
 
 	@Test public void testAorBoptional() throws Exception {
+		Grammar g = new Grammar(
+			"parser grammar P;\n"+
+			"a : (A{;}|B)?;");
+		String expecting =
+			"RuleStart_a_0->BlockStart_8\n" +
+			"BlockStart_8->s2\n" +
+			"BlockStart_8->s6\n" +
+			"BlockStart_8->BlockEnd_9\n" +
+			"s2-A->s3\n" +
+			"s6-B->s7\n" +
+			"BlockEnd_9->RuleStop_a_1\n" +
+			"s3->s4\n" +
+			"s7->BlockEnd_9\n" +
+			"RuleStop_a_1-EOF->s10\n" +
+			"s4-action_0:-1->s5\n" +
+			"s5->BlockEnd_9\n";
+		checkRule(g, "a", expecting);
+	}
+
+	@Test public void testSetAorBoptional() throws Exception {
 		Grammar g = new Grammar(
 			"parser grammar P;\n"+
 			"a : (A|B)?;");
 		String expecting =
-			"RuleStart_a_0->BlockStart_6\n" +
+			"RuleStart_a_0->BlockStart_8\n" +
-			"BlockStart_6->s2\n" +
+			"BlockStart_8->s6\n" +
-			"BlockStart_6->s4\n" +
+			"BlockStart_8->BlockEnd_9\n" +
-			"BlockStart_6->BlockEnd_7\n" +
+			"s6-{3..4}->s7\n" +
-			"s2-A->s3\n" +
+			"BlockEnd_9->RuleStop_a_1\n" +
-			"s4-B->s5\n" +
+			"s7->BlockEnd_9\n" +
-			"BlockEnd_7->RuleStop_a_1\n" +
+			"RuleStop_a_1-EOF->s10\n";
-			"s3->BlockEnd_7\n" +
-			"s5->BlockEnd_7\n" +
-			"RuleStop_a_1-EOF->s8\n";
 		checkRule(g, "a", expecting);
 	}
 
@@ -217,14 +251,9 @@ public class TestATNConstruction extends BaseTest {
 			"parser grammar P;\n"+
 			"a : (A | B) C;");
 		String expecting =
-			"RuleStart_a_0->BlockStart_6\n" +
+			"RuleStart_a_0->s6\n" +
-			"BlockStart_6->s2\n" +
+			"s6-{3..4}->s7\n" +
-			"BlockStart_6->s4\n" +
+			"s7->s8\n" +
-			"s2-A->s3\n" +
-			"s4-B->s5\n" +
-			"s3->BlockEnd_7\n" +
-			"s5->BlockEnd_7\n" +
-			"BlockEnd_7->s8\n" +
 			"s8-C->s9\n" +
 			"s9->RuleStop_a_1\n" +
 			"RuleStop_a_1-EOF->s10\n";
@@ -253,19 +282,15 @@ public class TestATNConstruction extends BaseTest {
 			"parser grammar P;\n"+
 			"a : (A|B)+;");
 		String expecting =
-			"RuleStart_a_0->PlusBlockStart_6\n" +
+			"RuleStart_a_0->PlusBlockStart_8\n" +
-			"PlusBlockStart_6->s2\n" +
+			"PlusBlockStart_8->s6\n" +
-			"PlusBlockStart_6->s4\n" +
+			"s6-{3..4}->s7\n" +
-			"s2-A->s3\n" +
+			"s7->BlockEnd_9\n" +
-			"s4-B->s5\n" +
+			"BlockEnd_9->PlusLoopBack_10\n" +
-			"s3->BlockEnd_7\n" +
+			"PlusLoopBack_10->s6\n" +
-			"s5->BlockEnd_7\n" +
+			"PlusLoopBack_10->s11\n" +
-			"BlockEnd_7->PlusLoopBack_8\n" +
+			"s11->RuleStop_a_1\n" +
-			"PlusLoopBack_8->s2\n" +
+			"RuleStop_a_1-EOF->s12\n";
-			"PlusLoopBack_8->s4\n" +
-			"PlusLoopBack_8->s9\n" +
-			"s9->RuleStop_a_1\n" +
-			"RuleStop_a_1-EOF->s10\n";
 		checkRule(g, "a", expecting);
 	}
 
@@ -340,18 +365,15 @@ public class TestATNConstruction extends BaseTest {
 			"parser grammar P;\n"+
 			"a : (A | B)* ;");
 		String expecting =
-			"RuleStart_a_0->StarBlockStart_6\n" +
+			"RuleStart_a_0->StarBlockStart_8\n" +
-			"StarBlockStart_6->s2\n" +
+			"StarBlockStart_8->s6\n" +
-			"StarBlockStart_6->s4\n" +
+			"StarBlockStart_8->s11\n" +
-			"StarBlockStart_6->s9\n" +
+			"s6-{3..4}->s7\n" +
-			"s2-A->s3\n" +
+			"s11->RuleStop_a_1\n" +
-			"s4-B->s5\n" +
+			"s7->BlockEnd_9\n" +
-			"s9->RuleStop_a_1\n" +
+			"RuleStop_a_1-EOF->s12\n" +
-			"s3->BlockEnd_7\n" +
+			"BlockEnd_9->StarLoopBack_10\n" +
-			"s5->BlockEnd_7\n" +
+			"StarLoopBack_10->StarBlockStart_8\n";
-			"RuleStop_a_1-EOF->s10\n" +
-			"BlockEnd_7->StarLoopBack_8\n" +
-			"StarLoopBack_8->StarBlockStart_6\n";
 		checkRule(g, "a", expecting);
 	}