Summary: we can have lexer commands like -> skip now.

moved model.actions package to chunk
type(foo) didn't match.
ATNBuilder.g now tracks the outer alternative number and calls new factory commands on the lexer commands.
Removed unnecessary resolveWithPredicate field from ATNConfig
Added lexerActionIndex field to ATNConfig since we need to track whether we passed an action in an alternative will rule in the lexer.
Renamed ruleIndex in DFAState and added the lexer action index so that we can execute lexer actions from the DFA.
added functions to the grammar tree visitor for the lexer commands.
Added templates for the lexer commands.
Augmented the lexer ATN factory so that it constructs plain old actions from the lexer commands it finds. That way, the code generator doesn't know any different and generates an action.
Augmented the lexer ATN simulator so that it fires a proper action index now. previously it only used the rule index, which of course doesn't work when you have more than one action in a rule.
rm'd dup code from OutputModelController
altered the epsilon edge removal optimization so that it could not remove actions in lexer rules.
Added list of valid lexer commands in Rule.

[git-p4: depot-paths = "//depot/code/antlr4/main/": change = 9877]

runtime/Java/src/org/antlr/v4/runtime/Recognizer.java

@@ -129,7 +129,6 @@ public abstract class Recognizer<Symbol, ATNInterpreter extends ATNSimulator> {
 		return true;
 	}
 
-	/** In lexer, both indexes are same; one action per rule. */
 	public void action(@Nullable RuleContext _localctx, int ruleIndex, int actionIndex) {
 	}
 

runtime/Java/src/org/antlr/v4/runtime/atn/ATNConfig.java

@@ -57,18 +57,6 @@ public class ATNConfig {
 	@Nullable
 	public RuleContext context;
 
-	/**
-	 * Indicates that we have reached this ATN configuration after
-	 * traversing a predicate transition. This is important because we
-	 * cannot cache DFA states derived from such configurations
-	 * otherwise predicates would not get executed again (DFAs don't
-	 * have predicated edges in v4).
-	 */
-	//public boolean traversedPredicate; // TODO: don't need
-
-	/** Ignore this config when examining config sets */
-//	public boolean resolved;
-
 	/**
 	 * We cannot execute predicates dependent upon local context unless
 	 * we know for sure we are in the correct context. Because there is
@@ -82,18 +70,12 @@ public class ATNConfig {
 	 */
 	public int reachesIntoOuterContext;
 
+	/** Capture lexer action we traverse */
+	public int lexerActionIndex = -1; // TOOD: move to subclass
+
     @NotNull
     public SemanticContext semanticContext = SemanticContext.NONE;
 
-    /** This bit is used to indicate a semantic predicate will be
-     *  used to resolve the conflict. Essentially, this is used
-     *  as an "ignore" bit so that upon a set of conflicting configurations,
-     *  such as (s|2|p) and (s|3|q), I can set (s|3) to resolved=true (and any
-     *  other configuration associated with alt 3) to make it look like that set
-     *  uniquely predicts an alt.
-     */
-    protected boolean resolveWithPredicate;
-
 	public ATNConfig(@NotNull ATNState state,
 					 int alt,
 					 @Nullable RuleContext context)
@@ -134,6 +116,7 @@ public class ATNConfig {
 		this.context = context;
 		this.reachesIntoOuterContext = c.reachesIntoOuterContext;
         this.semanticContext = semanticContext;
+		this.lexerActionIndex = c.lexerActionIndex;
 	}
 
 //	public ATNConfig(@NotNull ATNConfig c, @Nullable RuleContext context) {

runtime/Java/src/org/antlr/v4/runtime/atn/LexerATNSimulator.java

@@ -259,8 +259,10 @@ public class LexerATNSimulator extends ATNSimulator {
 			throw new LexerNoViableAltException(recog, input, startIndex, s.configset);
 		}
 
-		int ruleIndex = dfaPrevAccept.state.ruleIndex;
+		int ruleIndex = dfaPrevAccept.state.lexerRuleIndex;
-		accept(input, ruleIndex, dfaPrevAccept);
+		int actionIndex = dfaPrevAccept.state.lexerActionIndex;
+		accept(input, ruleIndex, actionIndex,
+			   dfaPrevAccept.index, dfaPrevAccept.line, dfaPrevAccept.charPos);
 		tracePredict(dfaPrevAccept.state.prediction);
 		return dfaPrevAccept.state.prediction;
 	}
@@ -342,7 +344,8 @@ public class LexerATNSimulator extends ATNSimulator {
 		}
 
 		int ruleIndex = atnPrevAccept.config.state.ruleIndex;
-		accept(input, ruleIndex, atnPrevAccept);
+		accept(input, ruleIndex, atnPrevAccept.config.lexerActionIndex,
+			   atnPrevAccept.index, atnPrevAccept.line, atnPrevAccept.charPos);
 		return atn.ruleToTokenType[ruleIndex];
 	}
 
@@ -377,19 +380,20 @@ public class LexerATNSimulator extends ATNSimulator {
 		}
 	}
 
-	protected void accept(@NotNull CharStream input, int ruleIndex, @NotNull ExecState prevAccept) {
+	protected void accept(@NotNull CharStream input, int ruleIndex, int actionIndex,
+						  int index, int line, int charPos)
+	{
 		if ( debug ) {
-			System.out.format("ACTION %s:%d\n", recog != null ? recog.getRuleNames()[ruleIndex] : ruleIndex, ruleIndex);
+			System.out.format("ACTION %s:%d\n", recog != null ? recog.getRuleNames()[ruleIndex] : ruleIndex, actionIndex);
 		}
 
-		int actionIndex = atn.ruleToActionIndex[ruleIndex];
 		if ( actionIndex>=0 && recog!=null ) recog.action(null, ruleIndex, actionIndex);
 
 		// seek to after last char in token
-		traceSeek(prevAccept.index);
+		traceSeek(index);
-		input.seek(prevAccept.index);
+		input.seek(index);
-		line = prevAccept.line;
+		this.line = line;
-		charPositionInLine = prevAccept.charPos;
+		this.charPositionInLine = charPos;
 		consume(input);
 	}
 
@@ -530,6 +534,7 @@ public class LexerATNSimulator extends ATNSimulator {
 		// ignore actions; just exec one per rule upon accept
 		else if ( t.getClass() == ActionTransition.class ) {
 			c = new ATNConfig(config, t.target);
+			c.lexerActionIndex = ((ActionTransition)t).actionIndex;
 		}
 		else if ( t.isEpsilon() ) {
 			c = new ATNConfig(config, t.target);
@@ -611,7 +616,7 @@ public class LexerATNSimulator extends ATNSimulator {
 		future. Rather than creating collections of semantic predicates
 		like v3 and testing them on prediction, v4 will test them on the
 		fly all the time using the ATN not the DFA. This is slower but
-		semantically it's not use that often. One of the key elements to
+		semantically it's not used that often. One of the key elements to
 		this predicate mechanism is not adding DFA states that see
 		predicates immediately afterwards in the ATN. For example,
 
@@ -639,14 +644,16 @@ public class LexerATNSimulator extends ATNSimulator {
 			{
 				firstConfigWithRuleStopState = c;
 			}
-			if ( c.semanticContext!=null && c.semanticContext!=SemanticContext.NONE ) traversedPredicate = true;
+			if ( c.semanticContext!=null && c.semanticContext!=SemanticContext.NONE ) {
-//			if ( c.traversedPredicate ) traversedPredicate = true;
+				traversedPredicate = true;
+			}
 		}
 
 		if ( firstConfigWithRuleStopState!=null ) {
 			newState.isAcceptState = true;
-			newState.ruleIndex = firstConfigWithRuleStopState.state.ruleIndex;
+			newState.lexerRuleIndex = firstConfigWithRuleStopState.state.ruleIndex;
-			newState.prediction = atn.ruleToTokenType[newState.ruleIndex];
+			newState.lexerActionIndex = firstConfigWithRuleStopState.lexerActionIndex;
+			newState.prediction = atn.ruleToTokenType[newState.lexerRuleIndex];
 		}
 
 		if ( traversedPredicate ) return null; // cannot cache

runtime/Java/src/org/antlr/v4/runtime/atn/v2ParserATNSimulator.java

@@ -759,7 +759,6 @@ public class v2ParserATNSimulator<Symbol> extends ATNSimulator {
 		for (ATNConfig c : configs) {
 			if ( c.semanticContext!=SemanticContext.NONE && ambigAlts.contains(c.alt) ) {
 				altToPred[c.alt] = SemanticContext.or(altToPred[c.alt], c.semanticContext);
-				c.resolveWithPredicate = true;
 				nPredAlts++;
 			}
 		}

runtime/Java/src/org/antlr/v4/runtime/dfa/DFAState.java

@@ -80,7 +80,8 @@ public class DFAState {
 
 	public int prediction; // if accept state, what ttype do we match? is "else" clause if predicated
 
-	public int ruleIndex; // if accept, exec what action?
+	public int lexerRuleIndex = -1;		// if accept, exec action in what rule?
+	public int lexerActionIndex = -1;	// if accept, exec what action?
 
 	// todo: rename as unique?
 	public boolean complete; // all alts predict "prediction"

tool/resources/org/antlr/v4/tool/templates/codegen/Java/Java.stg

@@ -434,6 +434,17 @@ setState(<p.stateNumber>);
 if (!(<chunks>)) throw new FailedPredicateException(this, <if(p.msg)><p.msg><else><failChunks><endif>);
 >>
 
+// lexer actions are not associated with model objects
+
+LexerSkipCommand()  ::= "skip();"
+LexerMoreCommand()  ::= "more();"
+LexerPopMode()     ::= "popMode();"
+
+LexerTypeCommand(arg)      ::= "type = <arg>;"
+LexerChannelCommand(arg)   ::= "channel = <arg>;"
+LexerModeCommand(arg)      ::= "mode = <arg>;"
+LexerPushModeCommand(arg)  ::= "pushMode(<arg>);"
+
 DefaultParserSuperClass(s) ::= "Parser"
 
 ActionText(t) ::= "<t.text>"
@@ -482,8 +493,7 @@ TokenListDecl(t) ::= "List\<Token> <t.name> = new ArrayList\<Token>();"
 RuleContextDecl(r) ::= "<r.ctxName> <r.name>;"
 RuleContextListDecl(rdecl) ::= "List\<<rdecl.ctxName>> <rdecl.name> = new ArrayList\<<rdecl.ctxName>>();"
 
-/** Default RuleContext type name for a Parser rule */
+LexerRuleContext() ::= "RuleContext"
-ParserRuleContext() ::= "ParserRuleContext\<?>"
 
 /** The rule context name is the rule followed by a suffix; e.g.,
  *	r becomes rContext.
@@ -538,7 +548,7 @@ AttributeDecl(d) ::= "<d.decl>"
 /** If we don't know location of label def x, use this template */
 labelref(x) ::= "<if(!x.isLocal)>_localctx.<endif><x.name>"
 
-// used for left-recursive rules
+// used for left-recursive rules 
 recRuleDefArg()						  ::= "int _p"
 recRuleArg()						  ::= "$_p"
 recRuleAltPredicate(ruleName,opPrec)  ::= "<opPrec> >= <recRuleArg()>"
@@ -592,7 +602,6 @@ Lexer(lexer, atn, actionFuncs, sempredFuncs) ::= <<
 public class <lexer.name> extends Lexer {
 	public static final int
 		<lexer.tokens:{k | <k>=<lexer.tokens.(k)>}; separator=", ", wrap, anchor>;
-	// Lexer modes
 	<rest(lexer.modes):{m| public static final int <m> = <i>;}; separator="\n">
 
 	public static final String[] tokenNames = {

tool/src/org/antlr/v4/automata/ATNFactory.java

@@ -58,6 +58,8 @@ public interface ATNFactory {
 
 	void setCurrentRuleName(String name);
 
+	void setCurrentOuterAlt(int alt);
+
 	Handle rule(GrammarAST ruleAST, String name, Handle blk);
 
 	ATNState newState();
@@ -77,8 +79,6 @@ public interface ATNFactory {
 
 	Handle range(GrammarAST a, GrammarAST b);
 
-//	Handle not(GrammarAST a);
-
 	/** For a non-lexer, just build a simple token reference atom.
 	 *  For a lexer, a string is a sequence of char to match.  That is,
 	 *  "fog" is treated as 'f' 'o' 'g' not as a single transition in
@@ -116,11 +116,11 @@ public interface ATNFactory {
 
 	/** Build what amounts to an epsilon transition with an action.
 	 *  The action goes into ATN though it is ignored during analysis.
-	 *  It slows things down a bit, but I must ignore predicates after
-	 *  having seen an action (5-5-2008).
 	 */
 	Handle action(ActionAST action);
 
+	Handle action(String action);
+
 	Handle alt(List<Handle> els);
 
 	/** From A|B|..|Z alternative block build
@@ -214,4 +214,10 @@ public interface ATNFactory {
 	 *  (^(. .+) | .) to be safe.
 	 */
 	Handle wildcardTree(GrammarAST associatedAST);
+
+	Handle lexerAltCommands(ATNFactory.Handle alt, ATNFactory.Handle cmds);
+
+	String lexerCallCommand(GrammarAST ID, GrammarAST arg);
+
+	String lexerCommand(GrammarAST ID);
 }

tool/src/org/antlr/v4/automata/LexerATNFactory.java

@@ -29,23 +29,35 @@
 
 package org.antlr.v4.automata;
 
+import org.antlr.runtime.CommonToken;
 import org.antlr.runtime.Token;
+import org.antlr.v4.codegen.CodeGenerator;
 import org.antlr.v4.misc.CharSupport;
 import org.antlr.v4.parse.ANTLRParser;
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.atn.*;
 import org.antlr.v4.runtime.misc.IntervalSet;
+import org.antlr.v4.tool.Grammar;
 import org.antlr.v4.tool.LexerGrammar;
 import org.antlr.v4.tool.Rule;
 import org.antlr.v4.tool.ast.ActionAST;
 import org.antlr.v4.tool.ast.GrammarAST;
 import org.antlr.v4.tool.ast.TerminalAST;
+import org.stringtemplate.v4.ST;
+import org.stringtemplate.v4.STGroup;
 
 import java.util.List;
 import java.util.Set;
 
 public class LexerATNFactory extends ParserATNFactory {
-	public LexerATNFactory(LexerGrammar g) { super(g); }
+	public STGroup codegenTemplates;
+	public LexerATNFactory(LexerGrammar g) {
+		super(g);
+		// use codegen to get correct language templates for lexer commands
+		String language = Grammar.getLanguageOption(g.ast);
+		CodeGenerator gen = new CodeGenerator(g.tool, null, language);
+		codegenTemplates = gen.templates;
+	}
 
 	public ATN createATN() {
 		// BUILD ALL START STATES (ONE PER MODE)
@@ -87,13 +99,48 @@ public class LexerATNFactory extends ParserATNFactory {
 
 	@Override
 	public Handle action(ActionAST action) {
-//		Handle h = super.action(action);
+		ATNState left = newState(action);
-//		ActionTransition a = (ActionTransition)h.left.transition(0);
+		ATNState right = newState(action);
-//		a.actionIndex = g.actions.get(action);
+		boolean isCtxDependent = false;
-//		return h;
+		int actionIndex = g.lexerActions.get(action);
-		// no actions in lexer ATN; just one on end and we exec via action number
+		ActionTransition a =
-		ATNState x = newState(action);
+			new ActionTransition(right, currentRule.index, actionIndex, isCtxDependent);
-		return new Handle(x, x); // return just one blank state
+		left.addTransition(a);
+		action.atnState = left;
+		Handle h = new Handle(left, right);
+		return h;
+	}
+
+	@Override
+	public Handle action(String action) {
+        // define action AST for this rule as if we had found in grammar
+        ActionAST ast =	new ActionAST(new CommonToken(ANTLRParser.ACTION, action));
+		currentRule.defineActionInAlt(currentOuterAlt, ast);
+		return action(ast);
+	}
+
+	@Override
+	public Handle lexerAltCommands(Handle alt, Handle cmds) {
+		Handle h = new Handle(alt.left, cmds.right);
+		epsilon(alt.right, cmds.left);
+		return h;
+	}
+
+	@Override
+	public String lexerCallCommand(GrammarAST ID, GrammarAST arg) {
+		ST cmdST = codegenTemplates.getInstanceOf("Lexer" +
+												  CharSupport.capitalize(ID.getText())+
+												  "Command");
+		cmdST.add("arg", arg.getText());
+		return cmdST.render();
+	}
+
+	@Override
+	public String lexerCommand(GrammarAST ID) {
+		ST cmdST = codegenTemplates.getInstanceOf("Lexer" +
+												  CharSupport.capitalize(ID.getText())+
+												  "Command");
+		return cmdST.render();
 	}
 
 	@Override
@@ -126,7 +173,7 @@ public class LexerATNFactory extends ParserATNFactory {
 		}
 		if ( invert ) {
 			// TODO: what? should be chars not token types
-			IntervalSet notSet = (IntervalSet)set.complement(Token.MIN_TOKEN_TYPE, g.getMaxTokenType());
+			IntervalSet notSet = set.complement(Token.MIN_TOKEN_TYPE, g.getMaxTokenType());
 			left.addTransition(new NotSetTransition(right, set, notSet));
 		}
 		else {

tool/src/org/antlr/v4/automata/ParserATNFactory.java

@@ -70,8 +70,10 @@ public class ParserATNFactory implements ATNFactory {
 					// we have p-x->q for x in {rule, action, pred, token, ...}
 					// if edge out of q is single epsilon to block end
 					// we can strip epsilon p-x->q-eps->r
-					if ( q.getNumberOfTransitions()==1 && q.transition(0).isEpsilon() ) {
+					Transition trans = q.transition(0);
-						ATNState r = q.transition(0).target;
+					if ( q.getNumberOfTransitions()==1 && trans.isEpsilon() &&
+					     !(trans instanceof ActionTransition) ) {
+						ATNState r = trans.target;
 						if ( r instanceof BlockEndState ||
 							r instanceof PlusLoopbackState ||
 							r instanceof StarLoopbackState )
@@ -99,6 +101,8 @@ public class ParserATNFactory implements ATNFactory {
 
 	public Rule currentRule;
 
+	public int currentOuterAlt;
+
 	public ParserATNFactory(@NotNull Grammar g) { this.g = g; atn = new ATN(); }
 
 	public ATN createATN() {
@@ -120,7 +124,7 @@ public class ParserATNFactory implements ATNFactory {
 			ATNBuilder b = new ATNBuilder(nodes,this);
 			try {
 				setCurrentRuleName(r.name);
-				Handle h = b.block(null);
+				Handle h = b.ruleBlock(null);
 				rule(r.ast, r.name, h);
 			}
 			catch (RecognitionException re) {
@@ -133,6 +137,11 @@ public class ParserATNFactory implements ATNFactory {
 		this.currentRule = g.getRule(name);
 	}
 
+	@Override
+	public void setCurrentOuterAlt(int alt) {
+		currentOuterAlt = alt;
+	}
+
 	/* start->ruleblock->end */
 	public Handle rule(GrammarAST ruleAST, String name, Handle blk) {
 		Rule r = g.getRule(name);
@@ -170,7 +179,7 @@ public class ParserATNFactory implements ATNFactory {
 			set.add(ttype);
 		}
 		if ( invert ) {
-			IntervalSet notSet = (IntervalSet)set.complement(Token.MIN_TOKEN_TYPE, g.getMaxTokenType());
+			IntervalSet notSet = set.complement(Token.MIN_TOKEN_TYPE, g.getMaxTokenType());
 			left.addTransition(new NotSetTransition(right, set, notSet));
 		}
 		else {
@@ -286,6 +295,11 @@ public class ParserATNFactory implements ATNFactory {
 		return new Handle(left, right);
 	}
 
+	@Override
+	public Handle action(String action) {
+		return null;
+	}
+
 	/** From A|B|..|Z alternative block build
      *
      *  o->o-A->o->o (last ATNState is BlockEndState pointed to by all alts)
@@ -619,4 +633,18 @@ public class ParserATNFactory implements ATNFactory {
 		return false;
 	}
 
+	@Override
+	public Handle lexerAltCommands(Handle alt, Handle cmds) {
+		return null;
+	}
+
+	@Override
+	public String lexerCallCommand(GrammarAST ID, GrammarAST arg) {
+		return null;
+	}
+
+	@Override
+	public String lexerCommand(GrammarAST ID) {
+		return null;
+	}
 }

tool/src/org/antlr/v4/codegen/ActionTranslator.java

@@ -32,7 +32,7 @@ package org.antlr.v4.codegen;
 import org.antlr.runtime.ANTLRStringStream;
 import org.antlr.runtime.Token;
 import org.antlr.v4.codegen.model.RuleFunction;
-import org.antlr.v4.codegen.model.actions.*;
+import org.antlr.v4.codegen.model.chunk.*;
 import org.antlr.v4.parse.ActionSplitter;
 import org.antlr.v4.parse.ActionSplitterListener;
 import org.antlr.v4.tool.Attribute;

tool/src/org/antlr/v4/codegen/OutputModelController.java

@@ -269,10 +269,15 @@ public class OutputModelController {
 		function.postamble = rulePostamble(function, r);
 	}
 
-	public void buildLexerRuleActions(Lexer lexer, Rule r) {
+	public void buildLexerRuleActions(Lexer lexer, final Rule r) {
 		CodeGenerator gen = delegate.getGenerator();
 		Grammar g = delegate.getGrammar();
 		String ctxType = gen.target.getRuleFunctionContextStructName(r);
+		RuleActionFunction raf = lexer.actionFuncs.get(r);
+		if ( raf==null ) {
+			raf = new RuleActionFunction(delegate, r, ctxType);
+			lexer.actionFuncs.put(r, raf);
+		}
 		for (ActionAST a : r.actions) {
 			if ( a instanceof PredAST ) {
 				PredAST p = (PredAST)a;
@@ -284,24 +289,6 @@ public class OutputModelController {
 				rsf.actions.put(g.sempreds.get(p), new Action(delegate, p));
 			}
 			else if ( a.getType()== ANTLRParser.ACTION ) {
-				RuleActionFunction raf = lexer.sempredFuncs.get(r);
-				if ( raf==null ) {
-					raf = new RuleActionFunction(delegate, r, ctxType);
-					lexer.actionFuncs.put(r, raf);
-				}
-				raf.actions.put(g.lexerActions.get(a), new Action(delegate, a));
-			}
-
-			if ( a instanceof PredAST ) {
-				PredAST p = (PredAST)a;
-				RuleSempredFunction rsf = new RuleSempredFunction(delegate, r, ctxType);
-				lexer.sempredFuncs.put(r, rsf);
-				rsf.actions.put(g.sempreds.get(p), new Action(delegate, p));
-			}
-			else if ( a.getType()==ANTLRParser.ACTION ) {
-				// lexer sees {{...}} and {..} as same; neither are done until accept
-				RuleActionFunction raf = new RuleActionFunction(delegate, r, ctxType);
-				lexer.actionFuncs.put(r, raf);
 				raf.actions.put(g.lexerActions.get(a), new Action(delegate, a));
 			}
 		}

tool/src/org/antlr/v4/codegen/Target.java

@@ -339,15 +339,9 @@ public class Target {
 
 	public String getRuleFunctionContextStructName(Rule r) {
 		if ( r.g.isLexer() ) {
-			return gen.templates.getInstanceOf("ParserRuleContext").render();
+			return gen.templates.getInstanceOf("LexerRuleContext").render();
 		}
 		return r.name+gen.templates.getInstanceOf("RuleContextNameSuffix").render();
-//		boolean hasNoExternallyVisibleElements =
-//			r.args==null && r.retvals==null && r.scope==null && r.getLabelNames()==null;
-//		if ( hasNoExternallyVisibleElements ) {
-//			return gen.templates.getInstanceOf("ParserRuleContext").render();
-//		}
-//		return r.name+"_ctx";
 	}
 
 	/** If we know which actual function, we can provide the actual ctx type.
@@ -358,16 +352,9 @@ public class Target {
 	public String getRuleFunctionContextStructName(RuleFunction function) {
 		Rule r = function.rule;
 		if ( r.g.isLexer() ) {
-			return gen.templates.getInstanceOf("ParserRuleContext").render();
+			return gen.templates.getInstanceOf("LexerRuleContext").render();
 		}
 		return r.name+gen.templates.getInstanceOf("RuleContextNameSuffix").render();
-//		boolean hasNoExternallyVisibleElements =
-//			r.args==null && r.retvals==null && r.scope==null && r.getLabelNames()==null;
-//
-//		if ( hasNoExternallyVisibleElements && function.ruleCtx.isEmpty() ) {
-//			return gen.templates.getInstanceOf("ParserRuleContext").render();
-//		}
-//		return r.name+"_ctx";
 	}
 
 	// should be same for all refs to same token like $ID within single rule function

tool/src/org/antlr/v4/codegen/model/Action.java

@@ -32,7 +32,8 @@ package org.antlr.v4.codegen.model;
 import org.antlr.runtime.CommonToken;
 import org.antlr.v4.codegen.ActionTranslator;
 import org.antlr.v4.codegen.OutputModelFactory;
-import org.antlr.v4.codegen.model.actions.ActionChunk;
+import org.antlr.v4.codegen.model.chunk.ActionChunk;
+import org.antlr.v4.codegen.model.chunk.ActionText;
 import org.antlr.v4.parse.ANTLRParser;
 import org.antlr.v4.tool.ast.ActionAST;
 import org.antlr.v4.tool.ast.GrammarAST;
@@ -57,9 +58,15 @@ public class Action extends RuleElement {
 
 	public Action(OutputModelFactory factory, String action) {
 		super(factory,null);
-		RuleFunction rf = factory.getCurrentRuleFunction();
 		ActionAST ast = new ActionAST(new CommonToken(ANTLRParser.ACTION, action));
-		ast.resolver = rf.rule;
+		RuleFunction rf = factory.getCurrentRuleFunction();
-		chunks = ActionTranslator.translateActionChunk(factory, rf, action, ast);
+		if ( rf!=null ) { // we can translate
+			ast.resolver = rf.rule;
+			chunks = ActionTranslator.translateActionChunk(factory, rf, action, ast);
+		}
+		else {
+			chunks = new ArrayList<ActionChunk>();
+			chunks.add(new ActionText(action));
+		}
 	}
 }

tool/src/org/antlr/v4/codegen/model/InvokeRule.java

@@ -32,7 +32,7 @@ package org.antlr.v4.codegen.model;
 import org.antlr.v4.codegen.ActionTranslator;
 import org.antlr.v4.codegen.CodeGenerator;
 import org.antlr.v4.codegen.OutputModelFactory;
-import org.antlr.v4.codegen.model.actions.ActionChunk;
+import org.antlr.v4.codegen.model.chunk.ActionChunk;
 import org.antlr.v4.codegen.model.decl.Decl;
 import org.antlr.v4.codegen.model.decl.RuleContextDecl;
 import org.antlr.v4.codegen.model.decl.RuleContextListDecl;

tool/src/org/antlr/v4/codegen/model/Parser.java

@@ -31,10 +31,11 @@ package org.antlr.v4.codegen.model;
 
 import org.antlr.v4.codegen.CodeGenerator;
 import org.antlr.v4.codegen.OutputModelFactory;
-import org.antlr.v4.codegen.model.actions.ActionChunk;
+import org.antlr.v4.codegen.model.chunk.ActionChunk;
-import org.antlr.v4.codegen.model.actions.ActionText;
+import org.antlr.v4.codegen.model.chunk.ActionText;
-import org.antlr.v4.codegen.model.actions.DefaultParserSuperClass;
+import org.antlr.v4.codegen.model.chunk.DefaultParserSuperClass;
-import org.antlr.v4.tool.*;
+import org.antlr.v4.tool.Grammar;
+import org.antlr.v4.tool.Rule;
 
 import java.util.*;
 

tool/src/org/antlr/v4/codegen/model/RuleActionFunction.java

@@ -39,6 +39,7 @@ public class RuleActionFunction extends OutputModelObject {
 	public String ctxType;
 	public int ruleIndex;
 
+	/** Map actionIndex to Action */
 	@ModelElement public LinkedHashMap<Integer, Action> actions =
 		new LinkedHashMap<Integer, Action>();
 

tool/src/org/antlr/v4/codegen/model/SemPred.java

@@ -29,9 +29,13 @@
 
 package org.antlr.v4.codegen.model;
 
-import org.antlr.v4.codegen.*;
+import org.antlr.v4.codegen.ActionTranslator;
-import org.antlr.v4.codegen.model.actions.ActionChunk;
+import org.antlr.v4.codegen.CodeGenerator;
-import org.antlr.v4.tool.ast.*;
+import org.antlr.v4.codegen.OutputModelFactory;
+import org.antlr.v4.codegen.model.chunk.ActionChunk;
+import org.antlr.v4.tool.ast.ActionAST;
+import org.antlr.v4.tool.ast.GrammarAST;
+import org.antlr.v4.tool.ast.PredAST;
 
 import java.util.List;
 

tool/src/org/antlr/v4/codegen/model/chunk/ActionChunk.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 import org.antlr.v4.codegen.model.OutputModelObject;
 

tool/src/org/antlr/v4/codegen/model/chunk/ActionText.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 /** */
 public class ActionText extends ActionChunk {

tool/src/org/antlr/v4/codegen/model/chunk/ArgRef.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 /** */
 public class ArgRef extends LocalRef {

tool/src/org/antlr/v4/codegen/model/chunk/DefaultParserSuperClass.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 public class DefaultParserSuperClass extends ActionChunk {
 }

tool/src/org/antlr/v4/codegen/model/chunk/LabelRef.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 public class LabelRef extends ActionChunk {
 	public String name;

tool/src/org/antlr/v4/codegen/model/chunk/LexerSetAttr.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 import java.util.List;
 

tool/src/org/antlr/v4/codegen/model/chunk/ListLabelRef.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 public class ListLabelRef extends LabelRef {
 	public ListLabelRef(String name) { super(name); }

tool/src/org/antlr/v4/codegen/model/chunk/LocalRef.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 public class LocalRef extends ActionChunk {
 	public String name;

tool/src/org/antlr/v4/codegen/model/chunk/NonLocalAttrRef.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 public class NonLocalAttrRef extends ActionChunk {
 	public String ruleName;

tool/src/org/antlr/v4/codegen/model/chunk/QRetValueRef.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 /** */
 public class QRetValueRef extends RetValueRef {

tool/src/org/antlr/v4/codegen/model/chunk/RetValueRef.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 /** */
 public class RetValueRef extends ActionChunk {

tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 /** */
 public class RulePropertyRef extends ActionChunk {

tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_ctx.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 public class RulePropertyRef_ctx extends RulePropertyRef {
     public RulePropertyRef_ctx(String label) {

tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_start.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 /** */
 public class RulePropertyRef_start extends RulePropertyRef {

tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_stop.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 /** */
 public class RulePropertyRef_stop extends RulePropertyRef {

tool/src/org/antlr/v4/codegen/model/chunk/RulePropertyRef_text.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 /** */
 public class RulePropertyRef_text extends RulePropertyRef {

tool/src/org/antlr/v4/codegen/model/chunk/SetAttr.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 import org.antlr.v4.codegen.model.ModelElement;
 

tool/src/org/antlr/v4/codegen/model/chunk/SetNonLocalAttr.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 import java.util.List;
 

tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_ctx.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 public class ThisRulePropertyRef_ctx extends RulePropertyRef {
     public ThisRulePropertyRef_ctx(String label) {

tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_start.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 /** */
 public class ThisRulePropertyRef_start extends RulePropertyRef {

tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_stop.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 /** */
 public class ThisRulePropertyRef_stop extends RulePropertyRef {

tool/src/org/antlr/v4/codegen/model/chunk/ThisRulePropertyRef_text.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 /** */
 public class ThisRulePropertyRef_text extends RulePropertyRef {

tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 /** */
 public class TokenPropertyRef extends ActionChunk {

tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_channel.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 /** */
 public class TokenPropertyRef_channel extends TokenPropertyRef {

tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_index.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 /** */
 public class TokenPropertyRef_index extends TokenPropertyRef {

tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_int.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 /** */
 public class TokenPropertyRef_int extends TokenPropertyRef {

tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_line.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 /** */
 public class TokenPropertyRef_line extends TokenPropertyRef {

tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_pos.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 /** */
 public class TokenPropertyRef_pos extends TokenPropertyRef {

tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_text.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 /** */
 public class TokenPropertyRef_text extends TokenPropertyRef {

tool/src/org/antlr/v4/codegen/model/chunk/TokenPropertyRef_type.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 /** */
 public class TokenPropertyRef_type extends TokenPropertyRef {

tool/src/org/antlr/v4/codegen/model/chunk/TokenRef.java

@@ -27,7 +27,7 @@
  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-package org.antlr.v4.codegen.model.actions;
+package org.antlr.v4.codegen.model.chunk;
 
 /** */
 public class TokenRef extends ActionChunk {

tool/src/org/antlr/v4/parse/ANTLRParser.g

@@ -648,7 +648,7 @@ lexerAction
 	;
 
 lexerActionExpr
-	:	ID 
+	:	id 
 	|	INT
 	;
 

tool/src/org/antlr/v4/parse/ATNBuilder.g

@@ -75,6 +75,21 @@ import org.antlr.v4.automata.ATNFactory;
 
 dummy : block[null] ; // avoid error about no start rule
 
+ruleBlock[GrammarAST ebnfRoot] returns [ATNFactory.Handle p]
+@init {
+    List<ATNFactory.Handle> alts = new ArrayList<ATNFactory.Handle>();
+    int alt = 1;
+    factory.setCurrentOuterAlt(alt);
+}
+    :	^(BLOCK
+            (^(OPTIONS .+))?
+            (   a=alternative
+                {alts.add($a.p); factory.setCurrentOuterAlt(++alt);}
+            )+
+        )
+    	{$p = factory.block((BlockAST)$BLOCK, ebnfRoot, alts);}
+    ;
+
 block[GrammarAST ebnfRoot] returns [ATNFactory.Handle p]
 @init {List<ATNFactory.Handle> alts = new ArrayList<ATNFactory.Handle>();}
     :	^(BLOCK (^(OPTIONS .+))? (a=alternative {alts.add($a.p);})+)
@@ -83,11 +98,32 @@ block[GrammarAST ebnfRoot] returns [ATNFactory.Handle p]
 
 alternative returns [ATNFactory.Handle p]
 @init {List<ATNFactory.Handle> els = new ArrayList<ATNFactory.Handle>();}
-    :	^(LEXER_ALT_ACTION a=alternative .*)	{$p = $a.p;}
+    :	^(LEXER_ALT_ACTION a=alternative lexerCommands)
+        {$p = factory.lexerAltCommands($a.p,$lexerCommands.p);}
     |	^(ALT EPSILON)							{$p = factory.epsilon($EPSILON);}
     |   ^(ALT (e=element {els.add($e.p);})+)	{$p = factory.alt(els);}
     ;
 
+lexerCommands returns [ATNFactory.Handle p]
+@init {StringBuilder cmds = new StringBuilder();}
+    :   (c=lexerCommand {cmds.append($c.cmd+" ");})+
+        {
+        $p = factory.action(cmds.toString());
+        }
+    ;
+
+lexerCommand returns [String cmd]
+	:	^(LEXER_ACTION_CALL ID lexerCommandExpr)
+        {$cmd = factory.lexerCallCommand($ID, $lexerCommandExpr.start);}
+	|	ID
+        {$cmd = factory.lexerCommand($ID);}
+	;
+
+lexerCommandExpr
+	:	ID
+	|	INT
+	;
+
 element returns [ATNFactory.Handle p]
 	:	labeledElement				{$p = $labeledElement.p;}
 	|	atom						{$p = $atom.p;}

tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g

@@ -153,6 +153,8 @@ public void wildcardRef(GrammarAST ref) { }
 public void actionInAlt(ActionAST action) { }
 public void sempredInAlt(PredAST pred) { }
 public void label(GrammarAST op, GrammarAST ID, GrammarAST element) { }
+public void lexerCallCommand(int outerAltNumber, GrammarAST ID, GrammarAST arg) { }
+public void lexerCommand(int outerAltNumber, GrammarAST ID) { }
 
 	public void traceIn(String ruleName, int ruleIndex)  {
 		System.err.println("enter "+ruleName+": "+input.LT(1));
@@ -357,7 +359,7 @@ outerAlternative
 	;
 
 lexerAlternative
-	:	^(LEXER_ALT_ACTION lexerElements lexerAction+)
+	:	^(LEXER_ALT_ACTION lexerElements lexerCommand+)
     |   lexerElements
     ;
 
@@ -404,12 +406,14 @@ alternative
 	|	^(ALT EPSILON)
     ;
 
-lexerAction
+lexerCommand
-	:	^(LEXER_ACTION_CALL lexerActionExpr)
+	:	^(LEXER_ACTION_CALL ID lexerCommandExpr)
+        {lexerCallCommand(currentOuterAltNumber, $ID, $lexerCommandExpr.start);}
 	|	ID
+        {lexerCommand(currentOuterAltNumber, $ID);}
 	;
 
-lexerActionExpr
+lexerCommandExpr
 	:	ID 
 	|	INT
 	;

tool/src/org/antlr/v4/tool/Rule.java

@@ -75,12 +75,25 @@ public class Rule implements AttributeResolver {
             add(new Attribute("int"));
         }};
 
-    public String name;
+	public static Set<String> validLexerCommands = new HashSet<String>() {{
+		// CALLS
+		add("mode");
+		add("pushMode");
+		add("type");
+		add("channel");
+
+		// ACTIONS
+		add("popMode");
+		add("skip");
+		add("more");
+	}};
+
+	public String name;
 	public List<GrammarAST> modifiers;
 
-    public RuleAST ast;
+	public RuleAST ast;
-    public AttributeDict args;
+	public AttributeDict args;
-    public AttributeDict retvals;
+	public AttributeDict retvals;
 	public AttributeDict locals;
 
 	/** In which grammar does this rule live? */
@@ -123,7 +136,7 @@ public class Rule implements AttributeResolver {
 	/** All rules have unique index 0..n-1 */
 	public int index;
 
-	public int actionIndex = -1; // if lexer; 0..n-1
+	public int actionIndex = -1; // if lexer; 0..n-1 for n actions in a rule
 
 	public Rule(Grammar g, String name, RuleAST ast, int numberOfAlts) {
 		this.g = g;
@@ -138,10 +151,15 @@ public class Rule implements AttributeResolver {
 		actions.add(actionAST);
 		alt[currentAlt].actions.add(actionAST);
 		if ( g.isLexer() ) {
-			actionIndex = g.lexerActions.size();
+			defineLexerAction(actionAST);
-			if ( g.lexerActions.get(actionAST)==null ) {
+		}
-				g.lexerActions.put(actionAST, actionIndex);
+	}
-			}
+
+	/** Lexer actions are numbered across rules 0..n-1 */
+	public void defineLexerAction(ActionAST actionAST) {
+		actionIndex = g.lexerActions.size();
+		if ( g.lexerActions.get(actionAST)==null ) {
+			g.lexerActions.put(actionAST, actionIndex);
 		}
 	}
 

tool/test/org/antlr/v4/test/TestLexerExec.java

@@ -44,11 +44,70 @@ public class TestLexerExec extends BaseTest {
 		assertEquals(expecting, found);
 	}
 
+	@Test public void testSkipCommand() throws Exception {
+		String grammar =
+			"lexer grammar L;\n"+
+			"I : '0'..'9'+ {System.out.println(\"I\");} ;\n"+
+			"WS : (' '|'\\n') -> skip ;";
+		String found = execLexer("L.g", grammar, "L", "34 34");
+		String expecting =
+			"I\n" +
+			"I\n" +
+			"[@0,0:1='34',<3>,1:0]\n" +
+			"[@1,3:4='34',<3>,1:3]\n" +
+			"[@2,5:4='<EOF>',<-1>,1:5]\n";
+		assertEquals(expecting, found);
+	}
+
+	@Test public void testMoreCommand() throws Exception {
+		String grammar =
+			"lexer grammar L;\n"+
+			"I : '0'..'9'+ {System.out.println(\"I\");} ;\n"+
+			"WS : '#' -> more ;";
+		String found = execLexer("L.g", grammar, "L", "34#10");
+		String expecting =
+			"I\n" +
+			"I\n" +
+			"[@0,0:1='34',<3>,1:0]\n" +
+			"[@1,2:4='#10',<3>,1:2]\n" +
+			"[@2,5:4='<EOF>',<-1>,1:5]\n";
+		assertEquals(expecting, found);
+	}
+
+	@Test public void testTypeCommand() throws Exception {
+		String grammar =
+			"lexer grammar L;\n"+
+			"I : '0'..'9'+ {System.out.println(\"I\");} ;\n"+
+			"HASH : '#' -> type(HASH) ;";
+		String found = execLexer("L.g", grammar, "L", "34#");
+		String expecting =
+			"I\n" +
+			"[@0,0:1='34',<3>,1:0]\n" +
+			"[@1,2:2='#',<4>,1:2]\n" +
+			"[@2,3:2='<EOF>',<-1>,1:3]\n";
+		assertEquals(expecting, found);
+	}
+
+	@Test public void testCombinedCommand() throws Exception {
+		String grammar =
+			"lexer grammar L;\n"+
+			"I : '0'..'9'+ {System.out.println(\"I\");} ;\n"+
+			"HASH : '#' -> type(HASH), skip, more  ;";
+		String found = execLexer("L.g", grammar, "L", "34#11");
+		String expecting =
+			"I\n" +
+			"I\n" +
+			"[@0,0:1='34',<3>,1:0]\n" +
+			"[@1,2:4='#11',<3>,1:2]\n" +
+			"[@2,5:4='<EOF>',<-1>,1:5]\n";
+		assertEquals(expecting, found);
+	}
+
 	@Test public void testLexerMode() throws Exception {
 		String grammar =
 			"lexer grammar L;\n" +
 			"STRING_START : '\"' {pushMode(STRING_MODE); more();} ;\n" +
-			"WS : ' '|'\n' {skip();} ;\n"+
+			"WS : (' '|'\n') {skip();} ;\n"+
 			"mode STRING_MODE;\n"+
 			"STRING : '\"' {popMode();} ;\n"+
 			"ANY : . {more();} ;\n";