Update to the latest release of ANTLR 4

reference/antlr4

@@ -1 +1 @@
-Subproject commit 2d2af2d2b87273bca73a12da41a00b6ae6bcedc1
+Subproject commit 851cd3ce131ac59782ba12bd95f6d1f694843e5b

runtime/CSharp/Antlr4.Runtime/ANTLRInputStream.cs

@@ -48,8 +48,7 @@ namespace Antlr4.Runtime
     /// or
     /// <code>char[]</code>
     /// to use.
-    /// <p/>
-    /// If you need encoding, pass in stream/reader with correct encoding.
+    /// <p>If you need encoding, pass in stream/reader with correct encoding.</p>
     /// </summary>
     public class AntlrInputStream : ICharStream
     {
@@ -284,8 +283,9 @@ namespace Antlr4.Runtime
                 // just jump; don't update stream state (line, ...)
                 return;
             }
-            // seek forward, consume until p hits index
-            while (p < index && index < n)
+            // seek forward, consume until p hits index or n (whichever comes first)
+            index = Math.Min(index, n);
+            while (p < index)
             {
                 Consume();
             }

runtime/CSharp/Antlr4.Runtime/Atn/ATN.cs

@@ -79,26 +79,22 @@ namespace Antlr4.Runtime.Atn
         /// <summary>For lexer ATNs, this maps the rule index to the resulting token type.</summary>
         /// <remarks>
         /// For lexer ATNs, this maps the rule index to the resulting token type.
-        /// <p/>
-        /// This is
+        /// For parser ATNs, this maps the rule index to the generated bypass token
+        /// type if the
+        /// <see cref="ATNDeserializationOptions.IsGenerateRuleBypassTransitions()">ATNDeserializationOptions.IsGenerateRuleBypassTransitions()</see>
+        /// deserialization option was specified; otherwise, this is
         /// <code>null</code>
-        /// for parser ATNs.
+        /// .
         /// </remarks>
         public int[] ruleToTokenType;
 
         /// <summary>
-        /// For lexer ATNs, this maps the rule index to the action which should be
-        /// executed following a match.
+        /// For lexer ATNs, this is an array of
+        /// <see cref="ILexerAction">ILexerAction</see>
+        /// objects which may
+        /// be referenced by action transitions in the ATN.
         /// </summary>
-        /// <remarks>
-        /// For lexer ATNs, this maps the rule index to the action which should be
-        /// executed following a match.
-        /// <p/>
-        /// This is
-        /// <code>null</code>
-        /// for parser ATNs.
-        /// </remarks>
-        public int[] ruleToActionIndex;
+        public ILexerAction[] lexerActions;
 
         [NotNull]
         public readonly IList<TokensStartState> modeToStartState = new List<TokensStartState>();
@@ -260,14 +256,13 @@ namespace Antlr4.Runtime.Atn
         /// symbols,
         /// <see cref="Antlr4.Runtime.IToken.Eof">Antlr4.Runtime.IToken.Eof</see>
         /// is added to the returned set.
-        /// <p/>
-        /// If
+        /// <p>If
         /// <code>context</code>
         /// is
         /// <code>null</code>
         /// , it is treated as
         /// <see cref="ParserRuleContext#EMPTY">ParserRuleContext#EMPTY</see>
-        /// .
+        /// .</p>
         /// </summary>
         /// <param name="stateNumber">the ATN state number</param>
         /// <param name="context">the full parse context</param>

runtime/CSharp/Antlr4.Runtime/Atn/ATNConfig.cs

@@ -83,21 +83,21 @@ namespace Antlr4.Runtime.Atn
 
         public static Antlr4.Runtime.Atn.ATNConfig Create(ATNState state, int alt, PredictionContext context)
         {
-            return Create(state, alt, context, Antlr4.Runtime.Atn.SemanticContext.None, -1);
+            return Create(state, alt, context, Antlr4.Runtime.Atn.SemanticContext.None, null);
         }
 
         public static Antlr4.Runtime.Atn.ATNConfig Create(ATNState state, int alt, PredictionContext context, Antlr4.Runtime.Atn.SemanticContext semanticContext)
         {
-            return Create(state, alt, context, semanticContext, -1);
+            return Create(state, alt, context, semanticContext, null);
         }
 
-        public static Antlr4.Runtime.Atn.ATNConfig Create(ATNState state, int alt, PredictionContext context, Antlr4.Runtime.Atn.SemanticContext semanticContext, int actionIndex)
+        public static Antlr4.Runtime.Atn.ATNConfig Create(ATNState state, int alt, PredictionContext context, Antlr4.Runtime.Atn.SemanticContext semanticContext, LexerActionExecutor lexerActionExecutor)
         {
             if (semanticContext != Antlr4.Runtime.Atn.SemanticContext.None)
             {
-                if (actionIndex != -1)
+                if (lexerActionExecutor != null)
                 {
-                    return new ATNConfig.ActionSemanticContextATNConfig(actionIndex, semanticContext, state, alt, context);
+                    return new ATNConfig.ActionSemanticContextATNConfig(lexerActionExecutor, semanticContext, state, alt, context, false);
                 }
                 else
                 {
@@ -106,9 +106,9 @@ namespace Antlr4.Runtime.Atn
             }
             else
             {
-                if (actionIndex != -1)
+                if (lexerActionExecutor != null)
                 {
-                    return new ATNConfig.ActionATNConfig(actionIndex, state, alt, context);
+                    return new ATNConfig.ActionATNConfig(lexerActionExecutor, state, alt, context, false);
                 }
                 else
                 {
@@ -207,11 +207,11 @@ namespace Antlr4.Runtime.Atn
             }
         }
 
-        public virtual int ActionIndex
+        public virtual LexerActionExecutor ActionExecutor
         {
             get
             {
-                return -1;
+                return null;
             }
         }
 
@@ -223,38 +223,44 @@ namespace Antlr4.Runtime.Atn
             }
         }
 
+        public virtual bool HasPassedThroughNonGreedyDecision()
+        {
+            return false;
+        }
+
         public Antlr4.Runtime.Atn.ATNConfig Clone()
         {
-            return Transform(this.State);
+            return Transform(this.State, false);
         }
 
-        public Antlr4.Runtime.Atn.ATNConfig Transform(ATNState state)
+        public Antlr4.Runtime.Atn.ATNConfig Transform(ATNState state, bool checkNonGreedy)
         {
-            return Transform(state, this.context, this.SemanticContext, this.ActionIndex);
+            return Transform(state, this.context, this.SemanticContext, checkNonGreedy, this.ActionExecutor);
         }
 
-        public Antlr4.Runtime.Atn.ATNConfig Transform(ATNState state, Antlr4.Runtime.Atn.SemanticContext semanticContext)
+        public Antlr4.Runtime.Atn.ATNConfig Transform(ATNState state, Antlr4.Runtime.Atn.SemanticContext semanticContext, bool checkNonGreedy)
         {
-            return Transform(state, this.context, semanticContext, this.ActionIndex);
+            return Transform(state, this.context, semanticContext, checkNonGreedy, this.ActionExecutor);
         }
 
-        public Antlr4.Runtime.Atn.ATNConfig Transform(ATNState state, PredictionContext context)
+        public Antlr4.Runtime.Atn.ATNConfig Transform(ATNState state, PredictionContext context, bool checkNonGreedy)
         {
-            return Transform(state, context, this.SemanticContext, this.ActionIndex);
+            return Transform(state, context, this.SemanticContext, checkNonGreedy, this.ActionExecutor);
         }
 
-        public Antlr4.Runtime.Atn.ATNConfig Transform(ATNState state, int actionIndex)
+        public Antlr4.Runtime.Atn.ATNConfig Transform(ATNState state, LexerActionExecutor lexerActionExecutor, bool checkNonGreedy)
         {
-            return Transform(state, context, this.SemanticContext, actionIndex);
+            return Transform(state, context, this.SemanticContext, checkNonGreedy, lexerActionExecutor);
         }
 
-        private Antlr4.Runtime.Atn.ATNConfig Transform(ATNState state, PredictionContext context, Antlr4.Runtime.Atn.SemanticContext semanticContext, int actionIndex)
+        private Antlr4.Runtime.Atn.ATNConfig Transform(ATNState state, PredictionContext context, Antlr4.Runtime.Atn.SemanticContext semanticContext, bool checkNonGreedy, LexerActionExecutor lexerActionExecutor)
         {
+            bool passedThroughNonGreedy = checkNonGreedy && CheckNonGreedyDecision(this, state);
             if (semanticContext != Antlr4.Runtime.Atn.SemanticContext.None)
             {
-                if (actionIndex != -1)
+                if (lexerActionExecutor != null || passedThroughNonGreedy)
                 {
-                    return new ATNConfig.ActionSemanticContextATNConfig(actionIndex, semanticContext, this, state, context);
+                    return new ATNConfig.ActionSemanticContextATNConfig(lexerActionExecutor, semanticContext, this, state, context, passedThroughNonGreedy);
                 }
                 else
                 {
@@ -263,9 +269,9 @@ namespace Antlr4.Runtime.Atn
             }
             else
             {
-                if (actionIndex != -1)
+                if (lexerActionExecutor != null || passedThroughNonGreedy)
                 {
-                    return new ATNConfig.ActionATNConfig(actionIndex, this, state, context);
+                    return new ATNConfig.ActionATNConfig(lexerActionExecutor, this, state, context, passedThroughNonGreedy);
                 }
                 else
                 {
@@ -274,17 +280,22 @@ namespace Antlr4.Runtime.Atn
             }
         }
 
+        private static bool CheckNonGreedyDecision(Antlr4.Runtime.Atn.ATNConfig source, ATNState target)
+        {
+            return source.HasPassedThroughNonGreedyDecision() || target is DecisionState && ((DecisionState)target).nonGreedy;
+        }
+
         public virtual Antlr4.Runtime.Atn.ATNConfig AppendContext(int context, PredictionContextCache contextCache)
         {
             PredictionContext appendedContext = Context.AppendContext(context, contextCache);
-            Antlr4.Runtime.Atn.ATNConfig result = Transform(State, appendedContext);
+            Antlr4.Runtime.Atn.ATNConfig result = Transform(State, appendedContext, false);
             return result;
         }
 
         public virtual Antlr4.Runtime.Atn.ATNConfig AppendContext(PredictionContext context, PredictionContextCache contextCache)
         {
             PredictionContext appendedContext = Context.AppendContext(context, contextCache);
-            Antlr4.Runtime.Atn.ATNConfig result = Transform(State, appendedContext);
+            Antlr4.Runtime.Atn.ATNConfig result = Transform(State, appendedContext, false);
             return result;
         }
 
@@ -364,7 +375,7 @@ namespace Antlr4.Runtime.Atn
                     return false;
                 }
             }
-            return this.State.stateNumber == other.State.stateNumber && this.Alt == other.Alt && this.ReachesIntoOuterContext == other.ReachesIntoOuterContext && this.Context.Equals(other.Context) && this.SemanticContext.Equals(other.SemanticContext) && this.ActionIndex == other.ActionIndex;
+            return this.State.stateNumber == other.State.stateNumber && this.Alt == other.Alt && this.ReachesIntoOuterContext == other.ReachesIntoOuterContext && this.Context.Equals(other.Context) && this.SemanticContext.Equals(other.SemanticContext) && this.HasPassedThroughNonGreedyDecision() == other.HasPassedThroughNonGreedyDecision() && ObjectEqualityComparator.Instance.Equals(this.ActionExecutor, other.ActionExecutor);
         }
 
         public override int GetHashCode()
@@ -375,7 +386,9 @@ namespace Antlr4.Runtime.Atn
             hashCode = MurmurHash.Update(hashCode, ReachesIntoOuterContext ? 1 : 0);
             hashCode = MurmurHash.Update(hashCode, Context);
             hashCode = MurmurHash.Update(hashCode, SemanticContext);
-            hashCode = MurmurHash.Finish(hashCode, 5);
+            hashCode = MurmurHash.Update(hashCode, HasPassedThroughNonGreedyDecision() ? 1 : 0);
+            hashCode = MurmurHash.Update(hashCode, ActionExecutor);
+            hashCode = MurmurHash.Finish(hashCode, 7);
             return hashCode;
         }
 
@@ -498,56 +511,74 @@ namespace Antlr4.Runtime.Atn
 
         private class ActionATNConfig : ATNConfig
         {
-            private readonly int actionIndex;
+            private readonly LexerActionExecutor lexerActionExecutor;
 
-            public ActionATNConfig(int actionIndex, ATNState state, int alt, PredictionContext context)
+            private readonly bool passedThroughNonGreedyDecision;
+
+            public ActionATNConfig(LexerActionExecutor lexerActionExecutor, ATNState state, int alt, PredictionContext context, bool passedThroughNonGreedyDecision)
                 : base(state, alt, context)
             {
-                this.actionIndex = actionIndex;
+                this.lexerActionExecutor = lexerActionExecutor;
+                this.passedThroughNonGreedyDecision = passedThroughNonGreedyDecision;
             }
 
-            protected internal ActionATNConfig(int actionIndex, ATNConfig c, ATNState state, PredictionContext context)
+            protected internal ActionATNConfig(LexerActionExecutor lexerActionExecutor, ATNConfig c, ATNState state, PredictionContext context, bool passedThroughNonGreedyDecision)
                 : base(c, state, context)
             {
                 if (c.SemanticContext != SemanticContext.None)
                 {
                     throw new NotSupportedException();
                 }
-                this.actionIndex = actionIndex;
+                this.lexerActionExecutor = lexerActionExecutor;
+                this.passedThroughNonGreedyDecision = passedThroughNonGreedyDecision;
             }
 
-            public override int ActionIndex
+            public override LexerActionExecutor ActionExecutor
             {
                 get
                 {
-                    return actionIndex;
+                    return lexerActionExecutor;
                 }
             }
+
+            public override bool HasPassedThroughNonGreedyDecision()
+            {
+                return passedThroughNonGreedyDecision;
+            }
         }
 
         private class ActionSemanticContextATNConfig : ATNConfig.SemanticContextATNConfig
         {
-            private readonly int actionIndex;
+            private readonly LexerActionExecutor lexerActionExecutor;
 
-            public ActionSemanticContextATNConfig(int actionIndex, SemanticContext semanticContext, ATNState state, int alt, PredictionContext context)
+            private readonly bool passedThroughNonGreedyDecision;
+
+            public ActionSemanticContextATNConfig(LexerActionExecutor lexerActionExecutor, SemanticContext semanticContext, ATNState state, int alt, PredictionContext context, bool passedThroughNonGreedyDecision)
                 : base(semanticContext, state, alt, context)
             {
-                this.actionIndex = actionIndex;
+                this.lexerActionExecutor = lexerActionExecutor;
+                this.passedThroughNonGreedyDecision = passedThroughNonGreedyDecision;
             }
 
-            public ActionSemanticContextATNConfig(int actionIndex, SemanticContext semanticContext, ATNConfig c, ATNState state, PredictionContext context)
+            public ActionSemanticContextATNConfig(LexerActionExecutor lexerActionExecutor, SemanticContext semanticContext, ATNConfig c, ATNState state, PredictionContext context, bool passedThroughNonGreedyDecision)
                 : base(semanticContext, c, state, context)
             {
-                this.actionIndex = actionIndex;
+                this.lexerActionExecutor = lexerActionExecutor;
+                this.passedThroughNonGreedyDecision = passedThroughNonGreedyDecision;
             }
 
-            public override int ActionIndex
+            public override LexerActionExecutor ActionExecutor
             {
                 get
                 {
-                    return actionIndex;
+                    return lexerActionExecutor;
                 }
             }
+
+            public override bool HasPassedThroughNonGreedyDecision()
+            {
+                return passedThroughNonGreedyDecision;
+            }
         }
     }
 }

runtime/CSharp/Antlr4.Runtime/Atn/ATNDeserializationOptions.cs

@@ -0,0 +1,126 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using System;
+using Antlr4.Runtime.Misc;
+using Sharpen;
+
+namespace Antlr4.Runtime.Atn
+{
+    /// <author>Sam Harwell</author>
+    public class ATNDeserializationOptions
+    {
+        private static readonly Antlr4.Runtime.Atn.ATNDeserializationOptions defaultOptions;
+
+        static ATNDeserializationOptions()
+        {
+            defaultOptions = new Antlr4.Runtime.Atn.ATNDeserializationOptions();
+            defaultOptions.MakeReadOnly();
+        }
+
+        private bool readOnly;
+
+        private bool verifyATN;
+
+        private bool generateRuleBypassTransitions;
+
+        private bool optimize;
+
+        public ATNDeserializationOptions()
+        {
+            this.verifyATN = true;
+            this.generateRuleBypassTransitions = false;
+            this.optimize = true;
+        }
+
+        public ATNDeserializationOptions(Antlr4.Runtime.Atn.ATNDeserializationOptions options)
+        {
+            this.verifyATN = options.verifyATN;
+            this.generateRuleBypassTransitions = options.generateRuleBypassTransitions;
+            this.optimize = options.optimize;
+        }
+
+        [NotNull]
+        public static Antlr4.Runtime.Atn.ATNDeserializationOptions GetDefaultOptions()
+        {
+            return defaultOptions;
+        }
+
+        public bool IsReadOnly()
+        {
+            return readOnly;
+        }
+
+        public void MakeReadOnly()
+        {
+            readOnly = true;
+        }
+
+        public bool IsVerifyATN()
+        {
+            return verifyATN;
+        }
+
+        public void SetVerifyATN(bool verifyATN)
+        {
+            ThrowIfReadOnly();
+            this.verifyATN = verifyATN;
+        }
+
+        public bool IsGenerateRuleBypassTransitions()
+        {
+            return generateRuleBypassTransitions;
+        }
+
+        public void SetGenerateRuleBypassTransitions(bool generateRuleBypassTransitions)
+        {
+            ThrowIfReadOnly();
+            this.generateRuleBypassTransitions = generateRuleBypassTransitions;
+        }
+
+        public bool IsOptimize()
+        {
+            return optimize;
+        }
+
+        public void SetOptimize(bool optimize)
+        {
+            ThrowIfReadOnly();
+            this.optimize = optimize;
+        }
+
+        protected internal virtual void ThrowIfReadOnly()
+        {
+            if (IsReadOnly())
+            {
+                throw new InvalidOperationException("The object is read only.");
+            }
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/Atn/ATNSerializer.cs

@@ -0,0 +1,720 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Text;
+using Antlr4.Runtime.Atn;
+using Antlr4.Runtime.Misc;
+using Sharpen;
+
+namespace Antlr4.Runtime.Atn
+{
+    public class ATNSerializer
+    {
+        public ATN atn;
+
+        private IList<string> ruleNames;
+
+        private IList<string> tokenNames;
+
+        public ATNSerializer(ATN atn, IList<string> ruleNames)
+        {
+            System.Diagnostics.Debug.Assert(atn.grammarType != null);
+            this.atn = atn;
+            this.ruleNames = ruleNames;
+        }
+
+        public ATNSerializer(ATN atn, IList<string> ruleNames, IList<string> tokenNames)
+        {
+            System.Diagnostics.Debug.Assert(atn.grammarType != null);
+            this.atn = atn;
+            this.ruleNames = ruleNames;
+            this.tokenNames = tokenNames;
+        }
+
+        /// <summary>
+        /// Serialize state descriptors, edge descriptors, and decision&rarr;state map
+        /// into list of ints:
+        /// grammar-type, (ANTLRParser.LEXER, ...)
+        /// max token type,
+        /// num states,
+        /// state-0-type ruleIndex, state-1-type ruleIndex, ...
+        /// </summary>
+        /// <remarks>
+        /// Serialize state descriptors, edge descriptors, and decision&rarr;state map
+        /// into list of ints:
+        /// grammar-type, (ANTLRParser.LEXER, ...)
+        /// max token type,
+        /// num states,
+        /// state-0-type ruleIndex, state-1-type ruleIndex, ... state-i-type ruleIndex optional-arg ...
+        /// num rules,
+        /// rule-1-start-state rule-1-args, rule-2-start-state  rule-2-args, ...
+        /// (args are token type,actionIndex in lexer else 0,0)
+        /// num modes,
+        /// mode-0-start-state, mode-1-start-state, ... (parser has 0 modes)
+        /// num sets
+        /// set-0-interval-count intervals, set-1-interval-count intervals, ...
+        /// num total edges,
+        /// src, trg, edge-type, edge arg1, optional edge arg2 (present always), ...
+        /// num decisions,
+        /// decision-0-start-state, decision-1-start-state, ...
+        /// Convenient to pack into unsigned shorts to make as Java string.
+        /// </remarks>
+        public virtual List<int> Serialize()
+        {
+            List<int> data = new List<int>();
+            data.Add(ATNDeserializer.SerializedVersion);
+            SerializeUUID(data, ATNDeserializer.SerializedUuid);
+            // convert grammar type to ATN const to avoid dependence on ANTLRParser
+            data.Add((int)(atn.grammarType));
+            data.Add(atn.maxTokenType);
+            int nedges = 0;
+            IDictionary<IntervalSet, int> setIndices = new Dictionary<IntervalSet, int>();
+            IList<IntervalSet> sets = new List<IntervalSet>();
+            // dump states, count edges and collect sets while doing so
+            List<int> nonGreedyStates = new List<int>();
+            List<int> sllStates = new List<int>();
+            List<int> precedenceStates = new List<int>();
+            data.Add(atn.states.Count);
+            foreach (ATNState s in atn.states)
+            {
+                if (s == null)
+                {
+                    // might be optimized away
+                    data.Add((int)(StateType.InvalidType));
+                    continue;
+                }
+                StateType stateType = s.StateType;
+                if (s is DecisionState)
+                {
+                    DecisionState decisionState = (DecisionState)s;
+                    if (decisionState.nonGreedy)
+                    {
+                        nonGreedyStates.Add(s.stateNumber);
+                    }
+                    if (decisionState.sll)
+                    {
+                        sllStates.Add(s.stateNumber);
+                    }
+                }
+                if (s is RuleStartState && ((RuleStartState)s).isPrecedenceRule)
+                {
+                    precedenceStates.Add(s.stateNumber);
+                }
+                data.Add((int)(stateType));
+                if (s.ruleIndex == -1)
+                {
+                    data.Add(char.MaxValue);
+                }
+                else
+                {
+                    data.Add(s.ruleIndex);
+                }
+                if (s.StateType == StateType.LoopEnd)
+                {
+                    data.Add(((LoopEndState)s).loopBackState.stateNumber);
+                }
+                else
+                {
+                    if (s is BlockStartState)
+                    {
+                        data.Add(((BlockStartState)s).endState.stateNumber);
+                    }
+                }
+                if (s.StateType != StateType.RuleStop)
+                {
+                    // the deserializer can trivially derive these edges, so there's no need to serialize them
+                    nedges += s.NumberOfTransitions;
+                }
+                for (int i = 0; i < s.NumberOfTransitions; i++)
+                {
+                    Transition t = s.Transition(i);
+                    TransitionType edgeType = Transition.serializationTypes.Get(t.GetType());
+                    if (edgeType == TransitionType.Set || edgeType == TransitionType.NotSet)
+                    {
+                        SetTransition st = (SetTransition)t;
+                        if (!setIndices.ContainsKey(st.set))
+                        {
+                            sets.AddItem(st.set);
+                            setIndices.Put(st.set, sets.Count - 1);
+                        }
+                    }
+                }
+            }
+            // non-greedy states
+            data.Add(nonGreedyStates.Size());
+            for (int i_1 = 0; i_1 < nonGreedyStates.Size(); i_1++)
+            {
+                data.Add(nonGreedyStates.Get(i_1));
+            }
+            // SLL decisions
+            data.Add(sllStates.Size());
+            for (int i_2 = 0; i_2 < sllStates.Size(); i_2++)
+            {
+                data.Add(sllStates.Get(i_2));
+            }
+            // precedence states
+            data.Add(precedenceStates.Size());
+            for (int i_3 = 0; i_3 < precedenceStates.Size(); i_3++)
+            {
+                data.Add(precedenceStates.Get(i_3));
+            }
+            int nrules = atn.ruleToStartState.Length;
+            data.Add(nrules);
+            for (int r = 0; r < nrules; r++)
+            {
+                ATNState ruleStartState = atn.ruleToStartState[r];
+                data.Add(ruleStartState.stateNumber);
+                bool leftFactored = ruleNames[ruleStartState.ruleIndex].IndexOf(ATNSimulator.RuleVariantDelimiter) >= 0;
+                data.Add(leftFactored ? 1 : 0);
+                if (atn.grammarType == ATNType.Lexer)
+                {
+                    if (atn.ruleToTokenType[r] == TokenConstants.Eof)
+                    {
+                        data.Add(char.MaxValue);
+                    }
+                    else
+                    {
+                        data.Add(atn.ruleToTokenType[r]);
+                    }
+                }
+            }
+            int nmodes = atn.modeToStartState.Count;
+            data.Add(nmodes);
+            if (nmodes > 0)
+            {
+                foreach (ATNState modeStartState in atn.modeToStartState)
+                {
+                    data.Add(modeStartState.stateNumber);
+                }
+            }
+            int nsets = sets.Count;
+            data.Add(nsets);
+            foreach (IntervalSet set in sets)
+            {
+                bool containsEof = set.Contains(TokenConstants.Eof);
+                if (containsEof && set.GetIntervals()[0].b == TokenConstants.Eof)
+                {
+                    data.Add(set.GetIntervals().Count - 1);
+                }
+                else
+                {
+                    data.Add(set.GetIntervals().Count);
+                }
+                data.Add(containsEof ? 1 : 0);
+                foreach (Interval I in set.GetIntervals())
+                {
+                    if (I.a == TokenConstants.Eof)
+                    {
+                        if (I.b == TokenConstants.Eof)
+                        {
+                            continue;
+                        }
+                        else
+                        {
+                            data.Add(0);
+                        }
+                    }
+                    else
+                    {
+                        data.Add(I.a);
+                    }
+                    data.Add(I.b);
+                }
+            }
+            data.Add(nedges);
+            foreach (ATNState s_1 in atn.states)
+            {
+                if (s_1 == null)
+                {
+                    // might be optimized away
+                    continue;
+                }
+                if (s_1.StateType == StateType.RuleStop)
+                {
+                    continue;
+                }
+                for (int i = 0; i_3 < s_1.NumberOfTransitions; i_3++)
+                {
+                    Transition t = s_1.Transition(i_3);
+                    if (atn.states[t.target.stateNumber] == null)
+                    {
+                        throw new InvalidOperationException("Cannot serialize a transition to a removed state.");
+                    }
+                    int src = s_1.stateNumber;
+                    int trg = t.target.stateNumber;
+                    TransitionType edgeType = Transition.serializationTypes.Get(t.GetType());
+                    int arg1 = 0;
+                    int arg2 = 0;
+                    int arg3 = 0;
+                    switch (edgeType)
+                    {
+                        case TransitionType.Rule:
+                        {
+                            trg = ((RuleTransition)t).followState.stateNumber;
+                            arg1 = ((RuleTransition)t).target.stateNumber;
+                            arg2 = ((RuleTransition)t).ruleIndex;
+                            arg3 = ((RuleTransition)t).precedence;
+                            break;
+                        }
+
+                        case TransitionType.Precedence:
+                        {
+                            PrecedencePredicateTransition ppt = (PrecedencePredicateTransition)t;
+                            arg1 = ppt.precedence;
+                            break;
+                        }
+
+                        case TransitionType.Predicate:
+                        {
+                            PredicateTransition pt = (PredicateTransition)t;
+                            arg1 = pt.ruleIndex;
+                            arg2 = pt.predIndex;
+                            arg3 = pt.isCtxDependent ? 1 : 0;
+                            break;
+                        }
+
+                        case TransitionType.Range:
+                        {
+                            arg1 = ((RangeTransition)t).from;
+                            arg2 = ((RangeTransition)t).to;
+                            if (arg1 == TokenConstants.Eof)
+                            {
+                                arg1 = 0;
+                                arg3 = 1;
+                            }
+                            break;
+                        }
+
+                        case TransitionType.Atom:
+                        {
+                            arg1 = ((AtomTransition)t).label;
+                            if (arg1 == TokenConstants.Eof)
+                            {
+                                arg1 = 0;
+                                arg3 = 1;
+                            }
+                            break;
+                        }
+
+                        case TransitionType.Action:
+                        {
+                            ActionTransition at = (ActionTransition)t;
+                            arg1 = at.ruleIndex;
+                            arg2 = at.actionIndex;
+                            if (arg2 == -1)
+                            {
+                                arg2 = unchecked((int)(0xFFFF));
+                            }
+                            arg3 = at.isCtxDependent ? 1 : 0;
+                            break;
+                        }
+
+                        case TransitionType.Set:
+                        {
+                            arg1 = setIndices.Get(((SetTransition)t).set);
+                            break;
+                        }
+
+                        case TransitionType.NotSet:
+                        {
+                            arg1 = setIndices.Get(((SetTransition)t).set);
+                            break;
+                        }
+
+                        case TransitionType.Wildcard:
+                        {
+                            break;
+                        }
+                    }
+                    data.Add(src);
+                    data.Add(trg);
+                    data.Add((int)(edgeType));
+                    data.Add(arg1);
+                    data.Add(arg2);
+                    data.Add(arg3);
+                }
+            }
+            int ndecisions = atn.decisionToState.Count;
+            data.Add(ndecisions);
+            foreach (DecisionState decStartState in atn.decisionToState)
+            {
+                data.Add(decStartState.stateNumber);
+            }
+            //
+            // LEXER ACTIONS
+            //
+            if (atn.grammarType == ATNType.Lexer)
+            {
+                data.Add(atn.lexerActions.Length);
+                foreach (ILexerAction action in atn.lexerActions)
+                {
+                    data.Add((int)(action.GetActionType()));
+                    switch (action.GetActionType())
+                    {
+                        case LexerActionType.Channel:
+                        {
+                            int channel = ((LexerChannelAction)action).GetChannel();
+                            data.Add(channel != -1 ? channel : unchecked((int)(0xFFFF)));
+                            data.Add(0);
+                            break;
+                        }
+
+                        case LexerActionType.Custom:
+                        {
+                            int ruleIndex = ((LexerCustomAction)action).GetRuleIndex();
+                            int actionIndex = ((LexerCustomAction)action).GetActionIndex();
+                            data.Add(ruleIndex != -1 ? ruleIndex : unchecked((int)(0xFFFF)));
+                            data.Add(actionIndex != -1 ? actionIndex : unchecked((int)(0xFFFF)));
+                            break;
+                        }
+
+                        case LexerActionType.Mode:
+                        {
+                            int mode = ((LexerModeAction)action).GetMode();
+                            data.Add(mode != -1 ? mode : unchecked((int)(0xFFFF)));
+                            data.Add(0);
+                            break;
+                        }
+
+                        case LexerActionType.More:
+                        {
+                            data.Add(0);
+                            data.Add(0);
+                            break;
+                        }
+
+                        case LexerActionType.PopMode:
+                        {
+                            data.Add(0);
+                            data.Add(0);
+                            break;
+                        }
+
+                        case LexerActionType.PushMode:
+                        {
+                            mode = ((LexerPushModeAction)action).GetMode();
+                            data.Add(mode != -1 ? mode : unchecked((int)(0xFFFF)));
+                            data.Add(0);
+                            break;
+                        }
+
+                        case LexerActionType.Skip:
+                        {
+                            data.Add(0);
+                            data.Add(0);
+                            break;
+                        }
+
+                        case LexerActionType.Type:
+                        {
+                            int type = ((LexerTypeAction)action).GetType();
+                            data.Add(type != -1 ? type : unchecked((int)(0xFFFF)));
+                            data.Add(0);
+                            break;
+                        }
+
+                        default:
+                        {
+                            string message = string.Format(CultureInfo.CurrentCulture, "The specified lexer action type %s is not valid.", action.GetActionType());
+                            throw new ArgumentException(message);
+                        }
+                    }
+                }
+            }
+            // don't adjust the first value since that's the version number
+            for (int i_4 = 1; i_4 < data.Size(); i_4++)
+            {
+                if (data.Get(i_4) < char.MinValue || data.Get(i_4) > char.MaxValue)
+                {
+                    throw new NotSupportedException("Serialized ATN data element out of range.");
+                }
+                int value = (data.Get(i_4) + 2) & unchecked((int)(0xFFFF));
+                data.Set(i_4, value);
+            }
+            return data;
+        }
+
+        public virtual string Decode(char[] data)
+        {
+            data = data.Clone();
+            // don't adjust the first value since that's the version number
+            for (int i = 1; i < data.Length; i++)
+            {
+                data[i] = (char)(data[i] - 2);
+            }
+            StringBuilder buf = new StringBuilder();
+            int p = 0;
+            int version = ATNDeserializer.ToInt(data[p++]);
+            if (version != ATNDeserializer.SerializedVersion)
+            {
+                string reason = string.Format("Could not deserialize ATN with version %d (expected %d).", version, ATNDeserializer.SerializedVersion);
+                throw new NotSupportedException(new InvalidClassException(typeof(ATN).FullName, reason));
+            }
+            UUID uuid = ATNDeserializer.ToUUID(data, p);
+            p += 8;
+            if (!uuid.Equals(ATNDeserializer.SerializedUuid))
+            {
+                string reason = string.Format(CultureInfo.CurrentCulture, "Could not deserialize ATN with UUID %s (expected %s).", uuid, ATNDeserializer.SerializedUuid);
+                throw new NotSupportedException(new InvalidClassException(typeof(ATN).FullName, reason));
+            }
+            p++;
+            // skip grammarType
+            int maxType = ATNDeserializer.ToInt(data[p++]);
+            buf.Append("max type ").Append(maxType).Append("\n");
+            int nstates = ATNDeserializer.ToInt(data[p++]);
+            for (int i_1 = 0; i_1 < nstates; i_1++)
+            {
+                StateType stype = StateType.Values()[ATNDeserializer.ToInt(data[p++])];
+                if (stype == StateType.InvalidType)
+                {
+                    continue;
+                }
+                // ignore bad type of states
+                int ruleIndex = ATNDeserializer.ToInt(data[p++]);
+                if (ruleIndex == char.MaxValue)
+                {
+                    ruleIndex = -1;
+                }
+                string arg = string.Empty;
+                if (stype == StateType.LoopEnd)
+                {
+                    int loopBackStateNumber = ATNDeserializer.ToInt(data[p++]);
+                    arg = " " + loopBackStateNumber;
+                }
+                else
+                {
+                    if (stype == StateType.PlusBlockStart || stype == StateType.StarBlockStart || stype == StateType.BlockStart)
+                    {
+                        int endStateNumber = ATNDeserializer.ToInt(data[p++]);
+                        arg = " " + endStateNumber;
+                    }
+                }
+                buf.Append(i_1).Append(":").Append(ATNState.serializationNames[(int)(stype)]).Append(" ").Append(ruleIndex).Append(arg).Append("\n");
+            }
+            int numNonGreedyStates = ATNDeserializer.ToInt(data[p++]);
+            for (int i_2 = 0; i_2 < numNonGreedyStates; i_2++)
+            {
+                int stateNumber = ATNDeserializer.ToInt(data[p++]);
+            }
+            int numSllStates = ATNDeserializer.ToInt(data[p++]);
+            for (int i_3 = 0; i_3 < numSllStates; i_3++)
+            {
+                int stateNumber = ATNDeserializer.ToInt(data[p++]);
+            }
+            int numPrecedenceStates = ATNDeserializer.ToInt(data[p++]);
+            for (int i_4 = 0; i_4 < numPrecedenceStates; i_4++)
+            {
+                int stateNumber = ATNDeserializer.ToInt(data[p++]);
+            }
+            int nrules = ATNDeserializer.ToInt(data[p++]);
+            for (int i_5 = 0; i_5 < nrules; i_5++)
+            {
+                int s = ATNDeserializer.ToInt(data[p++]);
+                bool leftFactored = ATNDeserializer.ToInt(data[p++]) != 0;
+                if (atn.grammarType == ATNType.Lexer)
+                {
+                    int arg1 = ATNDeserializer.ToInt(data[p++]);
+                    buf.Append("rule ").Append(i_5).Append(":").Append(s).Append(" ").Append(arg1).Append('\n');
+                }
+                else
+                {
+                    buf.Append("rule ").Append(i_5).Append(":").Append(s).Append('\n');
+                }
+            }
+            int nmodes = ATNDeserializer.ToInt(data[p++]);
+            for (int i_6 = 0; i_6 < nmodes; i_6++)
+            {
+                int s = ATNDeserializer.ToInt(data[p++]);
+                buf.Append("mode ").Append(i_6).Append(":").Append(s).Append('\n');
+            }
+            int nsets = ATNDeserializer.ToInt(data[p++]);
+            for (int i_7 = 0; i_7 < nsets; i_7++)
+            {
+                int nintervals = ATNDeserializer.ToInt(data[p++]);
+                buf.Append(i_7).Append(":");
+                bool containsEof = data[p++] != 0;
+                if (containsEof)
+                {
+                    buf.Append(GetTokenName(TokenConstants.Eof));
+                }
+                for (int j = 0; j < nintervals; j++)
+                {
+                    if (containsEof || j > 0)
+                    {
+                        buf.Append(", ");
+                    }
+                    buf.Append(GetTokenName(ATNDeserializer.ToInt(data[p]))).Append("..").Append(GetTokenName(ATNDeserializer.ToInt(data[p + 1])));
+                    p += 2;
+                }
+                buf.Append("\n");
+            }
+            int nedges = ATNDeserializer.ToInt(data[p++]);
+            for (int i_8 = 0; i_8 < nedges; i_8++)
+            {
+                int src = ATNDeserializer.ToInt(data[p]);
+                int trg = ATNDeserializer.ToInt(data[p + 1]);
+                int ttype = ATNDeserializer.ToInt(data[p + 2]);
+                int arg1 = ATNDeserializer.ToInt(data[p + 3]);
+                int arg2 = ATNDeserializer.ToInt(data[p + 4]);
+                int arg3 = ATNDeserializer.ToInt(data[p + 5]);
+                buf.Append(src).Append("->").Append(trg).Append(" ").Append(Transition.serializationNames[ttype]).Append(" ").Append(arg1).Append(",").Append(arg2).Append(",").Append(arg3).Append("\n");
+                p += 6;
+            }
+            int ndecisions = ATNDeserializer.ToInt(data[p++]);
+            for (int i_9 = 0; i_9 < ndecisions; i_9++)
+            {
+                int s = ATNDeserializer.ToInt(data[p++]);
+                buf.Append(i_9).Append(":").Append(s).Append("\n");
+            }
+            if (atn.grammarType == ATNType.Lexer)
+            {
+                int lexerActionCount = ATNDeserializer.ToInt(data[p++]);
+                for (int i_10 = 0; i_10 < lexerActionCount; i_10++)
+                {
+                    LexerActionType actionType = LexerActionType.Values()[ATNDeserializer.ToInt(data[p++])];
+                    int data1 = ATNDeserializer.ToInt(data[p++]);
+                    int data2 = ATNDeserializer.ToInt(data[p++]);
+                }
+            }
+            return buf.ToString();
+        }
+
+        public virtual string GetTokenName(int t)
+        {
+            if (t == -1)
+            {
+                return "EOF";
+            }
+            if (atn.grammarType == ATNType.Lexer && t >= char.MinValue && t <= char.MaxValue)
+            {
+                switch (t)
+                {
+                    case '\n':
+                    {
+                        return "'\\n'";
+                    }
+
+                    case '\r':
+                    {
+                        return "'\\r'";
+                    }
+
+                    case '\t':
+                    {
+                        return "'\\t'";
+                    }
+
+                    case '\b':
+                    {
+                        return "'\\b'";
+                    }
+
+                    case '\f':
+                    {
+                        return "'\\f'";
+                    }
+
+                    case '\\':
+                    {
+                        return "'\\\\'";
+                    }
+
+                    case '\'':
+                    {
+                        return "'\\''";
+                    }
+
+                    default:
+                    {
+                        if (Character.UnicodeBlock.Of((char)t) == Character.UnicodeBlock.BasicLatin && !char.IsISOControl((char)t))
+                        {
+                            return '\'' + char.ToString((char)t) + '\'';
+                        }
+                        // turn on the bit above max "\uFFFF" value so that we pad with zeros
+                        // then only take last 4 digits
+                        string hex = Sharpen.Runtime.Substring(Sharpen.Extensions.ToHexString(t | unchecked((int)(0x10000))).ToUpper(), 1, 5);
+                        string unicodeStr = "'\\u" + hex + "'";
+                        return unicodeStr;
+                    }
+                }
+            }
+            if (tokenNames != null && t >= 0 && t < tokenNames.Count)
+            {
+                return tokenNames[t];
+            }
+            return t.ToString();
+        }
+
+        /// <summary>Used by Java target to encode short/int array as chars in string.</summary>
+        /// <remarks>Used by Java target to encode short/int array as chars in string.</remarks>
+        public static string GetSerializedAsString(ATN atn, IList<string> ruleNames)
+        {
+            return new string(GetSerializedAsChars(atn, ruleNames));
+        }
+
+        public static List<int> GetSerialized(ATN atn, IList<string> ruleNames)
+        {
+            return new Antlr4.Runtime.Atn.ATNSerializer(atn, ruleNames).Serialize();
+        }
+
+        public static char[] GetSerializedAsChars(ATN atn, IList<string> ruleNames)
+        {
+            return Utils.ToCharArray(GetSerialized(atn, ruleNames));
+        }
+
+        public static string GetDecoded(ATN atn, IList<string> ruleNames, IList<string> tokenNames)
+        {
+            List<int> serialized = GetSerialized(atn, ruleNames);
+            char[] data = Utils.ToCharArray(serialized);
+            return new Antlr4.Runtime.Atn.ATNSerializer(atn, ruleNames, tokenNames).Decode(data);
+        }
+
+        private void SerializeUUID(List<int> data, UUID uuid)
+        {
+            SerializeLong(data, uuid.GetLeastSignificantBits());
+            SerializeLong(data, uuid.GetMostSignificantBits());
+        }
+
+        private void SerializeLong(List<int> data, long value)
+        {
+            SerializeInt(data, (int)value);
+            SerializeInt(data, (int)(value >> 32));
+        }
+
+        private void SerializeInt(List<int> data, int value)
+        {
+            data.Add((char)value);
+            data.Add((char)(value >> 16));
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/Atn/ILexerAction.cs

@@ -0,0 +1,97 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using Antlr4.Runtime;
+using Antlr4.Runtime.Atn;
+using Antlr4.Runtime.Misc;
+using Sharpen;
+
+namespace Antlr4.Runtime.Atn
+{
+    /// <summary>
+    /// Represents a single action which can be executed following the successful
+    /// match of a lexer rule.
+    /// </summary>
+    /// <remarks>
+    /// Represents a single action which can be executed following the successful
+    /// match of a lexer rule. Lexer actions are used for both embedded action syntax
+    /// and ANTLR 4's new lexer command syntax.
+    /// </remarks>
+    /// <author>Sam Harwell</author>
+    /// <since>4.2</since>
+    public interface ILexerAction
+    {
+        /// <summary>Gets the serialization type of the lexer action.</summary>
+        /// <remarks>Gets the serialization type of the lexer action.</remarks>
+        /// <returns>The serialization type of the lexer action.</returns>
+        [NotNull]
+        LexerActionType GetActionType();
+
+        /// <summary>Gets whether the lexer action is position-dependent.</summary>
+        /// <remarks>
+        /// Gets whether the lexer action is position-dependent. Position-dependent
+        /// actions may have different semantics depending on the
+        /// <see cref="Antlr4.Runtime.ICharStream">Antlr4.Runtime.ICharStream</see>
+        /// index at the time the action is executed.
+        /// <p>Many lexer commands, including
+        /// <code>type</code>
+        /// ,
+        /// <code>skip</code>
+        /// , and
+        /// <code>more</code>
+        /// , do not check the input index during their execution.
+        /// Actions like this are position-independent, and may be stored more
+        /// efficiently as part of the
+        /// <see cref="ATNConfig.ActionExecutor()">ATNConfig.ActionExecutor()</see>
+        /// .</p>
+        /// </remarks>
+        /// <returns>
+        /// 
+        /// <code>true</code>
+        /// if the lexer action semantics can be affected by the
+        /// position of the input
+        /// <see cref="Antlr4.Runtime.ICharStream">Antlr4.Runtime.ICharStream</see>
+        /// at the time it is executed;
+        /// otherwise,
+        /// <code>false</code>
+        /// .
+        /// </returns>
+        bool IsPositionDependent();
+
+        /// <summary>
+        /// Execute the lexer action in the context of the specified
+        /// <see cref="Antlr4.Runtime.Lexer">Antlr4.Runtime.Lexer</see>
+        /// .
+        /// <p>For position-dependent actions, the input stream must already be
+        /// positioned correctly prior to calling this method.</p>
+        /// </summary>
+        /// <param name="lexer">The lexer instance.</param>
+        void Execute(Lexer lexer);
+    }
+}

runtime/CSharp/Antlr4.Runtime/Atn/LL1Analyzer.cs

@@ -86,7 +86,7 @@ namespace Antlr4.Runtime.Atn
                 HashSet<ATNConfig> lookBusy = new HashSet<ATNConfig>();
                 bool seeThruPreds = false;
                 // fail to get lookahead upon pred
-                Look(s.Transition(alt).target, null, PredictionContext.EmptyFull, look[alt], lookBusy, new BitSet(), seeThruPreds, false);
+                Look(s.Transition(alt).target, null, PredictionContext.EmptyLocal, look[alt], lookBusy, new BitSet(), seeThruPreds, false);
                 // Wipe out lookahead for this alternative if we found nothing
                 // or we had a predicate when we !seeThruPreds
                 if (look[alt].Size() == 0 || look[alt].Contains(HitPred))
@@ -104,8 +104,7 @@ namespace Antlr4.Runtime.Atn
         /// specified
         /// <code>ctx</code>
         /// .
-        /// <p/>
-        /// If
+        /// <p>If
         /// <code>ctx</code>
         /// is
         /// <code>null</code>
@@ -121,7 +120,7 @@ namespace Antlr4.Runtime.Atn
         /// and the end of the outermost rule is
         /// reached,
         /// <see cref="Antlr4.Runtime.IToken.Eof">Antlr4.Runtime.IToken.Eof</see>
-        /// is added to the result set.
+        /// is added to the result set.</p>
         /// </summary>
         /// <param name="s">the ATN state</param>
         /// <param name="ctx">
@@ -141,7 +140,7 @@ namespace Antlr4.Runtime.Atn
         [NotNull]
         public virtual IntervalSet Look(ATNState s, PredictionContext ctx)
         {
-            return Look(s, null, ctx);
+            return Look(s, s.atn.ruleToStopState[s.ruleIndex], ctx);
         }
 
         /// <summary>
@@ -151,8 +150,7 @@ namespace Antlr4.Runtime.Atn
         /// specified
         /// <code>ctx</code>
         /// .
-        /// <p/>
-        /// If
+        /// <p>If
         /// <code>ctx</code>
         /// is
         /// <code>null</code>
@@ -164,11 +162,11 @@ namespace Antlr4.Runtime.Atn
         /// If
         /// <code>ctx</code>
         /// is not
-        /// <code>null</code>
+        /// <code>PredictionContext#EMPTY_LOCAL</code>
         /// and the end of the outermost rule is
         /// reached,
         /// <see cref="Antlr4.Runtime.IToken.Eof">Antlr4.Runtime.IToken.Eof</see>
-        /// is added to the result set.
+        /// is added to the result set.</p>
         /// </summary>
         /// <param name="s">the ATN state</param>
         /// <param name="stopState">
@@ -196,7 +194,8 @@ namespace Antlr4.Runtime.Atn
             IntervalSet r = new IntervalSet();
             bool seeThruPreds = true;
             // ignore preds; get all lookahead
-            Look(s, stopState, ctx, r, new HashSet<ATNConfig>(), new BitSet(), seeThruPreds, true);
+            bool addEOF = true;
+            Look(s, stopState, ctx, r, new HashSet<ATNConfig>(), new BitSet(), seeThruPreds, addEOF);
             return r;
         }
 
@@ -297,51 +296,46 @@ namespace Antlr4.Runtime.Atn
                 }
                 else
                 {
-                    if (ctx.IsEmpty && addEOF)
+                    if (ctx.IsEmpty)
                     {
-                        look.Add(TokenConstants.Eof);
+                        if (addEOF)
+                        {
+                            look.Add(TokenConstants.Eof);
+                        }
                         return;
                     }
                 }
             }
             if (s is RuleStopState)
             {
-                if (PredictionContext.IsEmptyLocal(ctx))
+                if (ctx.IsEmpty && !PredictionContext.IsEmptyLocal(ctx))
                 {
-                    look.Add(TokenConstants.Epsilon);
-                    return;
-                }
-                else
-                {
-                    if (ctx.IsEmpty && addEOF)
+                    if (addEOF)
                     {
                         look.Add(TokenConstants.Eof);
-                        return;
                     }
+                    return;
                 }
-                for (int i = 0; i < ctx.Size; i++)
+                bool removed = calledRuleStack.Get(s.ruleIndex);
+                try
                 {
-                    if (ctx.GetReturnState(i) != PredictionContext.EmptyFullStateKey)
+                    calledRuleStack.Clear(s.ruleIndex);
+                    for (int i = 0; i < ctx.Size; i++)
                     {
+                        if (ctx.GetReturnState(i) == PredictionContext.EmptyFullStateKey)
+                        {
+                            continue;
+                        }
                         ATNState returnState = atn.states[ctx.GetReturnState(i)];
                         //					System.out.println("popping back to "+retState);
-                        for (int j = 0; j < ctx.Size; j++)
-                        {
-                            bool removed = calledRuleStack.Get(returnState.ruleIndex);
-                            try
-                            {
-                                calledRuleStack.Clear(returnState.ruleIndex);
-                                Look(returnState, stopState, ctx.GetParent(j), look, lookBusy, calledRuleStack, seeThruPreds, addEOF);
-                            }
-                            finally
-                            {
-                                if (removed)
-                                {
-                                    calledRuleStack.Set(returnState.ruleIndex);
-                                }
-                            }
-                        }
-                        return;
+                        Look(returnState, stopState, ctx.GetParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF);
+                    }
+                }
+                finally
+                {
+                    if (removed)
+                    {
+                        calledRuleStack.Set(s.ruleIndex);
                     }
                 }
             }
@@ -349,21 +343,22 @@ namespace Antlr4.Runtime.Atn
             for (int i_1 = 0; i_1 < n; i_1++)
             {
                 Transition t = s.Transition(i_1);
-                if (t.GetType() == typeof(RuleTransition))
+                if (t is RuleTransition)
                 {
-                    if (calledRuleStack.Get(((RuleTransition)t).target.ruleIndex))
+                    RuleTransition ruleTransition = (RuleTransition)t;
+                    if (calledRuleStack.Get(ruleTransition.ruleIndex))
                     {
                         continue;
                     }
-                    PredictionContext newContext = ctx.GetChild(((RuleTransition)t).followState.stateNumber);
+                    PredictionContext newContext = ctx.GetChild(ruleTransition.followState.stateNumber);
                     try
                     {
-                        calledRuleStack.Set(((RuleTransition)t).target.ruleIndex);
+                        calledRuleStack.Set(ruleTransition.ruleIndex);
                         Look(t.target, stopState, newContext, look, lookBusy, calledRuleStack, seeThruPreds, addEOF);
                     }
                     finally
                     {
-                        calledRuleStack.Clear(((RuleTransition)t).target.ruleIndex);
+                        calledRuleStack.Clear(ruleTransition.ruleIndex);
                     }
                 }
                 else

runtime/CSharp/Antlr4.Runtime/Atn/LexerATNSimulator.cs

@@ -59,13 +59,12 @@ namespace Antlr4.Runtime.Atn
         /// and current character position in that line. Note that the Lexer is
         /// tracking the starting line and characterization of the token. These
         /// variables track the "state" of the simulator when it hits an accept state.
-        /// <p/>
-        /// We track these variables separately for the DFA and ATN simulation
+        /// <p>We track these variables separately for the DFA and ATN simulation
         /// because the DFA simulation often has to fail over to the ATN
         /// simulation. If the ATN simulation fails, we need the DFA to fall
         /// back to its previously accepted state, if any. If the ATN succeeds,
         /// then the ATN does the accept and the DFA simulator that invoked it
-        /// can simply return the predicted token type.
+        /// can simply return the predicted token type.</p>
         /// </summary>
         protected internal class SimState
         {
@@ -302,9 +301,12 @@ namespace Antlr4.Runtime.Atn
             if (reach.IsEmpty())
             {
                 // we got nowhere on t from s
-                // we got nowhere on t, don't throw out this knowledge; it'd
-                // cause a failover from DFA later.
-                AddDFAEdge(s, t, Error);
+                if (!reach.HasSemanticContext)
+                {
+                    // we got nowhere on t, don't throw out this knowledge; it'd
+                    // cause a failover from DFA later.
+                    AddDFAEdge(s, t, Error);
+                }
                 // stop when we can't match any more char
                 return Error;
             }
@@ -316,9 +318,8 @@ namespace Antlr4.Runtime.Atn
         {
             if (prevAccept.dfaState != null)
             {
-                int ruleIndex = prevAccept.dfaState.lexerRuleIndex;
-                int actionIndex = prevAccept.dfaState.lexerActionIndex;
-                Accept(input, ruleIndex, actionIndex, prevAccept.index, prevAccept.line, prevAccept.charPos);
+                LexerActionExecutor lexerActionExecutor = prevAccept.dfaState.lexerActionExecutor;
+                Accept(input, lexerActionExecutor, startIndex, prevAccept.index, prevAccept.line, prevAccept.charPos);
                 return prevAccept.dfaState.prediction;
             }
             else
@@ -348,7 +349,8 @@ namespace Antlr4.Runtime.Atn
             int skipAlt = ATN.InvalidAltNumber;
             foreach (ATNConfig c in closure)
             {
-                if (c.Alt == skipAlt)
+                bool currentAltReachedAcceptState = c.Alt == skipAlt;
+                if (currentAltReachedAcceptState && c.HasPassedThroughNonGreedyDecision())
                 {
                     continue;
                 }
@@ -360,7 +362,12 @@ namespace Antlr4.Runtime.Atn
                     ATNState target = GetReachableTarget(trans, t);
                     if (target != null)
                     {
-                        if (Closure(input, c.Transform(target), reach, true))
+                        LexerActionExecutor lexerActionExecutor = c.ActionExecutor;
+                        if (lexerActionExecutor != null)
+                        {
+                            lexerActionExecutor = lexerActionExecutor.FixOffsetBeforeMatch(input.Index - startIndex);
+                        }
+                        if (Closure(input, c.Transform(target, lexerActionExecutor, true), reach, currentAltReachedAcceptState, true))
                         {
                             // any remaining configs for this alt have a lower priority than
                             // the one that just reached an accept state.
@@ -372,12 +379,8 @@ namespace Antlr4.Runtime.Atn
             }
         }
 
-        protected internal virtual void Accept(ICharStream input, int ruleIndex, int actionIndex, int index, int line, int charPos)
+        protected internal virtual void Accept(ICharStream input, LexerActionExecutor lexerActionExecutor, int startIndex, int index, int line, int charPos)
         {
-            if (actionIndex >= 0 && recog != null)
-            {
-                recog.Action(null, ruleIndex, actionIndex);
-            }
             // seek to after last char in token
             input.Seek(index);
             this.line = line;
@@ -386,6 +389,10 @@ namespace Antlr4.Runtime.Atn
             {
                 Consume(input);
             }
+            if (lexerActionExecutor != null && recog != null)
+            {
+                lexerActionExecutor.Execute(recog, input, startIndex);
+            }
         }
 
         [Nullable]
@@ -407,7 +414,7 @@ namespace Antlr4.Runtime.Atn
             {
                 ATNState target = p.Transition(i).target;
                 ATNConfig c = ATNConfig.Create(target, i + 1, initialContext);
-                Closure(input, c, configs, false);
+                Closure(input, c, configs, false, false);
             }
             return configs;
         }
@@ -433,7 +440,7 @@ namespace Antlr4.Runtime.Atn
         /// <code>false</code>
         /// .
         /// </returns>
-        protected internal virtual bool Closure(ICharStream input, ATNConfig config, ATNConfigSet configs, bool speculative)
+        protected internal virtual bool Closure(ICharStream input, ATNConfig config, ATNConfigSet configs, bool currentAltReachedAcceptState, bool speculative)
         {
             if (config.State is RuleStopState)
             {
@@ -447,8 +454,8 @@ namespace Antlr4.Runtime.Atn
                 {
                     if (context.HasEmpty)
                     {
-                        configs.AddItem(config.Transform(config.State, PredictionContext.EmptyFull));
-                        return true;
+                        configs.AddItem(config.Transform(config.State, PredictionContext.EmptyFull, true));
+                        currentAltReachedAcceptState = true;
                     }
                 }
                 for (int i = 0; i < context.Size; i++)
@@ -462,17 +469,17 @@ namespace Antlr4.Runtime.Atn
                     // "pop" return state
                     ATNState returnState = atn.states[returnStateNumber];
                     ATNConfig c = ATNConfig.Create(returnState, config.Alt, newContext);
-                    if (Closure(input, c, configs, speculative))
-                    {
-                        return true;
-                    }
+                    currentAltReachedAcceptState = Closure(input, c, configs, currentAltReachedAcceptState, speculative);
                 }
-                return false;
+                return currentAltReachedAcceptState;
             }
             // optimization
             if (!config.State.OnlyHasEpsilonTransitions)
             {
-                configs.AddItem(config);
+                if (!currentAltReachedAcceptState || !config.HasPassedThroughNonGreedyDecision())
+                {
+                    configs.AddItem(config);
+                }
             }
             ATNState p = config.State;
             for (int i_1 = 0; i_1 < p.NumberOfOptimizedTransitions; i_1++)
@@ -481,13 +488,10 @@ namespace Antlr4.Runtime.Atn
                 ATNConfig c = GetEpsilonTarget(input, config, t, configs, speculative);
                 if (c != null)
                 {
-                    if (Closure(input, c, configs, speculative))
-                    {
-                        return true;
-                    }
+                    currentAltReachedAcceptState = Closure(input, c, configs, currentAltReachedAcceptState, speculative);
                 }
             }
-            return false;
+            return currentAltReachedAcceptState;
         }
 
         // side-effect: can alter configs.hasSemanticContext
@@ -502,12 +506,12 @@ namespace Antlr4.Runtime.Atn
                     RuleTransition ruleTransition = (RuleTransition)t;
                     if (optimize_tail_calls && ruleTransition.optimizedTailCall && !config.Context.HasEmpty)
                     {
-                        c = config.Transform(t.target);
+                        c = config.Transform(t.target, true);
                     }
                     else
                     {
                         PredictionContext newContext = config.Context.GetChild(ruleTransition.followState.stateNumber);
-                        c = config.Transform(t.target, newContext);
+                        c = config.Transform(t.target, newContext, true);
                     }
                     break;
                 }
@@ -523,7 +527,7 @@ namespace Antlr4.Runtime.Atn
                     configs.MarkExplicitSemanticContext();
                     if (EvaluatePredicate(input, pt.ruleIndex, pt.predIndex, speculative))
                     {
-                        c = config.Transform(t.target);
+                        c = config.Transform(t.target, true);
                     }
                     else
                     {
@@ -534,14 +538,36 @@ namespace Antlr4.Runtime.Atn
 
                 case TransitionType.Action:
                 {
-                    // ignore actions; just exec one per rule upon accept
-                    c = config.Transform(t.target, ((ActionTransition)t).actionIndex);
-                    break;
+                    if (config.Context.HasEmpty)
+                    {
+                        // execute actions anywhere in the start rule for a token.
+                        //
+                        // TODO: if the entry rule is invoked recursively, some
+                        // actions may be executed during the recursive call. The
+                        // problem can appear when hasEmpty() is true but
+                        // isEmpty() is false. In this case, the config needs to be
+                        // split into two contexts - one with just the empty path
+                        // and another with everything but the empty path.
+                        // Unfortunately, the current algorithm does not allow
+                        // getEpsilonTarget to return two configurations, so
+                        // additional modifications are needed before we can support
+                        // the split operation.
+                        LexerActionExecutor lexerActionExecutor = LexerActionExecutor.Append(config.ActionExecutor, atn.lexerActions[((ActionTransition)t).actionIndex]);
+                        c = config.Transform(t.target, lexerActionExecutor, true);
+                        break;
+                    }
+                    else
+                    {
+                        // ignore actions in referenced rules
+                        c = config.Transform(t.target, true);
+                        break;
+                    }
+                    goto case TransitionType.Epsilon;
                 }
 
                 case TransitionType.Epsilon:
                 {
-                    c = config.Transform(t.target);
+                    c = config.Transform(t.target, true);
                     break;
                 }
 
@@ -557,8 +583,7 @@ namespace Antlr4.Runtime.Atn
         /// <summary>Evaluate a predicate specified in the lexer.</summary>
         /// <remarks>
         /// Evaluate a predicate specified in the lexer.
-        /// <p/>
-        /// If
+        /// <p>If
         /// <code>speculative</code>
         /// is
         /// <code>true</code>
@@ -581,7 +606,7 @@ namespace Antlr4.Runtime.Atn
         /// to the original state before returning (i.e. undo the actions made by the
         /// call to
         /// <see cref="Consume(Antlr4.Runtime.ICharStream)">Consume(Antlr4.Runtime.ICharStream)</see>
-        /// .
+        /// .</p>
         /// </remarks>
         /// <param name="input">The input stream.</param>
         /// <param name="ruleIndex">The rule containing the predicate.</param>
@@ -697,9 +722,8 @@ namespace Antlr4.Runtime.Atn
             if (firstConfigWithRuleStopState != null)
             {
                 newState.isAcceptState = true;
-                newState.lexerRuleIndex = firstConfigWithRuleStopState.State.ruleIndex;
-                newState.lexerActionIndex = firstConfigWithRuleStopState.ActionIndex;
-                newState.prediction = atn.ruleToTokenType[newState.lexerRuleIndex];
+                newState.lexerActionExecutor = firstConfigWithRuleStopState.ActionExecutor;
+                newState.prediction = atn.ruleToTokenType[firstConfigWithRuleStopState.State.ruleIndex];
             }
             return atn.modeToDFA[mode].AddState(newState);
         }

runtime/CSharp/Antlr4.Runtime/Atn/LexerActionExecutor.cs

@@ -0,0 +1,293 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using Antlr4.Runtime;
+using Antlr4.Runtime.Atn;
+using Antlr4.Runtime.Misc;
+using Sharpen;
+
+namespace Antlr4.Runtime.Atn
+{
+    /// <summary>
+    /// Represents an executor for a sequence of lexer actions which traversed during
+    /// the matching operation of a lexer rule (token).
+    /// </summary>
+    /// <remarks>
+    /// Represents an executor for a sequence of lexer actions which traversed during
+    /// the matching operation of a lexer rule (token).
+    /// <p>The executor tracks position information for position-dependent lexer actions
+    /// efficiently, ensuring that actions appearing only at the end of the rule do
+    /// not cause bloating of the
+    /// <see cref="Antlr4.Runtime.Dfa.DFA">Antlr4.Runtime.Dfa.DFA</see>
+    /// created for the lexer.</p>
+    /// </remarks>
+    /// <author>Sam Harwell</author>
+    /// <since>4.2</since>
+    public class LexerActionExecutor
+    {
+        [NotNull]
+        private readonly ILexerAction[] lexerActions;
+
+        /// <summary>
+        /// Caches the result of
+        /// <see cref="hashCode">hashCode</see>
+        /// since the hash code is an element
+        /// of the performance-critical
+        /// <see cref="LexerATNConfig#hashCode">LexerATNConfig#hashCode</see>
+        /// operation.
+        /// </summary>
+        private readonly int hashCode;
+
+        /// <summary>
+        /// Constructs an executor for a sequence of
+        /// <see cref="ILexerAction">ILexerAction</see>
+        /// actions.
+        /// </summary>
+        /// <param name="lexerActions">The lexer actions to execute.</param>
+        public LexerActionExecutor(ILexerAction[] lexerActions)
+        {
+            this.lexerActions = lexerActions;
+            int hash = MurmurHash.Initialize();
+            foreach (ILexerAction lexerAction in lexerActions)
+            {
+                hash = MurmurHash.Update(hash, lexerAction);
+            }
+            this.hashCode = MurmurHash.Finish(hash, lexerActions.Length);
+        }
+
+        /// <summary>
+        /// Creates a
+        /// <see cref="LexerActionExecutor">LexerActionExecutor</see>
+        /// which executes the actions for
+        /// the input
+        /// <code>lexerActionExecutor</code>
+        /// followed by a specified
+        /// <code>lexerAction</code>
+        /// .
+        /// </summary>
+        /// <param name="lexerActionExecutor">
+        /// The executor for actions already traversed by
+        /// the lexer while matching a token within a particular
+        /// <see cref="ATNConfig">ATNConfig</see>
+        /// . If this is
+        /// <code>null</code>
+        /// , the method behaves as though
+        /// it were an empty executor.
+        /// </param>
+        /// <param name="lexerAction">
+        /// The lexer action to execute after the actions
+        /// specified in
+        /// <code>lexerActionExecutor</code>
+        /// .
+        /// </param>
+        /// <returns>
+        /// A
+        /// <see cref="LexerActionExecutor">LexerActionExecutor</see>
+        /// for executing the combine actions
+        /// of
+        /// <code>lexerActionExecutor</code>
+        /// and
+        /// <code>lexerAction</code>
+        /// .
+        /// </returns>
+        [NotNull]
+        public static Antlr4.Runtime.Atn.LexerActionExecutor Append(Antlr4.Runtime.Atn.LexerActionExecutor lexerActionExecutor, ILexerAction lexerAction)
+        {
+            if (lexerActionExecutor == null)
+            {
+                return new Antlr4.Runtime.Atn.LexerActionExecutor(new ILexerAction[] { lexerAction });
+            }
+            ILexerAction[] lexerActions = Arrays.CopyOf(lexerActionExecutor.lexerActions, lexerActionExecutor.lexerActions.Length + 1);
+            lexerActions[lexerActions.Length - 1] = lexerAction;
+            return new Antlr4.Runtime.Atn.LexerActionExecutor(lexerActions);
+        }
+
+        /// <summary>
+        /// Creates a
+        /// <see cref="LexerActionExecutor">LexerActionExecutor</see>
+        /// which encodes the current offset
+        /// for position-dependent lexer actions.
+        /// <p>Normally, when the executor encounters lexer actions where
+        /// <see cref="ILexerAction.IsPositionDependent()">ILexerAction.IsPositionDependent()</see>
+        /// returns
+        /// <code>true</code>
+        /// , it calls
+        /// <see cref="Antlr4.Runtime.IIntStream.Seek(int)">Antlr4.Runtime.IIntStream.Seek(int)</see>
+        /// on the input
+        /// <see cref="Antlr4.Runtime.ICharStream">Antlr4.Runtime.ICharStream</see>
+        /// to set the input
+        /// position to the <em>end</em> of the current token. This behavior provides
+        /// for efficient DFA representation of lexer actions which appear at the end
+        /// of a lexer rule, even when the lexer rule matches a variable number of
+        /// characters.</p>
+        /// <p>Prior to traversing a match transition in the ATN, the current offset
+        /// from the token start index is assigned to all position-dependent lexer
+        /// actions which have not already been assigned a fixed offset. By storing
+        /// the offsets relative to the token start index, the DFA representation of
+        /// lexer actions which appear in the middle of tokens remains efficient due
+        /// to sharing among tokens of the same length, regardless of their absolute
+        /// position in the input stream.</p>
+        /// <p>If the current executor already has offsets assigned to all
+        /// position-dependent lexer actions, the method returns
+        /// <code>this</code>
+        /// .</p>
+        /// </summary>
+        /// <param name="offset">
+        /// The current offset to assign to all position-dependent
+        /// lexer actions which do not already have offsets assigned.
+        /// </param>
+        /// <returns>
+        /// A
+        /// <see cref="LexerActionExecutor">LexerActionExecutor</see>
+        /// which stores input stream offsets
+        /// for all position-dependent lexer actions.
+        /// </returns>
+        public virtual Antlr4.Runtime.Atn.LexerActionExecutor FixOffsetBeforeMatch(int offset)
+        {
+            ILexerAction[] updatedLexerActions = null;
+            for (int i = 0; i < lexerActions.Length; i++)
+            {
+                if (lexerActions[i].IsPositionDependent() && !(lexerActions[i] is LexerIndexedCustomAction))
+                {
+                    if (updatedLexerActions == null)
+                    {
+                        updatedLexerActions = lexerActions.Clone();
+                    }
+                    updatedLexerActions[i] = new LexerIndexedCustomAction(offset, lexerActions[i]);
+                }
+            }
+            if (updatedLexerActions == null)
+            {
+                return this;
+            }
+            return new Antlr4.Runtime.Atn.LexerActionExecutor(updatedLexerActions);
+        }
+
+        /// <summary>Gets the lexer actions to be executed by this executor.</summary>
+        /// <remarks>Gets the lexer actions to be executed by this executor.</remarks>
+        /// <returns>The lexer actions to be executed by this executor.</returns>
+        [NotNull]
+        public virtual ILexerAction[] GetLexerActions()
+        {
+            return lexerActions;
+        }
+
+        /// <summary>
+        /// Execute the actions encapsulated by this executor within the context of a
+        /// particular
+        /// <see cref="Antlr4.Runtime.Lexer">Antlr4.Runtime.Lexer</see>
+        /// .
+        /// <p>This method calls
+        /// <see cref="Antlr4.Runtime.IIntStream.Seek(int)">Antlr4.Runtime.IIntStream.Seek(int)</see>
+        /// to set the position of the
+        /// <code>input</code>
+        /// 
+        /// <see cref="Antlr4.Runtime.ICharStream">Antlr4.Runtime.ICharStream</see>
+        /// prior to calling
+        /// <see cref="ILexerAction.Execute(Antlr4.Runtime.Lexer)">ILexerAction.Execute(Antlr4.Runtime.Lexer)</see>
+        /// on a position-dependent action. Before the
+        /// method returns, the input position will be restored to the same position
+        /// it was in when the method was invoked.</p>
+        /// </summary>
+        /// <param name="lexer">The lexer instance.</param>
+        /// <param name="input">
+        /// The input stream which is the source for the current token.
+        /// When this method is called, the current
+        /// <see cref="Antlr4.Runtime.IIntStream.Index()">Antlr4.Runtime.IIntStream.Index()</see>
+        /// for
+        /// <code>input</code>
+        /// should be the start of the following token, i.e. 1
+        /// character past the end of the current token.
+        /// </param>
+        /// <param name="startIndex">
+        /// The token start index. This value may be passed to
+        /// <see cref="Antlr4.Runtime.IIntStream.Seek(int)">Antlr4.Runtime.IIntStream.Seek(int)</see>
+        /// to set the
+        /// <code>input</code>
+        /// position to the beginning
+        /// of the token.
+        /// </param>
+        public virtual void Execute(Lexer lexer, ICharStream input, int startIndex)
+        {
+            bool requiresSeek = false;
+            int stopIndex = input.Index;
+            try
+            {
+                foreach (ILexerAction lexerAction in lexerActions)
+                {
+                    if (lexerAction is LexerIndexedCustomAction)
+                    {
+                        int offset = ((LexerIndexedCustomAction)lexerAction).GetOffset();
+                        input.Seek(startIndex + offset);
+                        lexerAction = ((LexerIndexedCustomAction)lexerAction).GetAction();
+                        requiresSeek = (startIndex + offset) != stopIndex;
+                    }
+                    else
+                    {
+                        if (lexerAction.IsPositionDependent())
+                        {
+                            input.Seek(stopIndex);
+                            requiresSeek = false;
+                        }
+                    }
+                    lexerAction.Execute(lexer);
+                }
+            }
+            finally
+            {
+                if (requiresSeek)
+                {
+                    input.Seek(stopIndex);
+                }
+            }
+        }
+
+        public override int GetHashCode()
+        {
+            return this.hashCode;
+        }
+
+        public override bool Equals(object obj)
+        {
+            if (obj == this)
+            {
+                return true;
+            }
+            else
+            {
+                if (!(obj is Antlr4.Runtime.Atn.LexerActionExecutor))
+                {
+                    return false;
+                }
+            }
+            Antlr4.Runtime.Atn.LexerActionExecutor other = (Antlr4.Runtime.Atn.LexerActionExecutor)obj;
+            return hashCode == other.hashCode && Arrays.Equals(lexerActions, other.lexerActions);
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/Atn/LexerActionType.cs

@@ -0,0 +1,48 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using Antlr4.Runtime.Atn;
+using Sharpen;
+
+namespace Antlr4.Runtime.Atn
+{
+    /// <author>Sam Harwell</author>
+    /// <since>4.2</since>
+    public enum LexerActionType
+    {
+        Channel,
+        Custom,
+        Mode,
+        More,
+        PopMode,
+        PushMode,
+        Skip,
+        Type
+    }
+}

runtime/CSharp/Antlr4.Runtime/Atn/LexerChannelAction.cs

@@ -0,0 +1,145 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using Antlr4.Runtime;
+using Antlr4.Runtime.Atn;
+using Antlr4.Runtime.Misc;
+using Sharpen;
+
+namespace Antlr4.Runtime.Atn
+{
+    /// <summary>
+    /// Implements the
+    /// <code>channel</code>
+    /// lexer action by calling
+    /// <see cref="Antlr4.Runtime.Lexer.Channel(int)">Antlr4.Runtime.Lexer.Channel(int)</see>
+    /// with the assigned channel.
+    /// </summary>
+    /// <author>Sam Harwell</author>
+    /// <since>4.2</since>
+    public sealed class LexerChannelAction : ILexerAction
+    {
+        private readonly int channel;
+
+        /// <summary>
+        /// Constructs a new
+        /// <code>channel</code>
+        /// action with the specified channel value.
+        /// </summary>
+        /// <param name="channel">
+        /// The channel value to pass to
+        /// <see cref="Antlr4.Runtime.Lexer.Channel(int)">Antlr4.Runtime.Lexer.Channel(int)</see>
+        /// .
+        /// </param>
+        public LexerChannelAction(int channel)
+        {
+            this.channel = channel;
+        }
+
+        /// <summary>
+        /// Gets the channel to use for the
+        /// <see cref="Antlr4.Runtime.IToken">Antlr4.Runtime.IToken</see>
+        /// created by the lexer.
+        /// </summary>
+        /// <returns>
+        /// The channel to use for the
+        /// <see cref="Antlr4.Runtime.IToken">Antlr4.Runtime.IToken</see>
+        /// created by the lexer.
+        /// </returns>
+        public int GetChannel()
+        {
+            return channel;
+        }
+
+        /// <summary><inheritDoc></inheritDoc></summary>
+        /// <returns>
+        /// This method returns
+        /// <see cref="LexerActionType.Channel">LexerActionType.Channel</see>
+        /// .
+        /// </returns>
+        public LexerActionType GetActionType()
+        {
+            return LexerActionType.Channel;
+        }
+
+        /// <summary><inheritDoc></inheritDoc></summary>
+        /// <returns>
+        /// This method returns
+        /// <code>false</code>
+        /// .
+        /// </returns>
+        public bool IsPositionDependent()
+        {
+            return false;
+        }
+
+        /// <summary>
+        /// <inheritDoc></inheritDoc>
+        /// <p>This action is implemented by calling
+        /// <see cref="Antlr4.Runtime.Lexer.Channel(int)">Antlr4.Runtime.Lexer.Channel(int)</see>
+        /// with the
+        /// value provided by
+        /// <see cref="GetChannel()">GetChannel()</see>
+        /// .</p>
+        /// </summary>
+        public void Execute(Lexer lexer)
+        {
+            lexer.Channel = channel;
+        }
+
+        public override int GetHashCode()
+        {
+            int hash = MurmurHash.Initialize();
+            hash = MurmurHash.Update(hash, (int)(GetActionType()));
+            hash = MurmurHash.Update(hash, channel);
+            return MurmurHash.Finish(hash, 2);
+        }
+
+        public override bool Equals(object obj)
+        {
+            if (obj == this)
+            {
+                return true;
+            }
+            else
+            {
+                if (!(obj is Antlr4.Runtime.Atn.LexerChannelAction))
+                {
+                    return false;
+                }
+            }
+            return channel == ((Antlr4.Runtime.Atn.LexerChannelAction)obj).channel;
+        }
+
+        public override string ToString()
+        {
+            return string.Format("channel(%d)", channel);
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/Atn/LexerCustomAction.cs

@@ -0,0 +1,174 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using Antlr4.Runtime;
+using Antlr4.Runtime.Atn;
+using Antlr4.Runtime.Misc;
+using Sharpen;
+
+namespace Antlr4.Runtime.Atn
+{
+    /// <summary>
+    /// Executes a custom lexer action by calling
+    /// <see cref="Antlr4.Runtime.Recognizer{Symbol, ATNInterpreter}.Action(Antlr4.Runtime.RuleContext, int, int)">Antlr4.Runtime.Recognizer&lt;Symbol, ATNInterpreter&gt;.Action(Antlr4.Runtime.RuleContext, int, int)</see>
+    /// with the
+    /// rule and action indexes assigned to the custom action. The implementation of
+    /// a custom action is added to the generated code for the lexer in an override
+    /// of
+    /// <see cref="Antlr4.Runtime.Recognizer{Symbol, ATNInterpreter}.Action(Antlr4.Runtime.RuleContext, int, int)">Antlr4.Runtime.Recognizer&lt;Symbol, ATNInterpreter&gt;.Action(Antlr4.Runtime.RuleContext, int, int)</see>
+    /// when the grammar is compiled.
+    /// <p>This class may represent embedded actions created with the <code>{...}</code>
+    /// syntax in ANTLR 4, as well as actions created for lexer commands where the
+    /// command argument could not be evaluated when the grammar was compiled.</p>
+    /// </summary>
+    /// <author>Sam Harwell</author>
+    /// <since>4.2</since>
+    public sealed class LexerCustomAction : ILexerAction
+    {
+        private readonly int ruleIndex;
+
+        private readonly int actionIndex;
+
+        /// <summary>
+        /// Constructs a custom lexer action with the specified rule and action
+        /// indexes.
+        /// </summary>
+        /// <remarks>
+        /// Constructs a custom lexer action with the specified rule and action
+        /// indexes.
+        /// </remarks>
+        /// <param name="ruleIndex">
+        /// The rule index to use for calls to
+        /// <see cref="Antlr4.Runtime.Recognizer{Symbol, ATNInterpreter}.Action(Antlr4.Runtime.RuleContext, int, int)">Antlr4.Runtime.Recognizer&lt;Symbol, ATNInterpreter&gt;.Action(Antlr4.Runtime.RuleContext, int, int)</see>
+        /// .
+        /// </param>
+        /// <param name="actionIndex">
+        /// The action index to use for calls to
+        /// <see cref="Antlr4.Runtime.Recognizer{Symbol, ATNInterpreter}.Action(Antlr4.Runtime.RuleContext, int, int)">Antlr4.Runtime.Recognizer&lt;Symbol, ATNInterpreter&gt;.Action(Antlr4.Runtime.RuleContext, int, int)</see>
+        /// .
+        /// </param>
+        public LexerCustomAction(int ruleIndex, int actionIndex)
+        {
+            this.ruleIndex = ruleIndex;
+            this.actionIndex = actionIndex;
+        }
+
+        /// <summary>
+        /// Gets the rule index to use for calls to
+        /// <see cref="Antlr4.Runtime.Recognizer{Symbol, ATNInterpreter}.Action(Antlr4.Runtime.RuleContext, int, int)">Antlr4.Runtime.Recognizer&lt;Symbol, ATNInterpreter&gt;.Action(Antlr4.Runtime.RuleContext, int, int)</see>
+        /// .
+        /// </summary>
+        /// <returns>The rule index for the custom action.</returns>
+        public int GetRuleIndex()
+        {
+            return ruleIndex;
+        }
+
+        /// <summary>
+        /// Gets the action index to use for calls to
+        /// <see cref="Antlr4.Runtime.Recognizer{Symbol, ATNInterpreter}.Action(Antlr4.Runtime.RuleContext, int, int)">Antlr4.Runtime.Recognizer&lt;Symbol, ATNInterpreter&gt;.Action(Antlr4.Runtime.RuleContext, int, int)</see>
+        /// .
+        /// </summary>
+        /// <returns>The action index for the custom action.</returns>
+        public int GetActionIndex()
+        {
+            return actionIndex;
+        }
+
+        /// <summary><inheritDoc></inheritDoc></summary>
+        /// <returns>
+        /// This method returns
+        /// <see cref="LexerActionType.Custom">LexerActionType.Custom</see>
+        /// .
+        /// </returns>
+        public LexerActionType GetActionType()
+        {
+            return LexerActionType.Custom;
+        }
+
+        /// <summary>Gets whether the lexer action is position-dependent.</summary>
+        /// <remarks>
+        /// Gets whether the lexer action is position-dependent. Position-dependent
+        /// actions may have different semantics depending on the
+        /// <see cref="Antlr4.Runtime.ICharStream">Antlr4.Runtime.ICharStream</see>
+        /// index at the time the action is executed.
+        /// <p>Custom actions are position-dependent since they may represent a
+        /// user-defined embedded action which makes calls to methods like
+        /// <see cref="Antlr4.Runtime.Lexer.Text()">Antlr4.Runtime.Lexer.Text()</see>
+        /// .</p>
+        /// </remarks>
+        /// <returns>
+        /// This method returns
+        /// <code>true</code>
+        /// .
+        /// </returns>
+        public bool IsPositionDependent()
+        {
+            return true;
+        }
+
+        /// <summary>
+        /// <inheritDoc></inheritDoc>
+        /// <p>Custom actions are implemented by calling
+        /// <see cref="Antlr4.Runtime.Recognizer{Symbol, ATNInterpreter}.Action(Antlr4.Runtime.RuleContext, int, int)">Antlr4.Runtime.Recognizer&lt;Symbol, ATNInterpreter&gt;.Action(Antlr4.Runtime.RuleContext, int, int)</see>
+        /// with the
+        /// appropriate rule and action indexes.</p>
+        /// </summary>
+        public void Execute(Lexer lexer)
+        {
+            lexer.Action(null, ruleIndex, actionIndex);
+        }
+
+        public override int GetHashCode()
+        {
+            int hash = MurmurHash.Initialize();
+            hash = MurmurHash.Update(hash, (int)(GetActionType()));
+            hash = MurmurHash.Update(hash, ruleIndex);
+            hash = MurmurHash.Update(hash, actionIndex);
+            return MurmurHash.Finish(hash, 3);
+        }
+
+        public override bool Equals(object obj)
+        {
+            if (obj == this)
+            {
+                return true;
+            }
+            else
+            {
+                if (!(obj is Antlr4.Runtime.Atn.LexerCustomAction))
+                {
+                    return false;
+                }
+            }
+            Antlr4.Runtime.Atn.LexerCustomAction other = (Antlr4.Runtime.Atn.LexerCustomAction)obj;
+            return ruleIndex == other.ruleIndex && actionIndex == other.actionIndex;
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/Atn/LexerIndexedCustomAction.cs

@@ -0,0 +1,189 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using Antlr4.Runtime;
+using Antlr4.Runtime.Atn;
+using Antlr4.Runtime.Misc;
+using Sharpen;
+
+namespace Antlr4.Runtime.Atn
+{
+    /// <summary>
+    /// This implementation of
+    /// <see cref="ILexerAction">ILexerAction</see>
+    /// is used for tracking input offsets
+    /// for position-dependent actions within a
+    /// <see cref="LexerActionExecutor">LexerActionExecutor</see>
+    /// .
+    /// <p>This action is not serialized as part of the ATN, and is only required for
+    /// position-dependent lexer actions which appear at a location other than the
+    /// end of a rule. For more information about DFA optimizations employed for
+    /// lexer actions, see
+    /// <see cref="LexerActionExecutor.Append(LexerActionExecutor, ILexerAction)">LexerActionExecutor.Append(LexerActionExecutor, ILexerAction)</see>
+    /// and
+    /// <see cref="LexerActionExecutor.FixOffsetBeforeMatch(int)">LexerActionExecutor.FixOffsetBeforeMatch(int)</see>
+    /// .</p>
+    /// </summary>
+    /// <author>Sam Harwell</author>
+    /// <since>4.2</since>
+    public sealed class LexerIndexedCustomAction : ILexerAction
+    {
+        private readonly int offset;
+
+        private readonly ILexerAction action;
+
+        /// <summary>
+        /// Constructs a new indexed custom action by associating a character offset
+        /// with a
+        /// <see cref="ILexerAction">ILexerAction</see>
+        /// .
+        /// <p>Note: This class is only required for lexer actions for which
+        /// <see cref="ILexerAction.IsPositionDependent()">ILexerAction.IsPositionDependent()</see>
+        /// returns
+        /// <code>true</code>
+        /// .</p>
+        /// </summary>
+        /// <param name="offset">
+        /// The offset into the input
+        /// <see cref="Antlr4.Runtime.ICharStream">Antlr4.Runtime.ICharStream</see>
+        /// , relative to
+        /// the token start index, at which the specified lexer action should be
+        /// executed.
+        /// </param>
+        /// <param name="action">
+        /// The lexer action to execute at a particular offset in the
+        /// input
+        /// <see cref="Antlr4.Runtime.ICharStream">Antlr4.Runtime.ICharStream</see>
+        /// .
+        /// </param>
+        public LexerIndexedCustomAction(int offset, ILexerAction action)
+        {
+            this.offset = offset;
+            this.action = action;
+        }
+
+        /// <summary>
+        /// Gets the location in the input
+        /// <see cref="Antlr4.Runtime.ICharStream">Antlr4.Runtime.ICharStream</see>
+        /// at which the lexer
+        /// action should be executed. The value is interpreted as an offset relative
+        /// to the token start index.
+        /// </summary>
+        /// <returns>
+        /// The location in the input
+        /// <see cref="Antlr4.Runtime.ICharStream">Antlr4.Runtime.ICharStream</see>
+        /// at which the lexer
+        /// action should be executed.
+        /// </returns>
+        public int GetOffset()
+        {
+            return offset;
+        }
+
+        /// <summary>Gets the lexer action to execute.</summary>
+        /// <remarks>Gets the lexer action to execute.</remarks>
+        /// <returns>
+        /// A
+        /// <see cref="ILexerAction">ILexerAction</see>
+        /// object which executes the lexer action.
+        /// </returns>
+        [NotNull]
+        public ILexerAction GetAction()
+        {
+            return action;
+        }
+
+        /// <summary><inheritDoc></inheritDoc></summary>
+        /// <returns>
+        /// This method returns the result of calling
+        /// <see cref="GetActionType()">GetActionType()</see>
+        /// on the
+        /// <see cref="ILexerAction">ILexerAction</see>
+        /// returned by
+        /// <see cref="GetAction()">GetAction()</see>
+        /// .
+        /// </returns>
+        public LexerActionType GetActionType()
+        {
+            return action.GetActionType();
+        }
+
+        /// <summary><inheritDoc></inheritDoc></summary>
+        /// <returns>
+        /// This method returns
+        /// <code>true</code>
+        /// .
+        /// </returns>
+        public bool IsPositionDependent()
+        {
+            return true;
+        }
+
+        /// <summary>
+        /// <inheritDoc></inheritDoc>
+        /// <p>This method calls
+        /// <see cref="Execute(Antlr4.Runtime.Lexer)">Execute(Antlr4.Runtime.Lexer)</see>
+        /// on the result of
+        /// <see cref="GetAction()">GetAction()</see>
+        /// using the provided
+        /// <code>lexer</code>
+        /// .</p>
+        /// </summary>
+        public void Execute(Lexer lexer)
+        {
+            // assume the input stream position was properly set by the calling code
+            action.Execute(lexer);
+        }
+
+        public override int GetHashCode()
+        {
+            int hash = MurmurHash.Initialize();
+            hash = MurmurHash.Update(hash, offset);
+            hash = MurmurHash.Update(hash, action);
+            return MurmurHash.Finish(hash, 2);
+        }
+
+        public override bool Equals(object obj)
+        {
+            if (obj == this)
+            {
+                return true;
+            }
+            else
+            {
+                if (!(obj is Antlr4.Runtime.Atn.LexerIndexedCustomAction))
+                {
+                    return false;
+                }
+            }
+            Antlr4.Runtime.Atn.LexerIndexedCustomAction other = (Antlr4.Runtime.Atn.LexerIndexedCustomAction)obj;
+            return offset == other.offset && action.Equals(other.action);
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/Atn/LexerModeAction.cs

@@ -0,0 +1,143 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using Antlr4.Runtime;
+using Antlr4.Runtime.Atn;
+using Antlr4.Runtime.Misc;
+using Sharpen;
+
+namespace Antlr4.Runtime.Atn
+{
+    /// <summary>
+    /// Implements the
+    /// <code>mode</code>
+    /// lexer action by calling
+    /// <see cref="Antlr4.Runtime.Lexer.Mode(int)">Antlr4.Runtime.Lexer.Mode(int)</see>
+    /// with
+    /// the assigned mode.
+    /// </summary>
+    /// <author>Sam Harwell</author>
+    /// <since>4.2</since>
+    public sealed class LexerModeAction : ILexerAction
+    {
+        private readonly int mode;
+
+        /// <summary>
+        /// Constructs a new
+        /// <code>mode</code>
+        /// action with the specified mode value.
+        /// </summary>
+        /// <param name="mode">
+        /// The mode value to pass to
+        /// <see cref="Antlr4.Runtime.Lexer.Mode(int)">Antlr4.Runtime.Lexer.Mode(int)</see>
+        /// .
+        /// </param>
+        public LexerModeAction(int mode)
+        {
+            this.mode = mode;
+        }
+
+        /// <summary>Get the lexer mode this action should transition the lexer to.</summary>
+        /// <remarks>Get the lexer mode this action should transition the lexer to.</remarks>
+        /// <returns>
+        /// The lexer mode for this
+        /// <code>mode</code>
+        /// command.
+        /// </returns>
+        public int GetMode()
+        {
+            return mode;
+        }
+
+        /// <summary><inheritDoc></inheritDoc></summary>
+        /// <returns>
+        /// This method returns
+        /// <see cref="LexerActionType.Mode">LexerActionType.Mode</see>
+        /// .
+        /// </returns>
+        public LexerActionType GetActionType()
+        {
+            return LexerActionType.Mode;
+        }
+
+        /// <summary><inheritDoc></inheritDoc></summary>
+        /// <returns>
+        /// This method returns
+        /// <code>false</code>
+        /// .
+        /// </returns>
+        public bool IsPositionDependent()
+        {
+            return false;
+        }
+
+        /// <summary>
+        /// <inheritDoc></inheritDoc>
+        /// <p>This action is implemented by calling
+        /// <see cref="Antlr4.Runtime.Lexer.Mode(int)">Antlr4.Runtime.Lexer.Mode(int)</see>
+        /// with the
+        /// value provided by
+        /// <see cref="GetMode()">GetMode()</see>
+        /// .</p>
+        /// </summary>
+        public void Execute(Lexer lexer)
+        {
+            lexer.Mode(mode);
+        }
+
+        public override int GetHashCode()
+        {
+            int hash = MurmurHash.Initialize();
+            hash = MurmurHash.Update(hash, (int)(GetActionType()));
+            hash = MurmurHash.Update(hash, mode);
+            return MurmurHash.Finish(hash, 2);
+        }
+
+        public override bool Equals(object obj)
+        {
+            if (obj == this)
+            {
+                return true;
+            }
+            else
+            {
+                if (!(obj is Antlr4.Runtime.Atn.LexerModeAction))
+                {
+                    return false;
+                }
+            }
+            return mode == ((Antlr4.Runtime.Atn.LexerModeAction)obj).mode;
+        }
+
+        public override string ToString()
+        {
+            return string.Format("mode(%d)", mode);
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/Atn/LexerMoreAction.cs

@@ -0,0 +1,117 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using Antlr4.Runtime;
+using Antlr4.Runtime.Atn;
+using Antlr4.Runtime.Misc;
+using Sharpen;
+
+namespace Antlr4.Runtime.Atn
+{
+    /// <summary>
+    /// Implements the
+    /// <code>more</code>
+    /// lexer action by calling
+    /// <see cref="Antlr4.Runtime.Lexer.More()">Antlr4.Runtime.Lexer.More()</see>
+    /// .
+    /// <p>The
+    /// <code>more</code>
+    /// command does not have any parameters, so this action is
+    /// implemented as a singleton instance exposed by
+    /// <see cref="Instance">Instance</see>
+    /// .</p>
+    /// </summary>
+    /// <author>Sam Harwell</author>
+    /// <since>4.2</since>
+    public sealed class LexerMoreAction : ILexerAction
+    {
+        /// <summary>Provides a singleton instance of this parameterless lexer action.</summary>
+        /// <remarks>Provides a singleton instance of this parameterless lexer action.</remarks>
+        public static readonly Antlr4.Runtime.Atn.LexerMoreAction Instance = new Antlr4.Runtime.Atn.LexerMoreAction();
+
+        /// <summary>
+        /// Constructs the singleton instance of the lexer
+        /// <code>more</code>
+        /// command.
+        /// </summary>
+        private LexerMoreAction()
+        {
+        }
+
+        /// <summary><inheritDoc></inheritDoc></summary>
+        /// <returns>
+        /// This method returns
+        /// <see cref="LexerActionType.More">LexerActionType.More</see>
+        /// .
+        /// </returns>
+        public LexerActionType GetActionType()
+        {
+            return LexerActionType.More;
+        }
+
+        /// <summary><inheritDoc></inheritDoc></summary>
+        /// <returns>
+        /// This method returns
+        /// <code>false</code>
+        /// .
+        /// </returns>
+        public bool IsPositionDependent()
+        {
+            return false;
+        }
+
+        /// <summary>
+        /// <inheritDoc></inheritDoc>
+        /// <p>This action is implemented by calling
+        /// <see cref="Antlr4.Runtime.Lexer.More()">Antlr4.Runtime.Lexer.More()</see>
+        /// .</p>
+        /// </summary>
+        public void Execute(Lexer lexer)
+        {
+            lexer.More();
+        }
+
+        public override int GetHashCode()
+        {
+            int hash = MurmurHash.Initialize();
+            hash = MurmurHash.Update(hash, (int)(GetActionType()));
+            return MurmurHash.Finish(hash, 1);
+        }
+
+        public override bool Equals(object obj)
+        {
+            return obj == this;
+        }
+
+        public override string ToString()
+        {
+            return "more";
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/Atn/LexerPopModeAction.cs

@@ -0,0 +1,117 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using Antlr4.Runtime;
+using Antlr4.Runtime.Atn;
+using Antlr4.Runtime.Misc;
+using Sharpen;
+
+namespace Antlr4.Runtime.Atn
+{
+    /// <summary>
+    /// Implements the
+    /// <code>popMode</code>
+    /// lexer action by calling
+    /// <see cref="Antlr4.Runtime.Lexer.PopMode()">Antlr4.Runtime.Lexer.PopMode()</see>
+    /// .
+    /// <p>The
+    /// <code>popMode</code>
+    /// command does not have any parameters, so this action is
+    /// implemented as a singleton instance exposed by
+    /// <see cref="Instance">Instance</see>
+    /// .</p>
+    /// </summary>
+    /// <author>Sam Harwell</author>
+    /// <since>4.2</since>
+    public sealed class LexerPopModeAction : ILexerAction
+    {
+        /// <summary>Provides a singleton instance of this parameterless lexer action.</summary>
+        /// <remarks>Provides a singleton instance of this parameterless lexer action.</remarks>
+        public static readonly Antlr4.Runtime.Atn.LexerPopModeAction Instance = new Antlr4.Runtime.Atn.LexerPopModeAction();
+
+        /// <summary>
+        /// Constructs the singleton instance of the lexer
+        /// <code>popMode</code>
+        /// command.
+        /// </summary>
+        private LexerPopModeAction()
+        {
+        }
+
+        /// <summary><inheritDoc></inheritDoc></summary>
+        /// <returns>
+        /// This method returns
+        /// <see cref="LexerActionType.PopMode">LexerActionType.PopMode</see>
+        /// .
+        /// </returns>
+        public LexerActionType GetActionType()
+        {
+            return LexerActionType.PopMode;
+        }
+
+        /// <summary><inheritDoc></inheritDoc></summary>
+        /// <returns>
+        /// This method returns
+        /// <code>false</code>
+        /// .
+        /// </returns>
+        public bool IsPositionDependent()
+        {
+            return false;
+        }
+
+        /// <summary>
+        /// <inheritDoc></inheritDoc>
+        /// <p>This action is implemented by calling
+        /// <see cref="Antlr4.Runtime.Lexer.PopMode()">Antlr4.Runtime.Lexer.PopMode()</see>
+        /// .</p>
+        /// </summary>
+        public void Execute(Lexer lexer)
+        {
+            lexer.PopMode();
+        }
+
+        public override int GetHashCode()
+        {
+            int hash = MurmurHash.Initialize();
+            hash = MurmurHash.Update(hash, (int)(GetActionType()));
+            return MurmurHash.Finish(hash, 1);
+        }
+
+        public override bool Equals(object obj)
+        {
+            return obj == this;
+        }
+
+        public override string ToString()
+        {
+            return "popMode";
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/Atn/LexerPushModeAction.cs

@@ -0,0 +1,142 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using Antlr4.Runtime;
+using Antlr4.Runtime.Atn;
+using Antlr4.Runtime.Misc;
+using Sharpen;
+
+namespace Antlr4.Runtime.Atn
+{
+    /// <summary>
+    /// Implements the
+    /// <code>pushMode</code>
+    /// lexer action by calling
+    /// <see cref="Antlr4.Runtime.Lexer.PushMode(int)">Antlr4.Runtime.Lexer.PushMode(int)</see>
+    /// with the assigned mode.
+    /// </summary>
+    /// <author>Sam Harwell</author>
+    /// <since>4.2</since>
+    public sealed class LexerPushModeAction : ILexerAction
+    {
+        private readonly int mode;
+
+        /// <summary>
+        /// Constructs a new
+        /// <code>pushMode</code>
+        /// action with the specified mode value.
+        /// </summary>
+        /// <param name="mode">
+        /// The mode value to pass to
+        /// <see cref="Antlr4.Runtime.Lexer.PushMode(int)">Antlr4.Runtime.Lexer.PushMode(int)</see>
+        /// .
+        /// </param>
+        public LexerPushModeAction(int mode)
+        {
+            this.mode = mode;
+        }
+
+        /// <summary>Get the lexer mode this action should transition the lexer to.</summary>
+        /// <remarks>Get the lexer mode this action should transition the lexer to.</remarks>
+        /// <returns>
+        /// The lexer mode for this
+        /// <code>pushMode</code>
+        /// command.
+        /// </returns>
+        public int GetMode()
+        {
+            return mode;
+        }
+
+        /// <summary><inheritDoc></inheritDoc></summary>
+        /// <returns>
+        /// This method returns
+        /// <see cref="LexerActionType.PushMode">LexerActionType.PushMode</see>
+        /// .
+        /// </returns>
+        public LexerActionType GetActionType()
+        {
+            return LexerActionType.PushMode;
+        }
+
+        /// <summary><inheritDoc></inheritDoc></summary>
+        /// <returns>
+        /// This method returns
+        /// <code>false</code>
+        /// .
+        /// </returns>
+        public bool IsPositionDependent()
+        {
+            return false;
+        }
+
+        /// <summary>
+        /// <inheritDoc></inheritDoc>
+        /// <p>This action is implemented by calling
+        /// <see cref="Antlr4.Runtime.Lexer.PushMode(int)">Antlr4.Runtime.Lexer.PushMode(int)</see>
+        /// with the
+        /// value provided by
+        /// <see cref="GetMode()">GetMode()</see>
+        /// .</p>
+        /// </summary>
+        public void Execute(Lexer lexer)
+        {
+            lexer.PushMode(mode);
+        }
+
+        public override int GetHashCode()
+        {
+            int hash = MurmurHash.Initialize();
+            hash = MurmurHash.Update(hash, (int)(GetActionType()));
+            hash = MurmurHash.Update(hash, mode);
+            return MurmurHash.Finish(hash, 2);
+        }
+
+        public override bool Equals(object obj)
+        {
+            if (obj == this)
+            {
+                return true;
+            }
+            else
+            {
+                if (!(obj is Antlr4.Runtime.Atn.LexerPushModeAction))
+                {
+                    return false;
+                }
+            }
+            return mode == ((Antlr4.Runtime.Atn.LexerPushModeAction)obj).mode;
+        }
+
+        public override string ToString()
+        {
+            return string.Format("pushMode(%d)", mode);
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/Atn/LexerSkipAction.cs

@@ -0,0 +1,117 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using Antlr4.Runtime;
+using Antlr4.Runtime.Atn;
+using Antlr4.Runtime.Misc;
+using Sharpen;
+
+namespace Antlr4.Runtime.Atn
+{
+    /// <summary>
+    /// Implements the
+    /// <code>skip</code>
+    /// lexer action by calling
+    /// <see cref="Antlr4.Runtime.Lexer.Skip()">Antlr4.Runtime.Lexer.Skip()</see>
+    /// .
+    /// <p>The
+    /// <code>skip</code>
+    /// command does not have any parameters, so this action is
+    /// implemented as a singleton instance exposed by
+    /// <see cref="Instance">Instance</see>
+    /// .</p>
+    /// </summary>
+    /// <author>Sam Harwell</author>
+    /// <since>4.2</since>
+    public sealed class LexerSkipAction : ILexerAction
+    {
+        /// <summary>Provides a singleton instance of this parameterless lexer action.</summary>
+        /// <remarks>Provides a singleton instance of this parameterless lexer action.</remarks>
+        public static readonly Antlr4.Runtime.Atn.LexerSkipAction Instance = new Antlr4.Runtime.Atn.LexerSkipAction();
+
+        /// <summary>
+        /// Constructs the singleton instance of the lexer
+        /// <code>skip</code>
+        /// command.
+        /// </summary>
+        private LexerSkipAction()
+        {
+        }
+
+        /// <summary><inheritDoc></inheritDoc></summary>
+        /// <returns>
+        /// This method returns
+        /// <see cref="LexerActionType.Skip">LexerActionType.Skip</see>
+        /// .
+        /// </returns>
+        public LexerActionType GetActionType()
+        {
+            return LexerActionType.Skip;
+        }
+
+        /// <summary><inheritDoc></inheritDoc></summary>
+        /// <returns>
+        /// This method returns
+        /// <code>false</code>
+        /// .
+        /// </returns>
+        public bool IsPositionDependent()
+        {
+            return false;
+        }
+
+        /// <summary>
+        /// <inheritDoc></inheritDoc>
+        /// <p>This action is implemented by calling
+        /// <see cref="Antlr4.Runtime.Lexer.Skip()">Antlr4.Runtime.Lexer.Skip()</see>
+        /// .</p>
+        /// </summary>
+        public void Execute(Lexer lexer)
+        {
+            lexer.Skip();
+        }
+
+        public override int GetHashCode()
+        {
+            int hash = MurmurHash.Initialize();
+            hash = MurmurHash.Update(hash, (int)(GetActionType()));
+            return MurmurHash.Finish(hash, 1);
+        }
+
+        public override bool Equals(object obj)
+        {
+            return obj == this;
+        }
+
+        public override string ToString()
+        {
+            return "skip";
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/Atn/LexerTypeAction.cs

@@ -0,0 +1,138 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using Antlr4.Runtime;
+using Antlr4.Runtime.Atn;
+using Antlr4.Runtime.Misc;
+using Sharpen;
+
+namespace Antlr4.Runtime.Atn
+{
+    /// <summary>
+    /// Implements the
+    /// <code>type</code>
+    /// lexer action by calling
+    /// <see cref="Antlr4.Runtime.Lexer.Type(int)">Antlr4.Runtime.Lexer.Type(int)</see>
+    /// with the assigned type.
+    /// </summary>
+    /// <author>Sam Harwell</author>
+    /// <since>4.2</since>
+    public class LexerTypeAction : ILexerAction
+    {
+        private readonly int type;
+
+        /// <summary>
+        /// Constructs a new
+        /// <code>type</code>
+        /// action with the specified token type value.
+        /// </summary>
+        /// <param name="type">
+        /// The type to assign to the token using
+        /// <see cref="Antlr4.Runtime.Lexer.Type(int)">Antlr4.Runtime.Lexer.Type(int)</see>
+        /// .
+        /// </param>
+        public LexerTypeAction(int type)
+        {
+            this.type = type;
+        }
+
+        /// <summary>Gets the type to assign to a token created by the lexer.</summary>
+        /// <remarks>Gets the type to assign to a token created by the lexer.</remarks>
+        /// <returns>The type to assign to a token created by the lexer.</returns>
+        public virtual int GetType()
+        {
+            return type;
+        }
+
+        /// <summary><inheritDoc></inheritDoc></summary>
+        /// <returns>
+        /// This method returns
+        /// <see cref="LexerActionType.Type">LexerActionType.Type</see>
+        /// .
+        /// </returns>
+        public virtual LexerActionType GetActionType()
+        {
+            return LexerActionType.Type;
+        }
+
+        /// <summary><inheritDoc></inheritDoc></summary>
+        /// <returns>
+        /// This method returns
+        /// <code>false</code>
+        /// .
+        /// </returns>
+        public virtual bool IsPositionDependent()
+        {
+            return false;
+        }
+
+        /// <summary>
+        /// <inheritDoc></inheritDoc>
+        /// <p>This action is implemented by calling
+        /// <see cref="Antlr4.Runtime.Lexer.Type(int)">Antlr4.Runtime.Lexer.Type(int)</see>
+        /// with the
+        /// value provided by
+        /// <see cref="GetType()">GetType()</see>
+        /// .</p>
+        /// </summary>
+        public virtual void Execute(Lexer lexer)
+        {
+            lexer.Type = type;
+        }
+
+        public override int GetHashCode()
+        {
+            int hash = MurmurHash.Initialize();
+            hash = MurmurHash.Update(hash, (int)(GetActionType()));
+            hash = MurmurHash.Update(hash, type);
+            return MurmurHash.Finish(hash, 2);
+        }
+
+        public override bool Equals(object obj)
+        {
+            if (obj == this)
+            {
+                return true;
+            }
+            else
+            {
+                if (!(obj is Antlr4.Runtime.Atn.LexerTypeAction))
+                {
+                    return false;
+                }
+            }
+            return type == ((Antlr4.Runtime.Atn.LexerTypeAction)obj).type;
+        }
+
+        public override string ToString()
+        {
+            return string.Format("type(%d)", type);
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/Atn/ParserATNSimulator.cs

@@ -37,165 +37,271 @@ using Sharpen;
 
 namespace Antlr4.Runtime.Atn
 {
-    /// <summary>The embodiment of the adaptive LL(*) parsing strategy.</summary>
+    /// <summary>The embodiment of the adaptive LL(*), ALL(*), parsing strategy.</summary>
     /// <remarks>
-    /// The embodiment of the adaptive LL(*) parsing strategy.
-    /// The basic complexity of the adaptive strategy makes it harder to
-    /// understand. We begin with ATN simulation to build paths in a
-    /// DFA. Subsequent prediction requests go through the DFA first. If
-    /// they reach a state without an edge for the current symbol, the
-    /// algorithm fails over to the ATN simulation to complete the DFA
-    /// path for the current input (until it finds a conflict state or
-    /// uniquely predicting state).
-    /// All of that is done without using the outer context because we
-    /// want to create a DFA that is not dependent upon the rule
-    /// invocation stack when we do a prediction.  One DFA works in all
-    /// contexts. We avoid using context not necessarily because it
-    /// slower, although it can be, but because of the DFA caching
-    /// problem.  The closure routine only considers the rule invocation
-    /// stack created during prediction beginning in the entry rule.  For
-    /// example, if prediction occurs without invoking another rule's
-    /// ATN, there are no context stacks in the configurations. When this
-    /// leads to a conflict, we don't know if it's an ambiguity or a
-    /// weakness in the strong LL(*) parsing strategy (versus full
-    /// LL(*)).
-    /// So, we simply retry the ATN simulation again, this time
-    /// using full outer context and filling a dummy DFA (to avoid
-    /// polluting the context insensitive DFA). Configuration context
-    /// stacks will be the full invocation stack from the start rule. If
-    /// we get a conflict using full context, then we can definitively
-    /// say we have a true ambiguity for that input sequence. If we don't
-    /// get a conflict, it implies that the decision is sensitive to the
-    /// outer context. (It is not context-sensitive in the sense of
-    /// context sensitive grammars.) We create a special DFA accept state
-    /// that maps rule context to a predicted alternative. That is the
-    /// only modification needed to handle full LL(*) prediction. In
-    /// general, full context prediction will use more lookahead than
-    /// necessary, but it pays to share the same DFA. For a schedule
-    /// proof that full context prediction uses that most the same amount
-    /// of lookahead as a context insensitive prediction, see the comment
-    /// on method retryWithContext().
-    /// So, the strategy is complex because we bounce back and forth from
-    /// the ATN to the DFA, simultaneously performing predictions and
-    /// extending the DFA according to previously unseen input
-    /// sequences. The retry with full context is a recursive call to the
-    /// same function naturally because it does the same thing, just with
-    /// a different initial context. The problem is, that we need to pass
-    /// in a "full context mode" parameter so that it knows to report
-    /// conflicts differently. It also knows not to do a retry, to avoid
-    /// infinite recursion, if it is already using full context.
-    /// Retry a simulation using full outer context.
-    /// One of the key assumptions here is that using full context
-    /// can use at most the same amount of input as a simulation
-    /// that is not useful context (i.e., it uses all possible contexts
-    /// that could invoke our entry rule. I believe that this is true
-    /// and the proof might go like this.
-    /// THEOREM:  The amount of input consumed during a full context
-    /// simulation is at most the amount of input consumed during a
-    /// non full context simulation.
-    /// PROOF: Let D be the DFA state at which non-context simulation
-    /// terminated. That means that D does not have a configuration for
-    /// which we can legally pursue more input. (It is legal to work only
-    /// on configurations for which there is no conflict with another
-    /// configuration.) Now we restrict ourselves to following ATN edges
-    /// associated with a single context. Choose any DFA state D' along
-    /// the path (same input) to D. That state has either the same number
-    /// of configurations or fewer. (If the number of configurations is
-    /// the same, then we have degenerated to the non-context case.) Now
-    /// imagine that we restrict to following edges associated with
-    /// another single context and that we reach DFA state D'' for the
-    /// same amount of input as D'. The non-context simulation merges D'
-    /// and D''. The union of the configuration sets either has the same
-    /// number of configurations as both D' and D'' or it has more. If it
-    /// has the same number, we are no worse off and the merge does not
-    /// force us to look for more input than we would otherwise have to
-    /// do. If the union has more configurations, it can introduce
-    /// conflicts but not new alternatives--we cannot conjure up alternatives
-    /// by computing closure on the DFA state.  Here are the cases for
-    /// D' union D'':
-    /// 1. No increase in configurations, D' = D''
-    /// 2. Add configuration that introduces a new alternative number.
-    /// This cannot happen because no new alternatives are introduced
-    /// while computing closure, even during start state computation.
-    /// 3. D'' adds a configuration that does not conflict with any
-    /// configuration in D'.  Simulating without context would then have
-    /// forced us to use more lookahead than D' (full context) alone.
-    /// 3. D'' adds a configuration that introduces a conflict with a
-    /// configuration in D'. There are 2 cases:
-    /// a. The conflict does not cause termination (D' union D''
-    /// is added to the work list). Again no context simulation requires
-    /// more input.
-    /// b. The conflict does cause termination, but this cannot happen.
-    /// By definition, we know that with ALL contexts merged we
-    /// don't terminate until D and D' uses less input than D. Therefore
-    /// no context simulation requires more input than full context
-    /// simulation.
-    /// We have covered all the cases and there is never a situation where
-    /// a single, full context simulation requires more input than a
-    /// no context simulation.
-    /// I spent a bunch of time thinking about this problem after finding
-    /// a case where context-sensitive ATN simulation looks beyond what they
-    /// no context simulation uses. the no context simulation for if then else
-    /// stops at the else whereas full context scans through to the end of the
-    /// statement to decide that the "else statement" clause is ambiguous. And
-    /// sometimes it is not ambiguous! Ok, I made an untrue assumption in my
-    /// proof which I won't bother going to. the important thing is what I'm
-    /// going to do about it. I thought I had a simple answer, but nope. It
-    /// turns out that the if then else case is perfect example of something
-    /// that has the following characteristics:
-    /// no context conflicts at k=1
-    /// full context at k=(1 + length of statement) can be both ambiguous and not
-    /// ambiguous depending on the input, though I think from different contexts.
-    /// But, the good news is that the k=1 case is a special case in that
-    /// SLL(1) and LL(1) have exactly the same power so we can conclude that
-    /// conflicts at k=1 are true ambiguities and we do not need to pursue
-    /// context-sensitive parsing. That covers a huge number of cases
-    /// including the if then else clause and the predicated precedence
-    /// parsing mechanism. whew! because that could be extremely expensive if
-    /// we had to do context.
-    /// Further, there is no point in doing full context if none of the
-    /// configurations dip into the outer context. This nicely handles cases
-    /// such as super constructor calls versus function calls. One grammar
-    /// might look like this:
-    /// ctorBody : '{' superCall? stat* '}' ;
-    /// Or, you might see something like
-    /// stat : superCall ';' | expression ';' | ... ;
-    /// In both cases I believe that no closure operations will dip into the
-    /// outer context. In the first case ctorBody in the worst case will stop
-    /// at the '}'. In the 2nd case it should stop at the ';'. Both cases
-    /// should stay within the entry rule and not dip into the outer context.
-    /// So, we now cover what I hope is the vast majority of the cases (in
-    /// particular the very important precedence parsing case). Anything that
-    /// needs k&gt;1 and dips into the outer context requires a full context
-    /// retry. In this case, I'm going to start out with a brain-dead solution
-    /// which is to mark the DFA state as context-sensitive when I get a
-    /// conflict. Any further DFA simulation that reaches that state will
-    /// launch an ATN simulation to get the prediction, without updating the
-    /// DFA or storing any context information. Later, I can make this more
-    /// efficient, but at least in this case I can guarantee that it will
-    /// always do the right thing. We are not making any assumptions about
-    /// lookahead depth.
-    /// Ok, writing this up so I can put in a comment.
-    /// Upon conflict in the no context simulation:
-    /// if k=1, report ambiguity and resolve to the minimum conflicting alternative
-    /// if k=1 and predicates, no report and include the predicate to
-    /// predicted alternative map in the DFA state
-    /// if k=* and we did not dip into the outer context, report ambiguity
-    /// and resolve to minimum conflicting alternative
-    /// if k&gt;1 and we dip into outer context, retry with full context
-    /// if conflict, report ambiguity and resolve to minimum conflicting
-    /// alternative, mark DFA as context-sensitive
-    /// If no conflict, report ctx sensitivity and mark DFA as context-sensitive
-    /// Technically, if full context k is less than no context k, we can
-    /// reuse the conflicting DFA state so we don't have to create special
-    /// DFA paths branching from context, but we can leave that for
-    /// optimization later if necessary.
-    /// if non-greedy, no report and resolve to the exit alternative
-    /// By default we do full context-sensitive LL(*) parsing not
-    /// Strong LL(*) parsing. If we fail with Strong LL(*) we
-    /// try full LL(*). That means we rewind and use context information
-    /// when closure operations fall off the end of the rule that
-    /// holds the decision were evaluating
+    /// The embodiment of the adaptive LL(*), ALL(*), parsing strategy.
+    /// <p>
+    /// The basic complexity of the adaptive strategy makes it harder to understand.
+    /// We begin with ATN simulation to build paths in a DFA. Subsequent prediction
+    /// requests go through the DFA first. If they reach a state without an edge for
+    /// the current symbol, the algorithm fails over to the ATN simulation to
+    /// complete the DFA path for the current input (until it finds a conflict state
+    /// or uniquely predicting state).</p>
+    /// <p>
+    /// All of that is done without using the outer context because we want to create
+    /// a DFA that is not dependent upon the rule invocation stack when we do a
+    /// prediction. One DFA works in all contexts. We avoid using context not
+    /// necessarily because it's slower, although it can be, but because of the DFA
+    /// caching problem. The closure routine only considers the rule invocation stack
+    /// created during prediction beginning in the decision rule. For example, if
+    /// prediction occurs without invoking another rule's ATN, there are no context
+    /// stacks in the configurations. When lack of context leads to a conflict, we
+    /// don't know if it's an ambiguity or a weakness in the strong LL(*) parsing
+    /// strategy (versus full LL(*)).</p>
+    /// <p>
+    /// When SLL yields a configuration set with conflict, we rewind the input and
+    /// retry the ATN simulation, this time using full outer context without adding
+    /// to the DFA. Configuration context stacks will be the full invocation stacks
+    /// from the start rule. If we get a conflict using full context, then we can
+    /// definitively say we have a true ambiguity for that input sequence. If we
+    /// don't get a conflict, it implies that the decision is sensitive to the outer
+    /// context. (It is not context-sensitive in the sense of context-sensitive
+    /// grammars.)</p>
+    /// <p>
+    /// The next time we reach this DFA state with an SLL conflict, through DFA
+    /// simulation, we will again retry the ATN simulation using full context mode.
+    /// This is slow because we can't save the results and have to "interpret" the
+    /// ATN each time we get that input.</p>
+    /// <p>
+    /// <strong>CACHING FULL CONTEXT PREDICTIONS</strong></p>
+    /// <p>
+    /// We could cache results from full context to predicted alternative easily and
+    /// that saves a lot of time but doesn't work in presence of predicates. The set
+    /// of visible predicates from the ATN start state changes depending on the
+    /// context, because closure can fall off the end of a rule. I tried to cache
+    /// tuples (stack context, semantic context, predicted alt) but it was slower
+    /// than interpreting and much more complicated. Also required a huge amount of
+    /// memory. The goal is not to create the world's fastest parser anyway. I'd like
+    /// to keep this algorithm simple. By launching multiple threads, we can improve
+    /// the speed of parsing across a large number of files.</p>
+    /// <p>
+    /// There is no strict ordering between the amount of input used by SLL vs LL,
+    /// which makes it really hard to build a cache for full context. Let's say that
+    /// we have input A B C that leads to an SLL conflict with full context X. That
+    /// implies that using X we might only use A B but we could also use A B C D to
+    /// resolve conflict. Input A B C D could predict alternative 1 in one position
+    /// in the input and A B C E could predict alternative 2 in another position in
+    /// input. The conflicting SLL configurations could still be non-unique in the
+    /// full context prediction, which would lead us to requiring more input than the
+    /// original A B C.	To make a	prediction cache work, we have to track	the exact
+    /// input	used during the previous prediction. That amounts to a cache that maps
+    /// X to a specific DFA for that context.</p>
+    /// <p>
+    /// Something should be done for left-recursive expression predictions. They are
+    /// likely LL(1) + pred eval. Easier to do the whole SLL unless error and retry
+    /// with full LL thing Sam does.</p>
+    /// <p>
+    /// <strong>AVOIDING FULL CONTEXT PREDICTION</strong></p>
+    /// <p>
+    /// We avoid doing full context retry when the outer context is empty, we did not
+    /// dip into the outer context by falling off the end of the decision state rule,
+    /// or when we force SLL mode.</p>
+    /// <p>
+    /// As an example of the not dip into outer context case, consider as super
+    /// constructor calls versus function calls. One grammar might look like
+    /// this:</p>
+    /// <pre>
+    /// ctorBody
+    /// : '{' superCall? stat* '}'
+    /// ;
+    /// </pre>
+    /// <p>
+    /// Or, you might see something like</p>
+    /// <pre>
+    /// stat
+    /// : superCall ';'
+    /// | expression ';'
+    /// | ...
+    /// ;
+    /// </pre>
+    /// <p>
+    /// In both cases I believe that no closure operations will dip into the outer
+    /// context. In the first case ctorBody in the worst case will stop at the '}'.
+    /// In the 2nd case it should stop at the ';'. Both cases should stay within the
+    /// entry rule and not dip into the outer context.</p>
+    /// <p>
+    /// <strong>PREDICATES</strong></p>
+    /// <p>
+    /// Predicates are always evaluated if present in either SLL or LL both. SLL and
+    /// LL simulation deals with predicates differently. SLL collects predicates as
+    /// it performs closure operations like ANTLR v3 did. It delays predicate
+    /// evaluation until it reaches and accept state. This allows us to cache the SLL
+    /// ATN simulation whereas, if we had evaluated predicates on-the-fly during
+    /// closure, the DFA state configuration sets would be different and we couldn't
+    /// build up a suitable DFA.</p>
+    /// <p>
+    /// When building a DFA accept state during ATN simulation, we evaluate any
+    /// predicates and return the sole semantically valid alternative. If there is
+    /// more than 1 alternative, we report an ambiguity. If there are 0 alternatives,
+    /// we throw an exception. Alternatives without predicates act like they have
+    /// true predicates. The simple way to think about it is to strip away all
+    /// alternatives with false predicates and choose the minimum alternative that
+    /// remains.</p>
+    /// <p>
+    /// When we start in the DFA and reach an accept state that's predicated, we test
+    /// those and return the minimum semantically viable alternative. If no
+    /// alternatives are viable, we throw an exception.</p>
+    /// <p>
+    /// During full LL ATN simulation, closure always evaluates predicates and
+    /// on-the-fly. This is crucial to reducing the configuration set size during
+    /// closure. It hits a landmine when parsing with the Java grammar, for example,
+    /// without this on-the-fly evaluation.</p>
+    /// <p>
+    /// <strong>SHARING DFA</strong></p>
+    /// <p>
+    /// All instances of the same parser share the same decision DFAs through a
+    /// static field. Each instance gets its own ATN simulator but they share the
+    /// same
+    /// <see cref="ATN.decisionToDFA">ATN.decisionToDFA</see>
+    /// field. They also share a
+    /// <see cref="PredictionContextCache">PredictionContextCache</see>
+    /// object that makes sure that all
+    /// <see cref="PredictionContext">PredictionContext</see>
+    /// objects are shared among the DFA states. This makes
+    /// a big size difference.</p>
+    /// <p>
+    /// <strong>THREAD SAFETY</strong></p>
+    /// <p>
+    /// The
+    /// <see cref="ParserATNSimulator">ParserATNSimulator</see>
+    /// locks on the
+    /// <see cref="ATN.decisionToDFA">ATN.decisionToDFA</see>
+    /// field when
+    /// it adds a new DFA object to that array.
+    /// <see cref="AddDFAEdge(Antlr4.Runtime.Dfa.DFAState, int, Antlr4.Runtime.Dfa.DFAState)">AddDFAEdge(Antlr4.Runtime.Dfa.DFAState, int, Antlr4.Runtime.Dfa.DFAState)</see>
+    /// locks on the DFA for the current decision when setting the
+    /// <see cref="DFAState#edges">DFAState#edges</see>
+    /// field.
+    /// <see cref="AddDFAState(Antlr4.Runtime.Dfa.DFA, ATNConfigSet, PredictionContextCache)">AddDFAState(Antlr4.Runtime.Dfa.DFA, ATNConfigSet, PredictionContextCache)</see>
+    /// locks on
+    /// the DFA for the current decision when looking up a DFA state to see if it
+    /// already exists. We must make sure that all requests to add DFA states that
+    /// are equivalent result in the same shared DFA object. This is because lots of
+    /// threads will be trying to update the DFA at once. The
+    /// <see cref="AddDFAState(Antlr4.Runtime.Dfa.DFA, ATNConfigSet, PredictionContextCache)">AddDFAState(Antlr4.Runtime.Dfa.DFA, ATNConfigSet, PredictionContextCache)</see>
+    /// method also locks inside the DFA lock
+    /// but this time on the shared context cache when it rebuilds the
+    /// configurations'
+    /// <see cref="PredictionContext">PredictionContext</see>
+    /// objects using cached
+    /// subgraphs/nodes. No other locking occurs, even during DFA simulation. This is
+    /// safe as long as we can guarantee that all threads referencing
+    /// <code>s.edge[t]</code>
+    /// get the same physical target
+    /// <see cref="Antlr4.Runtime.Dfa.DFAState">Antlr4.Runtime.Dfa.DFAState</see>
+    /// , or
+    /// <code>null</code>
+    /// . Once into the DFA, the DFA simulation does not reference the
+    /// <see cref="Antlr4.Runtime.Dfa.DFA.states">Antlr4.Runtime.Dfa.DFA.states</see>
+    /// map. It follows the
+    /// <see cref="DFAState#edges">DFAState#edges</see>
+    /// field to new
+    /// targets. The DFA simulator will either find
+    /// <see cref="DFAState#edges">DFAState#edges</see>
+    /// to be
+    /// <code>null</code>
+    /// , to be non-
+    /// <code>null</code>
+    /// and
+    /// <code>dfa.edges[t]</code>
+    /// null, or
+    /// <code>dfa.edges[t]</code>
+    /// to be non-null. The
+    /// <see cref="AddDFAEdge(Antlr4.Runtime.Dfa.DFAState, int, Antlr4.Runtime.Dfa.DFAState)">AddDFAEdge(Antlr4.Runtime.Dfa.DFAState, int, Antlr4.Runtime.Dfa.DFAState)</see>
+    /// method could be racing to set the field
+    /// but in either case the DFA simulator works; if
+    /// <code>null</code>
+    /// , and requests ATN
+    /// simulation. It could also race trying to get
+    /// <code>dfa.edges[t]</code>
+    /// , but either
+    /// way it will work because it's not doing a test and set operation.</p>
+    /// <p>
+    /// <strong>Starting with SLL then failing to combined SLL/LL (Two-Stage
+    /// Parsing)</strong></p>
+    /// <p>
+    /// Sam pointed out that if SLL does not give a syntax error, then there is no
+    /// point in doing full LL, which is slower. We only have to try LL if we get a
+    /// syntax error. For maximum speed, Sam starts the parser set to pure SLL
+    /// mode with the
+    /// <see cref="Antlr4.Runtime.BailErrorStrategy">Antlr4.Runtime.BailErrorStrategy</see>
+    /// :</p>
+    /// <pre>
+    /// parser.
+    /// <see cref="Antlr4.Runtime.Recognizer{Symbol, ATNInterpreter}.Interpreter()">getInterpreter()</see>
+    /// .
+    /// <see cref="PredictionMode(PredictionMode)">setPredictionMode</see>
+    /// <code>(</code>
+    /// <see cref="PredictionMode.Sll">PredictionMode.Sll</see>
+    /// <code>)</code>
+    /// ;
+    /// parser.
+    /// <see cref="Antlr4.Runtime.Parser.ErrorHandler(IAntlrErrorStrategy)">setErrorHandler</see>
+    /// (new
+    /// <see cref="Antlr4.Runtime.BailErrorStrategy">Antlr4.Runtime.BailErrorStrategy</see>
+    /// ());
+    /// </pre>
+    /// <p>
+    /// If it does not get a syntax error, then we're done. If it does get a syntax
+    /// error, we need to retry with the combined SLL/LL strategy.</p>
+    /// <p>
+    /// The reason this works is as follows. If there are no SLL conflicts, then the
+    /// grammar is SLL (at least for that input set). If there is an SLL conflict,
+    /// the full LL analysis must yield a set of viable alternatives which is a
+    /// subset of the alternatives reported by SLL. If the LL set is a singleton,
+    /// then the grammar is LL but not SLL. If the LL set is the same size as the SLL
+    /// set, the decision is SLL. If the LL set has size &gt; 1, then that decision
+    /// is truly ambiguous on the current input. If the LL set is smaller, then the
+    /// SLL conflict resolution might choose an alternative that the full LL would
+    /// rule out as a possibility based upon better context information. If that's
+    /// the case, then the SLL parse will definitely get an error because the full LL
+    /// analysis says it's not viable. If SLL conflict resolution chooses an
+    /// alternative within the LL set, them both SLL and LL would choose the same
+    /// alternative because they both choose the minimum of multiple conflicting
+    /// alternatives.</p>
+    /// <p>
+    /// Let's say we have a set of SLL conflicting alternatives
+    /// <code></code>
+    /// 
+    /// 1, 2, 3}} and
+    /// a smaller LL set called <em>s</em>. If <em>s</em> is
+    /// <code></code>
+    /// 
+    /// 2, 3}}, then SLL
+    /// parsing will get an error because SLL will pursue alternative 1. If
+    /// <em>s</em> is
+    /// <code></code>
+    /// 
+    /// 1, 2}} or
+    /// <code></code>
+    /// 
+    /// 1, 3}} then both SLL and LL will
+    /// choose the same alternative because alternative one is the minimum of either
+    /// set. If <em>s</em> is
+    /// <code></code>
+    /// 
+    /// 2}} or
+    /// <code></code>
+    /// 
+    /// 3}} then SLL will get a syntax
+    /// error. If <em>s</em> is
+    /// <code></code>
+    /// 
+    /// 1}} then SLL will succeed.</p>
+    /// <p>
+    /// Of course, if the input is invalid, then we will get an error for sure in
+    /// both SLL and LL parsing. Erroneous input will therefore require 2 passes over
+    /// the input.</p>
     /// </remarks>
     public class ParserATNSimulator : ATNSimulator
     {
@@ -212,6 +318,25 @@ namespace Antlr4.Runtime.Atn
 
         public bool always_try_local_context = true;
 
+        /// <summary>Determines whether the DFA is used for full-context predictions.</summary>
+        /// <remarks>
+        /// Determines whether the DFA is used for full-context predictions. When
+        /// <code>true</code>
+        /// , the DFA stores transition information for both full-context
+        /// and SLL parsing; otherwise, the DFA only stores SLL transition
+        /// information.
+        /// <p>
+        /// For some grammars, enabling the full-context DFA can result in a
+        /// substantial performance improvement. However, this improvement typically
+        /// comes at the expense of memory used for storing the cached DFA states,
+        /// configuration sets, and prediction contexts.</p>
+        /// <p>
+        /// The default value is
+        /// <code>false</code>
+        /// .</p>
+        /// </remarks>
+        public bool enable_global_context_dfa = false;
+
         public bool optimize_unique_closure = true;
 
         public bool optimize_ll1 = true;
@@ -299,7 +424,7 @@ namespace Antlr4.Runtime.Atn
         {
             DFA dfa = atn.decisionToDFA[decision];
             System.Diagnostics.Debug.Assert(dfa != null);
-            if (optimize_ll1 && !dfa.IsEmpty())
+            if (optimize_ll1 && !dfa.IsPrecedenceDfa() && !dfa.IsEmpty())
             {
                 int ll_1 = input.La(1);
                 if (ll_1 >= 0 && ll_1 <= short.MaxValue)
@@ -359,15 +484,41 @@ namespace Antlr4.Runtime.Atn
         {
             if (!useContext)
             {
-                if (dfa.s0.Get() == null)
+                if (dfa.IsPrecedenceDfa())
                 {
-                    return null;
+                    // the start state for a precedence DFA depends on the current
+                    // parser precedence, and is provided by a DFA method.
+                    DFAState state = dfa.GetPrecedenceStartState(parser.GetPrecedence(), false);
+                    if (state == null)
+                    {
+                        return null;
+                    }
+                    return new SimulatorState(outerContext, state, false, outerContext);
                 }
-                return new SimulatorState(outerContext, dfa.s0.Get(), false, outerContext);
+                else
+                {
+                    if (dfa.s0.Get() == null)
+                    {
+                        return null;
+                    }
+                    return new SimulatorState(outerContext, dfa.s0.Get(), false, outerContext);
+                }
+            }
+            if (!enable_global_context_dfa)
+            {
+                return null;
             }
             ParserRuleContext remainingContext = outerContext;
             System.Diagnostics.Debug.Assert(outerContext != null);
-            DFAState s0 = dfa.s0full.Get();
+            DFAState s0;
+            if (dfa.IsPrecedenceDfa())
+            {
+                s0 = dfa.GetPrecedenceStartState(parser.GetPrecedence(), true);
+            }
+            else
+            {
+                s0 = dfa.s0full.Get();
+            }
             while (remainingContext != null && s0 != null && s0.IsContextSensitive)
             {
                 remainingContext = SkipTailCalls(remainingContext);
@@ -634,7 +785,7 @@ namespace Antlr4.Runtime.Atn
                     int predictedAlt = conflictingAlts == null ? GetUniqueAlt(D.configs) : ATN.InvalidAltNumber;
                     if (predictedAlt != ATN.InvalidAltNumber)
                     {
-                        if (optimize_ll1 && input.Index == startIndex && nextState.outerContext == nextState.remainingOuterContext && dfa.decision >= 0 && !D.configs.HasSemanticContext)
+                        if (optimize_ll1 && input.Index == startIndex && !dfa.IsPrecedenceDfa() && nextState.outerContext == nextState.remainingOuterContext && dfa.decision >= 0 && !D.configs.HasSemanticContext)
                         {
                             if (t >= 0 && t <= short.MaxValue)
                             {
@@ -884,7 +1035,7 @@ namespace Antlr4.Runtime.Atn
                         ATNState target = GetReachableTarget(c, trans, t);
                         if (target != null)
                         {
-                            reachIntermediate.Add(c.Transform(target), contextCache);
+                            reachIntermediate.Add(c.Transform(target, false), contextCache);
                         }
                     }
                 }
@@ -998,7 +1149,7 @@ namespace Antlr4.Runtime.Atn
         [NotNull]
         protected internal virtual SimulatorState ComputeStartState(DFA dfa, ParserRuleContext globalContext, bool useContext)
         {
-            DFAState s0 = useContext ? dfa.s0full.Get() : dfa.s0.Get();
+            DFAState s0 = dfa.IsPrecedenceDfa() ? dfa.GetPrecedenceStartState(parser.GetPrecedence(), useContext) : useContext ? dfa.s0full.Get() : dfa.s0.Get();
             if (s0 != null)
             {
                 if (!useContext)
@@ -1016,6 +1167,23 @@ namespace Antlr4.Runtime.Atn
             PredictionContextCache contextCache = new PredictionContextCache();
             if (useContext)
             {
+                if (!enable_global_context_dfa)
+                {
+                    while (remainingGlobalContext != null)
+                    {
+                        if (remainingGlobalContext.IsEmpty())
+                        {
+                            previousContext = PredictionContext.EmptyFullStateKey;
+                            remainingGlobalContext = null;
+                        }
+                        else
+                        {
+                            previousContext = GetReturnState(remainingGlobalContext);
+                            initialContext = initialContext.AppendContext(previousContext, contextCache);
+                            remainingGlobalContext = ((ParserRuleContext)remainingGlobalContext.Parent);
+                        }
+                    }
+                }
                 while (s0 != null && s0.IsContextSensitive && remainingGlobalContext != null)
                 {
                     DFAState next;
@@ -1063,18 +1231,48 @@ namespace Antlr4.Runtime.Atn
                 bool collectPredicates = true;
                 Closure(reachIntermediate, configs, collectPredicates, hasMoreContext, contextCache);
                 bool stepIntoGlobal = configs.DipsIntoOuterContext;
-                DFAState next = AddDFAState(dfa, configs, contextCache);
-                if (s0 == null)
+                DFAState next;
+                if (useContext && !enable_global_context_dfa)
                 {
-                    AtomicReference<DFAState> reference = useContext ? dfa.s0full : dfa.s0;
-                    if (!reference.CompareAndSet(null, next))
-                    {
-                        next = reference.Get();
-                    }
+                    s0 = AddDFAState(dfa, configs, contextCache);
+                    break;
                 }
                 else
                 {
-                    s0.SetContextTarget(previousContext, next);
+                    if (s0 == null)
+                    {
+                        if (!dfa.IsPrecedenceDfa() && dfa.atnStartState is StarLoopEntryState)
+                        {
+                            if (((StarLoopEntryState)dfa.atnStartState).precedenceRuleDecision)
+                            {
+                                dfa.SetPrecedenceDfa(true);
+                            }
+                        }
+                        if (!dfa.IsPrecedenceDfa())
+                        {
+                            AtomicReference<DFAState> reference = useContext ? dfa.s0full : dfa.s0;
+                            next = AddDFAState(dfa, configs, contextCache);
+                            if (!reference.CompareAndSet(null, next))
+                            {
+                                next = reference.Get();
+                            }
+                        }
+                        else
+                        {
+                            configs = ApplyPrecedenceFilter(configs, globalContext, contextCache);
+                            next = AddDFAState(dfa, configs, contextCache);
+                            dfa.SetPrecedenceStartState(parser.GetPrecedence(), useContext, next);
+                        }
+                    }
+                    else
+                    {
+                        if (dfa.IsPrecedenceDfa())
+                        {
+                            configs = ApplyPrecedenceFilter(configs, globalContext, contextCache);
+                        }
+                        next = AddDFAState(dfa, configs, contextCache);
+                        s0.SetContextTarget(previousContext, next);
+                    }
                 }
                 s0 = next;
                 if (!useContext || !stepIntoGlobal)
@@ -1103,6 +1301,91 @@ namespace Antlr4.Runtime.Atn
             return new SimulatorState(globalContext, s0, useContext, remainingGlobalContext);
         }
 
+        /// <summary>
+        /// This method transforms the start state computed by
+        /// <see cref="ComputeStartState(Antlr4.Runtime.Dfa.DFA, Antlr4.Runtime.ParserRuleContext, bool)">ComputeStartState(Antlr4.Runtime.Dfa.DFA, Antlr4.Runtime.ParserRuleContext, bool)</see>
+        /// to the special start state used by a
+        /// precedence DFA for a particular precedence value. The transformation
+        /// process applies the following changes to the start state's configuration
+        /// set.
+        /// <ol>
+        /// <li>Evaluate the precedence predicates for each configuration using
+        /// <see cref="SemanticContext.EvalPrecedence(Antlr4.Runtime.Recognizer{Symbol, ATNInterpreter}, Antlr4.Runtime.RuleContext)">SemanticContext.EvalPrecedence(Antlr4.Runtime.Recognizer&lt;Symbol, ATNInterpreter&gt;, Antlr4.Runtime.RuleContext)</see>
+        /// .</li>
+        /// <li>Remove all configurations which predict an alternative greater than
+        /// 1, for which another configuration that predicts alternative 1 is in the
+        /// same ATN state. This transformation is valid for the following reasons:
+        /// <ul>
+        /// <li>The closure block cannot contain any epsilon transitions which bypass
+        /// the body of the closure, so all states reachable via alternative 1 are
+        /// part of the precedence alternatives of the transformed left-recursive
+        /// rule.</li>
+        /// <li>The "primary" portion of a left recursive rule cannot contain an
+        /// epsilon transition, so the only way an alternative other than 1 can exist
+        /// in a state that is also reachable via alternative 1 is by nesting calls
+        /// to the left-recursive rule, with the outer calls not being at the
+        /// preferred precedence level.</li>
+        /// </ul>
+        /// </li>
+        /// </ol>
+        /// </summary>
+        /// <param name="configs">
+        /// The configuration set computed by
+        /// <see cref="ComputeStartState(Antlr4.Runtime.Dfa.DFA, Antlr4.Runtime.ParserRuleContext, bool)">ComputeStartState(Antlr4.Runtime.Dfa.DFA, Antlr4.Runtime.ParserRuleContext, bool)</see>
+        /// as the start state for the DFA.
+        /// </param>
+        /// <returns>
+        /// The transformed configuration set representing the start state
+        /// for a precedence DFA at a particular precedence level (determined by
+        /// calling
+        /// <see cref="Antlr4.Runtime.Parser.GetPrecedence()">Antlr4.Runtime.Parser.GetPrecedence()</see>
+        /// ).
+        /// </returns>
+        [NotNull]
+        protected internal virtual ATNConfigSet ApplyPrecedenceFilter(ATNConfigSet configs, ParserRuleContext globalContext, PredictionContextCache contextCache)
+        {
+            HashSet<int> statesFromAlt1 = new HashSet<int>();
+            ATNConfigSet configSet = new ATNConfigSet();
+            foreach (ATNConfig config in configs)
+            {
+                // handle alt 1 first
+                if (config.Alt != 1)
+                {
+                    continue;
+                }
+                SemanticContext updatedContext = config.SemanticContext.EvalPrecedence(parser, globalContext);
+                if (updatedContext == null)
+                {
+                    // the configuration was eliminated
+                    continue;
+                }
+                statesFromAlt1.AddItem(config.State.stateNumber);
+                if (updatedContext != config.SemanticContext)
+                {
+                    configSet.Add(config.Transform(config.State, updatedContext, false), contextCache);
+                }
+                else
+                {
+                    configSet.Add(config, contextCache);
+                }
+            }
+            foreach (ATNConfig config_1 in configs)
+            {
+                if (config_1.Alt == 1)
+                {
+                    // already handled
+                    continue;
+                }
+                if (statesFromAlt1.Contains(config_1.State.stateNumber))
+                {
+                    // eliminated
+                    continue;
+                }
+                configSet.Add(config_1, contextCache);
+            }
+            return configSet;
+        }
+
         [Nullable]
         protected internal virtual ATNState GetReachableTarget(ATNConfig source, Transition trans, int ttype)
         {
@@ -1196,7 +1479,7 @@ namespace Antlr4.Runtime.Atn
             }
             if (!containsPredicate)
             {
-                pairs = null;
+                return null;
             }
             //		System.out.println(Arrays.toString(altToPred)+"->"+pairs);
             return Sharpen.Collections.ToArray(pairs, new DFAState.PredPrediction[pairs.Count]);
@@ -1293,7 +1576,7 @@ namespace Antlr4.Runtime.Atn
                     {
                         return;
                     }
-                    config = config.Transform(config.State, PredictionContext.EmptyLocal);
+                    config = config.Transform(config.State, PredictionContext.EmptyLocal, false);
                 }
                 else
                 {
@@ -1308,7 +1591,7 @@ namespace Antlr4.Runtime.Atn
                         if (config.Context == PredictionContext.EmptyFull)
                         {
                             // no need to keep full context overhead when we step out
-                            config = config.Transform(config.State, PredictionContext.EmptyLocal);
+                            config = config.Transform(config.State, PredictionContext.EmptyLocal, false);
                         }
                     }
                 }
@@ -1422,7 +1705,7 @@ namespace Antlr4.Runtime.Atn
 
                 case TransitionType.Epsilon:
                 {
-                    return config.Transform(t.target);
+                    return config.Transform(t.target, false);
                 }
 
                 default:
@@ -1435,7 +1718,7 @@ namespace Antlr4.Runtime.Atn
         [NotNull]
         protected internal virtual ATNConfig ActionTransition(ATNConfig config, Antlr4.Runtime.Atn.ActionTransition t)
         {
-            return config.Transform(t.target);
+            return config.Transform(t.target, false);
         }
 
         [Nullable]
@@ -1445,11 +1728,11 @@ namespace Antlr4.Runtime.Atn
             if (collectPredicates && inContext)
             {
                 SemanticContext newSemCtx = SemanticContext.And(config.SemanticContext, pt.GetPredicate());
-                c = config.Transform(pt.target, newSemCtx);
+                c = config.Transform(pt.target, newSemCtx, false);
             }
             else
             {
-                c = config.Transform(pt.target);
+                c = config.Transform(pt.target, false);
             }
             return c;
         }
@@ -1461,11 +1744,11 @@ namespace Antlr4.Runtime.Atn
             if (collectPredicates && (!pt.isCtxDependent || (pt.isCtxDependent && inContext)))
             {
                 SemanticContext newSemCtx = SemanticContext.And(config.SemanticContext, pt.GetPredicate());
-                c = config.Transform(pt.target, newSemCtx);
+                c = config.Transform(pt.target, newSemCtx, false);
             }
             else
             {
-                c = config.Transform(pt.target);
+                c = config.Transform(pt.target, false);
             }
             return c;
         }
@@ -1490,12 +1773,12 @@ namespace Antlr4.Runtime.Atn
                     newContext = config.Context.GetChild(returnState.stateNumber);
                 }
             }
-            return config.Transform(t.target, newContext);
+            return config.Transform(t.target, newContext, false);
         }
 
-        private sealed class _IComparer_1536 : IComparer<ATNConfig>
+        private sealed class _IComparer_1741 : IComparer<ATNConfig>
         {
-            public _IComparer_1536()
+            public _IComparer_1741()
             {
             }
 
@@ -1515,7 +1798,7 @@ namespace Antlr4.Runtime.Atn
             }
         }
 
-        private static readonly IComparer<ATNConfig> StateAltSortComparator = new _IComparer_1536();
+        private static readonly IComparer<ATNConfig> StateAltSortComparator = new _IComparer_1741();
 
         private BitSet IsConflicted(ATNConfigSet configset, PredictionContextCache contextCache)
         {
@@ -1817,7 +2100,7 @@ namespace Antlr4.Runtime.Atn
         [NotNull]
         protected internal virtual DFAState AddDFAEdge(DFA dfa, DFAState fromState, int t, List<int> contextTransitions, ATNConfigSet toConfigs, PredictionContextCache contextCache)
         {
-            System.Diagnostics.Debug.Assert(dfa.IsContextSensitive() || contextTransitions == null || contextTransitions.IsEmpty());
+            System.Diagnostics.Debug.Assert(contextTransitions == null || contextTransitions.IsEmpty() || dfa.IsContextSensitive());
             DFAState from = fromState;
             DFAState to = AddDFAState(dfa, toConfigs, contextCache);
             if (contextTransitions != null)
@@ -1885,15 +2168,19 @@ namespace Antlr4.Runtime.Atn
         [NotNull]
         protected internal virtual DFAState AddDFAState(DFA dfa, ATNConfigSet configs, PredictionContextCache contextCache)
         {
-            if (!configs.IsReadOnly)
+            bool enableDfa = enable_global_context_dfa || !configs.IsOutermostConfigSet;
+            if (enableDfa)
             {
-                configs.OptimizeConfigs(this);
-            }
-            DFAState proposed = CreateDFAState(configs);
-            DFAState existing = dfa.states.Get(proposed);
-            if (existing != null)
-            {
-                return existing;
+                if (!configs.IsReadOnly)
+                {
+                    configs.OptimizeConfigs(this);
+                }
+                DFAState proposed = CreateDFAState(configs);
+                DFAState existing = dfa.states.Get(proposed);
+                if (existing != null)
+                {
+                    return existing;
+                }
             }
             if (!configs.IsReadOnly)
             {
@@ -1904,10 +2191,10 @@ namespace Antlr4.Runtime.Atn
                     {
                         int size = configs.Count;
                         configs.StripHiddenConfigs();
-                        if (configs.Count < size)
+                        if (enableDfa && configs.Count < size)
                         {
-                            proposed = CreateDFAState(configs);
-                            existing = dfa.states.Get(proposed);
+                            DFAState proposed = CreateDFAState(configs);
+                            DFAState existing = dfa.states.Get(proposed);
                             if (existing != null)
                             {
                                 return existing;
@@ -1936,6 +2223,10 @@ namespace Antlr4.Runtime.Atn
             {
                 PredicateDFAState(newState, configs, decisionState.NumberOfTransitions);
             }
+            if (!enableDfa)
+            {
+                return newState;
+            }
             DFAState added = dfa.AddState(newState);
             if (debug && added == newState)
             {

runtime/CSharp/Antlr4.Runtime/Atn/PredictionContext.cs

@@ -59,7 +59,7 @@ namespace Antlr4.Runtime.Atn
         /// <pre>
         /// private int referenceHashCode() {
         /// int hash =
-        /// <see cref="Antlr4.Runtime.Misc.MurmurHash.Initialize()">Antlr4.Runtime.Misc.MurmurHash.Initialize()</see>
+        /// <see cref="Antlr4.Runtime.Misc.MurmurHash.Initialize()">MurmurHash.initialize</see>
         /// (
         /// <see cref="InitialHash">InitialHash</see>
         /// );
@@ -67,22 +67,22 @@ namespace Antlr4.Runtime.Atn
         /// <see cref="Size()">Size()</see>
         /// ; i++) {
         /// hash =
-        /// <see cref="Antlr4.Runtime.Misc.MurmurHash.Update(int, int)">Antlr4.Runtime.Misc.MurmurHash.Update(int, int)</see>
+        /// <see cref="Antlr4.Runtime.Misc.MurmurHash.Update(int, int)">MurmurHash.update</see>
         /// (hash,
-        /// <see cref="GetParent(int)">GetParent(int)</see>
+        /// <see cref="GetParent(int)">getParent</see>
         /// (i));
         /// }
         /// for (int i = 0; i &lt;
         /// <see cref="Size()">Size()</see>
         /// ; i++) {
         /// hash =
-        /// <see cref="Antlr4.Runtime.Misc.MurmurHash.Update(int, int)">Antlr4.Runtime.Misc.MurmurHash.Update(int, int)</see>
+        /// <see cref="Antlr4.Runtime.Misc.MurmurHash.Update(int, int)">MurmurHash.update</see>
         /// (hash,
-        /// <see cref="GetReturnState(int)">GetReturnState(int)</see>
+        /// <see cref="GetReturnState(int)">getReturnState</see>
         /// (i));
         /// }
         /// hash =
-        /// <see cref="Antlr4.Runtime.Misc.MurmurHash.Finish(int, int)">Antlr4.Runtime.Misc.MurmurHash.Finish(int, int)</see>
+        /// <see cref="Antlr4.Runtime.Misc.MurmurHash.Finish(int, int)">MurmurHash.finish</see>
         /// (hash, 2 *
         /// <see cref="Size()">Size()</see>
         /// );

runtime/CSharp/Antlr4.Runtime/Atn/PredictionMode.cs

@@ -34,42 +34,82 @@ using Sharpen;
 
 namespace Antlr4.Runtime.Atn
 {
+    /// <summary>
+    /// This enumeration defines the prediction modes available in ANTLR 4 along with
+    /// utility methods for analyzing configuration sets for conflicts and/or
+    /// ambiguities.
+    /// </summary>
+    /// <remarks>
+    /// This enumeration defines the prediction modes available in ANTLR 4 along with
+    /// utility methods for analyzing configuration sets for conflicts and/or
+    /// ambiguities.
+    /// </remarks>
     [System.Serializable]
     public sealed class PredictionMode
     {
-        /// <summary>
-        /// Do only local context prediction (SLL style) and using
-        /// heuristic which almost always works but is much faster
-        /// than precise answer.
-        /// </summary>
+        /// <summary>The SLL(*) prediction mode.</summary>
         /// <remarks>
-        /// Do only local context prediction (SLL style) and using
-        /// heuristic which almost always works but is much faster
-        /// than precise answer.
+        /// The SLL(*) prediction mode. This prediction mode ignores the current
+        /// parser context when making predictions. This is the fastest prediction
+        /// mode, and provides correct results for many grammars. This prediction
+        /// mode is more powerful than the prediction mode provided by ANTLR 3, but
+        /// may result in syntax errors for grammar and input combinations which are
+        /// not SLL.
+        /// <p>
+        /// When using this prediction mode, the parser will either return a correct
+        /// parse tree (i.e. the same parse tree that would be returned with the
+        /// <see cref="Ll">Ll</see>
+        /// prediction mode), or it will report a syntax error. If a
+        /// syntax error is encountered when using the
+        /// <see cref="Sll">Sll</see>
+        /// prediction mode,
+        /// it may be due to either an actual syntax error in the input or indicate
+        /// that the particular combination of grammar and input requires the more
+        /// powerful
+        /// <see cref="Ll">Ll</see>
+        /// prediction abilities to complete successfully.</p>
+        /// <p>
+        /// This prediction mode does not provide any guarantees for prediction
+        /// behavior for syntactically-incorrect inputs.</p>
         /// </remarks>
         public static readonly PredictionMode Sll = new PredictionMode();
 
-        /// <summary>Full LL(*) that always gets right answer.</summary>
+        /// <summary>The LL(*) prediction mode.</summary>
         /// <remarks>
-        /// Full LL(*) that always gets right answer. For speed
-        /// reasons, we terminate the prediction process when we know for
-        /// sure which alt to predict. We don't always know what
-        /// the ambiguity is in this mode.
+        /// The LL(*) prediction mode. This prediction mode allows the current parser
+        /// context to be used for resolving SLL conflicts that occur during
+        /// prediction. This is the fastest prediction mode that guarantees correct
+        /// parse results for all combinations of grammars with syntactically correct
+        /// inputs.
+        /// <p>
+        /// When using this prediction mode, the parser will make correct decisions
+        /// for all syntactically-correct grammar and input combinations. However, in
+        /// cases where the grammar is truly ambiguous this prediction mode might not
+        /// report a precise answer for <em>exactly which</em> alternatives are
+        /// ambiguous.</p>
+        /// <p>
+        /// This prediction mode does not provide any guarantees for prediction
+        /// behavior for syntactically-incorrect inputs.</p>
         /// </remarks>
         public static readonly PredictionMode Ll = new PredictionMode();
 
-        /// <summary>
-        /// Tell the full LL prediction algorithm to pursue lookahead until
-        /// it has uniquely predicted an alternative without conflict or it's
-        /// certain that it's found an ambiguous input sequence.
-        /// </summary>
+        /// <summary>The LL(*) prediction mode with exact ambiguity detection.</summary>
         /// <remarks>
-        /// Tell the full LL prediction algorithm to pursue lookahead until
-        /// it has uniquely predicted an alternative without conflict or it's
-        /// certain that it's found an ambiguous input sequence.  when this
-        /// variable is false. When true, the prediction process will
-        /// continue looking for the exact ambiguous sequence even if
-        /// it has already figured out which alternative to predict.
+        /// The LL(*) prediction mode with exact ambiguity detection. In addition to
+        /// the correctness guarantees provided by the
+        /// <see cref="Ll">Ll</see>
+        /// prediction mode,
+        /// this prediction mode instructs the prediction algorithm to determine the
+        /// complete and exact set of ambiguous alternatives for every ambiguous
+        /// decision encountered while parsing.
+        /// <p>
+        /// This prediction mode may be used for diagnosing ambiguities during
+        /// grammar development. Due to the performance overhead of calculating sets
+        /// of ambiguous alternatives, this prediction mode should be avoided when
+        /// the exact results are not necessary.</p>
+        /// <p>
+        /// This prediction mode does not provide any guarantees for prediction
+        /// behavior for syntactically-incorrect inputs.</p>
         /// </remarks>
         public static readonly PredictionMode LlExactAmbigDetection = new PredictionMode();
 
@@ -91,7 +131,13 @@ namespace Antlr4.Runtime.Atn
             {
             }
 
-            /// <summary>Code is function of (s, _, ctx, _)</summary>
+            /// <summary>
+            /// The hash code is only a function of the
+            /// <see cref="ATNState.stateNumber">ATNState.stateNumber</see>
+            /// and
+            /// <see cref="ATNConfig#context">ATNConfig#context</see>
+            /// .
+            /// </summary>
             public override int GetHashCode(ATNConfig o)
             {
                 int hashCode = MurmurHash.Initialize(7);
@@ -118,23 +164,20 @@ namespace Antlr4.Runtime.Atn
         /// <summary>Computes the SLL prediction termination condition.</summary>
         /// <remarks>
         /// Computes the SLL prediction termination condition.
-        /// <p/>
+        /// <p>
         /// This method computes the SLL prediction termination condition for both of
-        /// the following cases.
+        /// the following cases.</p>
         /// <ul>
         /// <li>The usual SLL+LL fallback upon SLL conflict</li>
         /// <li>Pure SLL without LL fallback</li>
         /// </ul>
-        /// <p/>
-        /// <strong>COMBINED SLL+LL PARSING</strong>
-        /// <p/>
-        /// When LL-fallback is enabled upon SLL conflict, correct predictions are
+        /// <p><strong>COMBINED SLL+LL PARSING</strong></p>
+        /// <p>When LL-fallback is enabled upon SLL conflict, correct predictions are
         /// ensured regardless of how the termination condition is computed by this
         /// method. Due to the substantially higher cost of LL prediction, the
         /// prediction should only fall back to LL when the additional lookahead
-        /// cannot lead to a unique SLL prediction.
-        /// <p/>
-        /// Assuming combined SLL+LL parsing, an SLL configuration set with only
+        /// cannot lead to a unique SLL prediction.</p>
+        /// <p>Assuming combined SLL+LL parsing, an SLL configuration set with only
         /// conflicting subsets should fall back to full LL, even if the
         /// configuration sets don't resolve to the same alternative (e.g.
         /// <code></code>
@@ -144,22 +187,19 @@ namespace Antlr4.Runtime.Atn
         /// 
         /// 3,4}}. If there is at least one non-conflicting
         /// configuration, SLL could continue with the hopes that more lookahead will
-        /// resolve via one of those non-conflicting configurations.
-        /// <p/>
-        /// Here's the prediction termination rule them: SLL (for SLL+LL parsing)
+        /// resolve via one of those non-conflicting configurations.</p>
+        /// <p>Here's the prediction termination rule them: SLL (for SLL+LL parsing)
         /// stops when it sees only conflicting configuration subsets. In contrast,
-        /// full LL keeps going when there is uncertainty.
-        /// <p/>
-        /// <strong>HEURISTIC</strong>
-        /// <p/>
-        /// As a heuristic, we stop prediction when we see any conflicting subset
+        /// full LL keeps going when there is uncertainty.</p>
+        /// <p><strong>HEURISTIC</strong></p>
+        /// <p>As a heuristic, we stop prediction when we see any conflicting subset
         /// unless we see a state that only has one alternative associated with it.
         /// The single-alt-state thing lets prediction continue upon rules like
-        /// (otherwise, it would admit defeat too soon):
-        /// <p/>
+        /// (otherwise, it would admit defeat too soon):</p>
+        /// <p>
         /// <code>[12|1|[], 6|2|[], 12|2|[]]. s : (ID | ID ID?) ';' ;</code>
-        /// <p/>
-        /// When the ATN simulation reaches the state before
+        /// </p>
+        /// <p>When the ATN simulation reaches the state before
         /// <code>';'</code>
         /// , it has a
         /// DFA state that looks like:
@@ -172,47 +212,40 @@ namespace Antlr4.Runtime.Atn
         /// processing this node because alternative to has another way to continue,
         /// via
         /// <code>[6|2|[]]</code>
-        /// .
-        /// <p/>
-        /// It also let's us continue for this rule:
-        /// <p/>
+        /// .</p>
+        /// <p>It also let's us continue for this rule:</p>
+        /// <p>
         /// <code>[1|1|[], 1|2|[], 8|3|[]] a : A | A | A B ;</code>
-        /// <p/>
-        /// After matching input A, we reach the stop state for rule A, state 1.
+        /// </p>
+        /// <p>After matching input A, we reach the stop state for rule A, state 1.
         /// State 8 is the state right before B. Clearly alternatives 1 and 2
         /// conflict and no amount of further lookahead will separate the two.
         /// However, alternative 3 will be able to continue and so we do not stop
         /// working on this state. In the previous example, we're concerned with
         /// states associated with the conflicting alternatives. Here alt 3 is not
         /// associated with the conflicting configs, but since we can continue
-        /// looking for input reasonably, don't declare the state done.
-        /// <p/>
-        /// <strong>PURE SLL PARSING</strong>
-        /// <p/>
-        /// To handle pure SLL parsing, all we have to do is make sure that we
+        /// looking for input reasonably, don't declare the state done.</p>
+        /// <p><strong>PURE SLL PARSING</strong></p>
+        /// <p>To handle pure SLL parsing, all we have to do is make sure that we
         /// combine stack contexts for configurations that differ only by semantic
-        /// predicate. From there, we can do the usual SLL termination heuristic.
-        /// <p/>
-        /// <strong>PREDICATES IN SLL+LL PARSING</strong>
-        /// <p/>
-        /// SLL decisions don't evaluate predicates until after they reach DFA stop
+        /// predicate. From there, we can do the usual SLL termination heuristic.</p>
+        /// <p><strong>PREDICATES IN SLL+LL PARSING</strong></p>
+        /// <p>SLL decisions don't evaluate predicates until after they reach DFA stop
         /// states because they need to create the DFA cache that works in all
         /// semantic situations. In contrast, full LL evaluates predicates collected
         /// during start state computation so it can ignore predicates thereafter.
         /// This means that SLL termination detection can totally ignore semantic
-        /// predicates.
-        /// <p/>
-        /// Implementation-wise,
+        /// predicates.</p>
+        /// <p>Implementation-wise,
         /// <see cref="ATNConfigSet">ATNConfigSet</see>
         /// combines stack contexts but not
         /// semantic predicate contexts so we might see two configurations like the
-        /// following.
-        /// <p/>
+        /// following.</p>
+        /// <p>
         /// <code></code>
         /// (s, 1, x,
-        /// ), (s, 1, x', {p})}
-        /// <p/>
-        /// Before testing these configurations against others, we have to merge
+        /// ), (s, 1, x', {p})}</p>
+        /// <p>Before testing these configurations against others, we have to merge
         /// <code>x</code>
         /// and
         /// <code>x'</code>
@@ -220,18 +253,17 @@ namespace Antlr4.Runtime.Atn
         /// For example, we test
         /// <code>(x+x')==x''</code>
         /// when looking for conflicts in
-        /// the following configurations.
-        /// <p/>
+        /// the following configurations.</p>
+        /// <p>
         /// <code></code>
         /// (s, 1, x,
-        /// ), (s, 1, x', {p}), (s, 2, x'', {})}
-        /// <p/>
-        /// If the configuration set has predicates (as indicated by
+        /// ), (s, 1, x', {p}), (s, 2, x'', {})}</p>
+        /// <p>If the configuration set has predicates (as indicated by
         /// <see cref="ATNConfigSet.HasSemanticContext()">ATNConfigSet.HasSemanticContext()</see>
         /// ), this algorithm makes a copy of
         /// the configurations to strip out all of the predicates so that a standard
         /// <see cref="ATNConfigSet">ATNConfigSet</see>
-        /// will merge everything ignoring predicates.
+        /// will merge everything ignoring predicates.</p>
         /// </remarks>
         public static bool HasSLLConflictTerminatingPrediction(PredictionMode mode, ATNConfigSet configs)
         {
@@ -251,7 +283,7 @@ namespace Antlr4.Runtime.Atn
                     ATNConfigSet dup = new ATNConfigSet();
                     foreach (ATNConfig c in configs)
                     {
-                        c = c.Transform(c.State, SemanticContext.None);
+                        c = c.Transform(c.State, SemanticContext.None, false);
                         dup.AddItem(c);
                     }
                     configs = dup;
@@ -331,17 +363,15 @@ namespace Antlr4.Runtime.Atn
         /// <summary>Full LL prediction termination.</summary>
         /// <remarks>
         /// Full LL prediction termination.
-        /// <p/>
-        /// Can we stop looking ahead during ATN simulation or is there some
+        /// <p>Can we stop looking ahead during ATN simulation or is there some
         /// uncertainty as to which alternative we will ultimately pick, after
         /// consuming more input? Even if there are partial conflicts, we might know
         /// that everything is going to resolve to the same minimum alternative. That
         /// means we can stop since no more lookahead will change that fact. On the
         /// other hand, there might be multiple conflicts that resolve to different
         /// minimums. That means we need more look ahead to decide which of those
-        /// alternatives we should predict.
-        /// <p/>
-        /// The basic idea is to split the set of configurations
+        /// alternatives we should predict.</p>
+        /// <p>The basic idea is to split the set of configurations
         /// <code>C</code>
         /// , into
         /// conflicting subsets
@@ -387,43 +417,36 @@ namespace Antlr4.Runtime.Atn
         /// # map hash/equals uses s and x, not
         /// alt and not pred
         /// </pre>
-        /// <p/>
-        /// The values in
+        /// <p>The values in
         /// <code>map</code>
         /// are the set of
         /// <code>A_s,ctx</code>
-        /// sets.
-        /// <p/>
-        /// If
+        /// sets.</p>
+        /// <p>If
         /// <code>|A_s,ctx|=1</code>
         /// then there is no conflict associated with
         /// <code>s</code>
         /// and
         /// <code>ctx</code>
-        /// .
-        /// <p/>
-        /// Reduce the subsets to singletons by choosing a minimum of each subset. If
+        /// .</p>
+        /// <p>Reduce the subsets to singletons by choosing a minimum of each subset. If
         /// the union of these alternative subsets is a singleton, then no amount of
         /// more lookahead will help us. We will always pick that alternative. If,
         /// however, there is more than one alternative, then we are uncertain which
         /// alternative to predict and must continue looking for resolution. We may
         /// or may not discover an ambiguity in the future, even if there are no
-        /// conflicting subsets this round.
-        /// <p/>
-        /// The biggest sin is to terminate early because it means we've made a
+        /// conflicting subsets this round.</p>
+        /// <p>The biggest sin is to terminate early because it means we've made a
         /// decision but were uncertain as to the eventual outcome. We haven't used
         /// enough lookahead. On the other hand, announcing a conflict too late is no
         /// big deal; you will still have the conflict. It's just inefficient. It
-        /// might even look until the end of file.
-        /// <p/>
-        /// No special consideration for semantic predicates is required because
+        /// might even look until the end of file.</p>
+        /// <p>No special consideration for semantic predicates is required because
         /// predicates are evaluated on-the-fly for full LL prediction, ensuring that
         /// no configuration contains a semantic context during the termination
-        /// check.
-        /// <p/>
-        /// <strong>CONFLICTING CONFIGS</strong>
-        /// <p/>
-        /// Two configurations
+        /// check.</p>
+        /// <p><strong>CONFLICTING CONFIGS</strong></p>
+        /// <p>Two configurations
         /// <code>(s, i, x)</code>
         /// and
         /// <code>(s, j, x')</code>
@@ -470,33 +493,28 @@ namespace Antlr4.Runtime.Atn
         /// is not in conflict with alternative
         /// <code>i</code>
         /// . The algorithm
-        /// should keep going, looking for more lookahead due to the uncertainty.
-        /// <p/>
-        /// For simplicity, I'm doing a equality check between
+        /// should keep going, looking for more lookahead due to the uncertainty.</p>
+        /// <p>For simplicity, I'm doing a equality check between
         /// <code>x</code>
         /// and
         /// <code>x'</code>
         /// that lets the algorithm continue to consume lookahead longer
         /// than necessary. The reason I like the equality is of course the
         /// simplicity but also because that is the test you need to detect the
-        /// alternatives that are actually in conflict.
-        /// <p/>
-        /// <strong>CONTINUE/STOP RULE</strong>
-        /// <p/>
-        /// Continue if union of resolved alternative sets from non-conflicting and
+        /// alternatives that are actually in conflict.</p>
+        /// <p><strong>CONTINUE/STOP RULE</strong></p>
+        /// <p>Continue if union of resolved alternative sets from non-conflicting and
         /// conflicting alternative subsets has more than one alternative. We are
-        /// uncertain about which alternative to predict.
-        /// <p/>
-        /// The complete set of alternatives,
+        /// uncertain about which alternative to predict.</p>
+        /// <p>The complete set of alternatives,
         /// <code>[i for (_,i,_)]</code>
         /// , tells us which
         /// alternatives are still in the running for the amount of input we've
         /// consumed at this point. The conflicting sets let us to strip away
         /// configurations that won't lead to more states because we resolve
         /// conflicts to the configuration with a minimum alternate for the
-        /// conflicting set.
-        /// <p/>
-        /// <strong>CASES</strong>
+        /// conflicting set.</p>
+        /// <p><strong>CASES</strong></p>
         /// <ul>
         /// <li>no conflicts and more than 1 alternative in set =&gt; continue</li>
         /// <li>
@@ -605,15 +623,12 @@ namespace Antlr4.Runtime.Atn
         /// 
         /// 1,3}} =&gt; continue</li>
         /// </ul>
-        /// <strong>EXACT AMBIGUITY DETECTION</strong>
-        /// <p/>
-        /// If all states report the same conflicting set of alternatives, then we
-        /// know we have the exact ambiguity set.
-        /// <p/>
-        /// <code>|A_<em>i</em>|&gt;1</code> and
-        /// <code>A_<em>i</em> = A_<em>j</em></code> for all <em>i</em>, <em>j</em>.
-        /// <p/>
-        /// In other words, we continue examining lookahead until all
+        /// <p><strong>EXACT AMBIGUITY DETECTION</strong></p>
+        /// <p>If all states report the same conflicting set of alternatives, then we
+        /// know we have the exact ambiguity set.</p>
+        /// <p><code>|A_<em>i</em>|&gt;1</code> and
+        /// <code>A_<em>i</em> = A_<em>j</em></code> for all <em>i</em>, <em>j</em>.</p>
+        /// <p>In other words, we continue examining lookahead until all
         /// <code>A_i</code>
         /// have more than one alternative and all
         /// <code>A_i</code>
@@ -633,7 +648,7 @@ namespace Antlr4.Runtime.Atn
         /// {1,2}}} or
         /// <code></code>
         /// 
-        /// {1,2},{1,2}}}, etc...
+        /// {1,2},{1,2}}}, etc...</p>
         /// </remarks>
         public static int ResolvesToJustOneViableAlt(IEnumerable<BitSet> altsets)
         {

runtime/CSharp/Antlr4.Runtime/Atn/RuleTransition.cs

@@ -27,6 +27,7 @@
  *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
+using System;
 using Antlr4.Runtime.Atn;
 using Antlr4.Runtime.Misc;
 using Sharpen;
@@ -48,6 +49,13 @@ namespace Antlr4.Runtime.Atn
 
         public bool optimizedTailCall;
 
+        [Obsolete]
+        [System.ObsoleteAttribute(@"UseRuleTransition(RuleStartState, int, int, ATNState) instead.")]
+        public RuleTransition(RuleStartState ruleStart, int ruleIndex, ATNState followState)
+            : this(ruleStart, ruleIndex, 0, followState)
+        {
+        }
+
         public RuleTransition(RuleStartState ruleStart, int ruleIndex, int precedence, ATNState followState)
             : base(ruleStart)
         {

runtime/CSharp/Antlr4.Runtime/Atn/SemanticContext.cs

@@ -48,8 +48,7 @@ namespace Antlr4.Runtime.Atn
     /// , or a sum of products
     /// <code>p1||p2</code>
     /// .
-    /// <p/>
-    /// I have scoped the
+    /// <p>I have scoped the
     /// <see cref="AND">AND</see>
     /// ,
     /// <see cref="OR">OR</see>
@@ -57,14 +56,21 @@ namespace Antlr4.Runtime.Atn
     /// <see cref="Predicate">Predicate</see>
     /// subclasses of
     /// <see cref="SemanticContext">SemanticContext</see>
-    /// within the scope of this outer class.
+    /// within the scope of this outer class.</p>
     /// </remarks>
     public abstract class SemanticContext
     {
+        /// <summary>
+        /// The default
+        /// <see cref="SemanticContext">SemanticContext</see>
+        /// , which is semantically equivalent to
+        /// a predicate of the form
+        /// <code></code>
+        /// 
+        /// true}?}.
+        /// </summary>
         public static readonly SemanticContext None = new SemanticContext.Predicate();
 
-        public SemanticContext parent;
-
         /// <summary>
         /// For context independent predicates, we evaluate them without a local
         /// context (i.e., null context).
@@ -75,15 +81,51 @@ namespace Antlr4.Runtime.Atn
         /// having to create proper rule-specific context during prediction (as
         /// opposed to the parser, which creates them naturally). In a practical
         /// sense, this avoids a cast exception from RuleContext to myruleContext.
-        /// <p/>
-        /// For context dependent predicates, we must pass in a local context so that
+        /// <p>For context dependent predicates, we must pass in a local context so that
         /// references such as $arg evaluate properly as _localctx.arg. We only
         /// capture context dependent predicates in the context in which we begin
         /// prediction, so we passed in the outer context here in case of context
-        /// dependent predicate evaluation.
+        /// dependent predicate evaluation.</p>
         /// </remarks>
         public abstract bool Eval<_T0>(Recognizer<_T0> parser, RuleContext outerContext);
 
+        /// <summary>Evaluate the precedence predicates for the context and reduce the result.</summary>
+        /// <remarks>Evaluate the precedence predicates for the context and reduce the result.</remarks>
+        /// <param name="parser">The parser instance.</param>
+        /// <param name="outerContext">The current parser context object.</param>
+        /// <returns>
+        /// The simplified semantic context after precedence predicates are
+        /// evaluated, which will be one of the following values.
+        /// <ul>
+        /// <li>
+        /// <see cref="None">None</see>
+        /// : if the predicate simplifies to
+        /// <code>true</code>
+        /// after
+        /// precedence predicates are evaluated.</li>
+        /// <li>
+        /// <code>null</code>
+        /// : if the predicate simplifies to
+        /// <code>false</code>
+        /// after
+        /// precedence predicates are evaluated.</li>
+        /// <li>
+        /// <code>this</code>
+        /// : if the semantic context is not changed as a result of
+        /// precedence predicate evaluation.</li>
+        /// <li>A non-
+        /// <code>null</code>
+        /// 
+        /// <see cref="SemanticContext">SemanticContext</see>
+        /// : the new simplified
+        /// semantic context after precedence predicates are evaluated.</li>
+        /// </ul>
+        /// </returns>
+        public virtual SemanticContext EvalPrecedence<_T0>(Recognizer<_T0> parser, RuleContext outerContext)
+        {
+            return this;
+        }
+
         public class Predicate : SemanticContext
         {
             public readonly int ruleIndex;
@@ -162,6 +204,18 @@ namespace Antlr4.Runtime.Atn
                 return parser.Precpred(outerContext, precedence);
             }
 
+            public override SemanticContext EvalPrecedence<_T0>(Recognizer<_T0> parser, RuleContext outerContext)
+            {
+                if (parser.Precpred(outerContext, precedence))
+                {
+                    return SemanticContext.None;
+                }
+                else
+                {
+                    return null;
+                }
+            }
+
             public virtual int CompareTo(SemanticContext.PrecedencePredicate o)
             {
                 return precedence - o.precedence;
@@ -194,6 +248,14 @@ namespace Antlr4.Runtime.Atn
             }
         }
 
+        /// <summary>
+        /// A semantic context which is true whenever none of the contained contexts
+        /// is false.
+        /// </summary>
+        /// <remarks>
+        /// A semantic context which is true whenever none of the contained contexts
+        /// is false.
+        /// </remarks>
         public class AND : SemanticContext
         {
             [NotNull]
@@ -247,6 +309,12 @@ namespace Antlr4.Runtime.Atn
                 return MurmurHash.HashCode(opnds, typeof(SemanticContext.AND).GetHashCode());
             }
 
+            /// <summary>
+            /// <inheritDoc></inheritDoc>
+            /// <p>
+            /// The evaluation of predicates by this context is short-circuiting, but
+            /// unordered.</p>
+            /// </summary>
             public override bool Eval<_T0>(Recognizer<_T0> parser, RuleContext outerContext)
             {
                 foreach (SemanticContext opnd in opnds)
@@ -259,12 +327,59 @@ namespace Antlr4.Runtime.Atn
                 return true;
             }
 
+            public override SemanticContext EvalPrecedence<_T0>(Recognizer<_T0> parser, RuleContext outerContext)
+            {
+                bool differs = false;
+                IList<SemanticContext> operands = new List<SemanticContext>();
+                foreach (SemanticContext context in opnds)
+                {
+                    SemanticContext evaluated = context.EvalPrecedence(parser, outerContext);
+                    differs |= (evaluated != context);
+                    if (evaluated == null)
+                    {
+                        // The AND context is false if any element is false
+                        return null;
+                    }
+                    else
+                    {
+                        if (evaluated != None)
+                        {
+                            // Reduce the result by skipping true elements
+                            operands.AddItem(evaluated);
+                        }
+                    }
+                }
+                if (!differs)
+                {
+                    return this;
+                }
+                if (operands.IsEmpty())
+                {
+                    // all elements were true, so the AND context is true
+                    return None;
+                }
+                SemanticContext result = operands[0];
+                for (int i = 1; i < operands.Count; i++)
+                {
+                    result = SemanticContext.And(result, operands[i]);
+                }
+                return result;
+            }
+
             public override string ToString()
             {
                 return Utils.Join(opnds, "&&");
             }
         }
 
+        /// <summary>
+        /// A semantic context which is true whenever at least one of the contained
+        /// contexts is true.
+        /// </summary>
+        /// <remarks>
+        /// A semantic context which is true whenever at least one of the contained
+        /// contexts is true.
+        /// </remarks>
         public class OR : SemanticContext
         {
             [NotNull]
@@ -318,6 +433,12 @@ namespace Antlr4.Runtime.Atn
                 return MurmurHash.HashCode(opnds, typeof(SemanticContext.OR).GetHashCode());
             }
 
+            /// <summary>
+            /// <inheritDoc></inheritDoc>
+            /// <p>
+            /// The evaluation of predicates by this context is short-circuiting, but
+            /// unordered.</p>
+            /// </summary>
             public override bool Eval<_T0>(Recognizer<_T0> parser, RuleContext outerContext)
             {
                 foreach (SemanticContext opnd in opnds)
@@ -330,6 +451,45 @@ namespace Antlr4.Runtime.Atn
                 return false;
             }
 
+            public override SemanticContext EvalPrecedence<_T0>(Recognizer<_T0> parser, RuleContext outerContext)
+            {
+                bool differs = false;
+                IList<SemanticContext> operands = new List<SemanticContext>();
+                foreach (SemanticContext context in opnds)
+                {
+                    SemanticContext evaluated = context.EvalPrecedence(parser, outerContext);
+                    differs |= (evaluated != context);
+                    if (evaluated == None)
+                    {
+                        // The OR context is true if any element is true
+                        return None;
+                    }
+                    else
+                    {
+                        if (evaluated != null)
+                        {
+                            // Reduce the result by skipping false elements
+                            operands.AddItem(evaluated);
+                        }
+                    }
+                }
+                if (!differs)
+                {
+                    return this;
+                }
+                if (operands.IsEmpty())
+                {
+                    // all elements were false, so the OR context is false
+                    return null;
+                }
+                SemanticContext result = operands[0];
+                for (int i = 1; i < operands.Count; i++)
+                {
+                    result = SemanticContext.Or(result, operands[i]);
+                }
+                return result;
+            }
+
             public override string ToString()
             {
                 return Utils.Join(opnds, "||");

runtime/CSharp/Antlr4.Runtime/Atn/StarLoopEntryState.cs

@@ -36,6 +36,23 @@ namespace Antlr4.Runtime.Atn
     {
         public StarLoopbackState loopBackState;
 
+        /// <summary>
+        /// Indicates whether this state can benefit from a precedence DFA during SLL
+        /// decision making.
+        /// </summary>
+        /// <remarks>
+        /// Indicates whether this state can benefit from a precedence DFA during SLL
+        /// decision making.
+        /// <p>This is a computed property that is calculated during ATN deserialization
+        /// and stored for use in
+        /// <see cref="ParserATNSimulator">ParserATNSimulator</see>
+        /// and
+        /// <see cref="Antlr4.Runtime.ParserInterpreter">Antlr4.Runtime.ParserInterpreter</see>
+        /// .</p>
+        /// </remarks>
+        /// <seealso cref="Antlr4.Runtime.Dfa.DFA.IsPrecedenceDfa()">Antlr4.Runtime.Dfa.DFA.IsPrecedenceDfa()</seealso>
+        public bool precedenceRuleDecision;
+
         public override Antlr4.Runtime.Atn.StateType StateType
         {
             get

runtime/CSharp/Antlr4.Runtime/Atn/Transition.cs

@@ -39,15 +39,13 @@ namespace Antlr4.Runtime.Atn
     /// <remarks>
     /// An ATN transition between any two ATN states.  Subclasses define
     /// atom, set, epsilon, action, predicate, rule transitions.
-    /// <p/>
-    /// This is a one way link.  It emanates from a state (usually via a list of
-    /// transitions) and has a target state.
-    /// <p/>
-    /// Since we never have to change the ATN transitions once we construct it,
+    /// <p>This is a one way link.  It emanates from a state (usually via a list of
+    /// transitions) and has a target state.</p>
+    /// <p>Since we never have to change the ATN transitions once we construct it,
     /// we can fix these transitions as specific classes. The DFA transitions
     /// on the other hand need to update the labels as it adds transitions to
     /// the states. We'll use the term Edge for the DFA to distinguish them from
-    /// ATN transitions.
+    /// ATN transitions.</p>
     /// </remarks>
     public abstract class Transition
     {

runtime/CSharp/Antlr4.Runtime/BailErrorStrategy.cs

@@ -32,12 +32,39 @@ using Sharpen;
 
 namespace Antlr4.Runtime
 {
-    /// <summary>Bail out of parser at first syntax error.</summary>
-    /// <remarks>
-    /// Bail out of parser at first syntax error. Do this to use it:
-    /// <p/>
+    /// <summary>
+    /// This implementation of
+    /// <see cref="IAntlrErrorStrategy">IAntlrErrorStrategy</see>
+    /// responds to syntax errors
+    /// by immediately canceling the parse operation with a
+    /// <see cref="ParseCanceledException">ParseCanceledException</see>
+    /// . The implementation ensures that the
+    /// <see cref="ParserRuleContext.exception">ParserRuleContext.exception</see>
+    /// field is set for all parse tree nodes
+    /// that were not completed prior to encountering the error.
+    /// <p>
+    /// This error strategy is useful in the following scenarios.</p>
+    /// <ul>
+    /// <li><strong>Two-stage parsing:</strong> This error strategy allows the first
+    /// stage of two-stage parsing to immediately terminate if an error is
+    /// encountered, and immediately fall back to the second stage. In addition to
+    /// avoiding wasted work by attempting to recover from errors here, the empty
+    /// implementation of
+    /// <see cref="Sync(Parser)">Sync(Parser)</see>
+    /// improves the performance of
+    /// the first stage.</li>
+    /// <li><strong>Silent validation:</strong> When syntax errors are not being
+    /// reported or logged, and the parse result is simply ignored if errors occur,
+    /// the
+    /// <see cref="BailErrorStrategy">BailErrorStrategy</see>
+    /// avoids wasting work on recovering from errors
+    /// when the result will be ignored either way.</li>
+    /// </ul>
+    /// <p>
     /// <code>myparser.setErrorHandler(new BailErrorStrategy());</code>
-    /// </remarks>
+    /// </p>
+    /// </summary>
+    /// <seealso cref="Parser.ErrorHandler(IAntlrErrorStrategy)">Parser.ErrorHandler(IAntlrErrorStrategy)</seealso>
     public class BailErrorStrategy : DefaultErrorStrategy
     {
         /// <summary>

runtime/CSharp/Antlr4.Runtime/BaseErrorListener.cs

@@ -34,6 +34,13 @@ using Sharpen;
 
 namespace Antlr4.Runtime
 {
+    /// <summary>
+    /// Provides an empty default implementation of
+    /// <see cref="IANTLRErrorListener{Symbol}">IANTLRErrorListener&lt;Symbol&gt;</see>
+    /// . The
+    /// default implementation of each method does nothing, but can be overridden as
+    /// necessary.
+    /// </summary>
     /// <author>Sam Harwell</author>
     public class BaseErrorListener : IParserErrorListener
     {

runtime/CSharp/Antlr4.Runtime/BufferedTokenStream.cs

@@ -36,41 +36,45 @@ using Sharpen;
 
 namespace Antlr4.Runtime
 {
-    /// <summary>Buffer all input tokens but do on-demand fetching of new tokens from lexer.</summary>
-    /// <remarks>
-    /// Buffer all input tokens but do on-demand fetching of new tokens from lexer.
-    /// Useful when the parser or lexer has to set context/mode info before proper
-    /// lexing of future tokens. The ST template parser needs this, for example,
-    /// because it has to constantly flip back and forth between inside/output
-    /// templates. E.g.,
-    /// <code></code>
-    /// &lt;names:
-    /// hi, <it>}&gt;} has to parse names as part of an
-    /// expression but
-    /// <code>"hi, <it>"</code>
-    /// as a nested template.
-    /// <p/>
-    /// You can't use this stream if you pass whitespace or other off-channel tokens
-    /// to the parser. The stream can't ignore off-channel tokens.
-    /// (
-    /// <see cref="UnbufferedTokenStream">UnbufferedTokenStream</see>
-    /// is the same way.) Use
+    /// <summary>
+    /// This implementation of
+    /// <see cref="ITokenStream">ITokenStream</see>
+    /// loads tokens from a
+    /// <see cref="ITokenSource">ITokenSource</see>
+    /// on-demand, and places the tokens in a buffer to provide
+    /// access to any previous token by index.
+    /// <p>
+    /// This token stream ignores the value of
+    /// <see cref="IToken.Channel()">IToken.Channel()</see>
+    /// . If your
+    /// parser requires the token stream filter tokens to only those on a particular
+    /// channel, such as
+    /// <see cref="IToken.DefaultChannel">IToken.DefaultChannel</see>
+    /// or
+    /// <see cref="IToken.HiddenChannel">IToken.HiddenChannel</see>
+    /// , use a filtering token stream such a
     /// <see cref="CommonTokenStream">CommonTokenStream</see>
-    /// .
-    /// </remarks>
+    /// .</p>
+    /// </summary>
     public class BufferedTokenStream : ITokenStream
     {
+        /// <summary>
+        /// The
+        /// <see cref="ITokenSource">ITokenSource</see>
+        /// from which tokens for this stream are fetched.
+        /// </summary>
         [NotNull]
         protected internal ITokenSource tokenSource;
 
-        /// <summary>
-        /// Record every single token pulled from the source so we can reproduce
-        /// chunks of it later.
-        /// </summary>
+        /// <summary>A collection of all tokens fetched from the token source.</summary>
         /// <remarks>
-        /// Record every single token pulled from the source so we can reproduce
-        /// chunks of it later. This list captures everything so we can access
-        /// complete input text.
+        /// A collection of all tokens fetched from the token source. The list is
+        /// considered a complete view of the input once
+        /// <see cref="fetchedEOF">fetchedEOF</see>
+        /// is set
+        /// to
+        /// <code>true</code>
+        /// .
         /// </remarks>
         protected internal IList<IToken> tokens = new List<IToken>(100);
 
@@ -78,7 +82,8 @@ namespace Antlr4.Runtime
         /// The index into
         /// <see cref="tokens">tokens</see>
         /// of the current token (next token to
-        /// consume).
+        /// <see cref="Consume()">Consume()</see>
+        /// ).
         /// <see cref="tokens">tokens</see>
         /// <code>[</code>
         /// <see cref="p">p</see>
@@ -86,27 +91,46 @@ namespace Antlr4.Runtime
         /// should be
         /// <see cref="Lt(int)">LT(1)</see>
         /// .
-        /// <see cref="p">p</see>
-        /// <code>=-1</code>
-        /// indicates need to initialize
-        /// with first token. The constructor doesn't get a token. First call to
-        /// <see cref="Lt(int)">LT(1)</see>
-        /// or whatever gets the first token and sets
-        /// <see cref="p">p</see>
-        /// <code>=0;</code>
-        /// .
+        /// <p>This field is set to -1 when the stream is first constructed or when
+        /// <see cref="SetTokenSource(ITokenSource)">SetTokenSource(ITokenSource)</see>
+        /// is called, indicating that the first token has
+        /// not yet been fetched from the token source. For additional information,
+        /// see the documentation of
+        /// <see cref="IIntStream">IIntStream</see>
+        /// for a description of
+        /// Initializing Methods.</p>
         /// </summary>
         protected internal int p = -1;
 
         /// <summary>
-        /// Set to
-        /// <code>true</code>
-        /// when the EOF token is fetched. Do not continue fetching
-        /// tokens after that point, or multiple EOF tokens could end up in the
+        /// Indicates whether the
+        /// <see cref="IToken.Eof">IToken.Eof</see>
+        /// token has been fetched from
+        /// <see cref="tokenSource">tokenSource</see>
+        /// and added to
         /// <see cref="tokens">tokens</see>
-        /// array.
+        /// . This field improves
+        /// performance for the following cases:
+        /// <ul>
+        /// <li>
+        /// <see cref="Consume()">Consume()</see>
+        /// : The lookahead check in
+        /// <see cref="Consume()">Consume()</see>
+        /// to prevent
+        /// consuming the EOF symbol is optimized by checking the values of
+        /// <see cref="fetchedEOF">fetchedEOF</see>
+        /// and
+        /// <see cref="p">p</see>
+        /// instead of calling
+        /// <see cref="La(int)">La(int)</see>
+        /// .</li>
+        /// <li>
+        /// <see cref="Fetch(int)">Fetch(int)</see>
+        /// : The check to prevent adding multiple EOF symbols into
+        /// <see cref="tokens">tokens</see>
+        /// is trivial with this field.</li>
+        /// <ul>
         /// </summary>
-        /// <seealso cref="Fetch(int)">Fetch(int)</seealso>
         protected internal bool fetchedEOF;
 
         public BufferedTokenStream(ITokenSource tokenSource)
@@ -165,7 +189,27 @@ namespace Antlr4.Runtime
 
         public virtual void Consume()
         {
-            if (La(1) == IntStreamConstants.Eof)
+            bool skipEofCheck;
+            if (p >= 0)
+            {
+                if (fetchedEOF)
+                {
+                    // the last token in tokens is EOF. skip check if p indexes any
+                    // fetched token except the last.
+                    skipEofCheck = p < tokens.Count - 1;
+                }
+                else
+                {
+                    // no EOF token in tokens. skip check if p indexes a fetched token.
+                    skipEofCheck = p < tokens.Count;
+                }
+            }
+            else
+            {
+                // not yet initialized
+                skipEofCheck = false;
+            }
+            if (!skipEofCheck && La(1) == IntStreamConstants.Eof)
             {
                 throw new InvalidOperationException("cannot consume EOF");
             }
@@ -282,6 +326,7 @@ namespace Antlr4.Runtime
             return tokens[p - k];
         }
 
+        [NotNull]
         public virtual IToken Lt(int k)
         {
             LazyInit();
@@ -318,11 +363,10 @@ namespace Antlr4.Runtime
         /// . If an
         /// exception is thrown in this method, the current stream index should not be
         /// changed.
-        /// <p/>
-        /// For example,
+        /// <p>For example,
         /// <see cref="CommonTokenStream">CommonTokenStream</see>
         /// overrides this method to ensure that
-        /// the seek target is always an on-channel token.
+        /// the seek target is always an on-channel token.</p>
         /// </remarks>
         /// <param name="i">The target token index.</param>
         /// <returns>The adjusted target token index.</returns>

runtime/CSharp/Antlr4.Runtime/CommonToken.cs

@@ -39,40 +39,119 @@ namespace Antlr4.Runtime
     {
         private const long serialVersionUID = -6708843461296520577L;
 
+        /// <summary>
+        /// An empty
+        /// <see cref="Antlr4.Runtime.Misc.Tuple2{T1, T2}">Antlr4.Runtime.Misc.Tuple2&lt;T1, T2&gt;</see>
+        /// which is used as the default value of
+        /// <see cref="source">source</see>
+        /// for tokens that do not have a source.
+        /// </summary>
         protected internal static readonly Tuple<ITokenSource, ICharStream> EmptySource = Tuple.Create<ITokenSource, ICharStream>(null, null);
 
+        /// <summary>
+        /// This is the backing field for
+        /// <see cref="Type()">Type()</see>
+        /// and
+        /// <see cref="Type(int)">Type(int)</see>
+        /// .
+        /// </summary>
         protected internal int type;
 
+        /// <summary>
+        /// This is the backing field for
+        /// <see cref="Line()">Line()</see>
+        /// and
+        /// <see cref="Line(int)">Line(int)</see>
+        /// .
+        /// </summary>
         protected internal int line;
 
+        /// <summary>
+        /// This is the backing field for
+        /// <see cref="Column()">Column()</see>
+        /// and
+        /// <see cref="Column(int)">Column(int)</see>
+        /// .
+        /// </summary>
         protected internal int charPositionInLine = -1;
 
+        /// <summary>
+        /// This is the backing field for
+        /// <see cref="Channel()">Channel()</see>
+        /// and
+        /// <see cref="Channel(int)">Channel(int)</see>
+        /// .
+        /// </summary>
         protected internal int channel = TokenConstants.DefaultChannel;
 
+        /// <summary>
+        /// This is the backing field for
+        /// <see cref="TokenSource()">TokenSource()</see>
+        /// and
+        /// <see cref="InputStream()">InputStream()</see>
+        /// .
+        /// <p>
+        /// These properties share a field to reduce the memory footprint of
+        /// <see cref="CommonToken">CommonToken</see>
+        /// . Tokens created by a
+        /// <see cref="CommonTokenFactory">CommonTokenFactory</see>
+        /// from
+        /// the same source and input stream share a reference to the same
+        /// <see cref="Antlr4.Runtime.Misc.Tuple2{T1, T2}">Antlr4.Runtime.Misc.Tuple2&lt;T1, T2&gt;</see>
+        /// containing these values.</p>
+        /// </summary>
+        [NotNull]
         protected internal Tuple<ITokenSource, ICharStream> source;
 
-        /// <summary>We need to be able to change the text once in a while.</summary>
-        /// <remarks>
-        /// We need to be able to change the text once in a while.  If
-        /// this is non-null, then getText should return this.  Note that
-        /// start/stop are not affected by changing this.
-        /// </remarks>
+        /// <summary>
+        /// This is the backing field for
+        /// <see cref="Text()">Text()</see>
+        /// when the token text is
+        /// explicitly set in the constructor or via
+        /// <see cref="Text(string)">Text(string)</see>
+        /// .
+        /// </summary>
+        /// <seealso cref="Text()">Text()</seealso>
         protected internal string text;
 
-        /// <summary>What token number is this from 0..n-1 tokens; &lt; 0 implies invalid index</summary>
+        /// <summary>
+        /// This is the backing field for
+        /// <see cref="TokenIndex()">TokenIndex()</see>
+        /// and
+        /// <see cref="TokenIndex(int)">TokenIndex(int)</see>
+        /// .
+        /// </summary>
         protected internal int index = -1;
 
-        /// <summary>The char position into the input buffer where this token starts</summary>
+        /// <summary>
+        /// This is the backing field for
+        /// <see cref="StartIndex()">StartIndex()</see>
+        /// and
+        /// <see cref="SetStartIndex(int)">SetStartIndex(int)</see>
+        /// .
+        /// </summary>
         protected internal int start;
 
-        /// <summary>The char position into the input buffer where this token stops</summary>
+        /// <summary>
+        /// This is the backing field for
+        /// <see cref="StopIndex()">StopIndex()</see>
+        /// and
+        /// <see cref="SetStopIndex(int)">SetStopIndex(int)</see>
+        /// .
+        /// </summary>
         protected internal int stop;
 
+        /// <summary>
+        /// Constructs a new
+        /// <see cref="CommonToken">CommonToken</see>
+        /// with the specified token type.
+        /// </summary>
+        /// <param name="type">The token type.</param>
         public CommonToken(int type)
         {
             // set to invalid position
-            // TODO: can store these in map in token stream rather than as field here
             this.type = type;
+            this.source = EmptySource;
         }
 
         public CommonToken(Tuple<ITokenSource, ICharStream> source, int type, int channel, int start, int stop)
@@ -89,6 +168,14 @@ namespace Antlr4.Runtime
             }
         }
 
+        /// <summary>
+        /// Constructs a new
+        /// <see cref="CommonToken">CommonToken</see>
+        /// with the specified token type and
+        /// text.
+        /// </summary>
+        /// <param name="type">The token type.</param>
+        /// <param name="text">The text of the token.</param>
         public CommonToken(int type, string text)
         {
             this.type = type;
@@ -97,9 +184,41 @@ namespace Antlr4.Runtime
             this.source = EmptySource;
         }
 
+        /// <summary>
+        /// Constructs a new
+        /// <see cref="CommonToken">CommonToken</see>
+        /// as a copy of another
+        /// <see cref="IToken">IToken</see>
+        /// .
+        /// <p>
+        /// If
+        /// <code>oldToken</code>
+        /// is also a
+        /// <see cref="CommonToken">CommonToken</see>
+        /// instance, the newly
+        /// constructed token will share a reference to the
+        /// <see cref="text">text</see>
+        /// field and
+        /// the
+        /// <see cref="Antlr4.Runtime.Misc.Tuple2{T1, T2}">Antlr4.Runtime.Misc.Tuple2&lt;T1, T2&gt;</see>
+        /// stored in
+        /// <see cref="source">source</see>
+        /// . Otherwise,
+        /// <see cref="text">text</see>
+        /// will
+        /// be assigned the result of calling
+        /// <see cref="Text()">Text()</see>
+        /// , and
+        /// <see cref="source">source</see>
+        /// will be constructed from the result of
+        /// <see cref="IToken.TokenSource()">IToken.TokenSource()</see>
+        /// and
+        /// <see cref="IToken.InputStream()">IToken.InputStream()</see>
+        /// .</p>
+        /// </summary>
+        /// <param name="oldToken">The token to copy.</param>
         public CommonToken(IToken oldToken)
         {
-            text = oldToken.Text;
             type = oldToken.Type;
             line = oldToken.Line;
             index = oldToken.TokenIndex;
@@ -109,10 +228,12 @@ namespace Antlr4.Runtime
             stop = oldToken.StopIndex;
             if (oldToken is Antlr4.Runtime.CommonToken)
             {
+                text = ((Antlr4.Runtime.CommonToken)oldToken).text;
                 source = ((Antlr4.Runtime.CommonToken)oldToken).source;
             }
             else
             {
+                text = oldToken.Text;
                 source = Tuple.Create(oldToken.TokenSource, oldToken.InputStream);
             }
         }
@@ -143,13 +264,22 @@ namespace Antlr4.Runtime
             }
         }
 
-        /// <summary>Override the text for this token.</summary>
+        /// <summary>Explicitly set the text for this token.</summary>
         /// <remarks>
-        /// Override the text for this token.  getText() will return this text
-        /// rather than pulling from the buffer.  Note that this does not mean
-        /// that start/stop indexes are not valid.  It means that that input
-        /// was converted to a new string in the token object.
+        /// Explicitly set the text for this token. If {code text} is not
+        /// <code>null</code>
+        /// , then
+        /// <see cref="Text()">Text()</see>
+        /// will return this value rather than
+        /// extracting the text from the input.
         /// </remarks>
+        /// <value>
+        /// The explicit text of the token, or
+        /// <code>null</code>
+        /// if the text
+        /// should be obtained from the input along with the start and stop indexes
+        /// of the token.
+        /// </value>
         public virtual string Text
         {
             get

runtime/CSharp/Antlr4.Runtime/CommonTokenFactory.cs

@@ -34,31 +34,91 @@ using Sharpen;
 
 namespace Antlr4.Runtime
 {
+    /// <summary>
+    /// This default implementation of
+    /// <see cref="ITokenFactory">ITokenFactory</see>
+    /// creates
+    /// <see cref="CommonToken">CommonToken</see>
+    /// objects.
+    /// </summary>
     public class CommonTokenFactory : ITokenFactory
     {
+        /// <summary>
+        /// The default
+        /// <see cref="CommonTokenFactory">CommonTokenFactory</see>
+        /// instance.
+        /// <p>
+        /// This token factory does not explicitly copy token text when constructing
+        /// tokens.</p>
+        /// </summary>
         public static readonly ITokenFactory Default = new Antlr4.Runtime.CommonTokenFactory();
 
-        /// <summary>Copy text for token out of input char stream.</summary>
-        /// <remarks>
-        /// Copy text for token out of input char stream. Useful when input
-        /// stream is unbuffered.
-        /// </remarks>
-        /// <seealso cref="UnbufferedCharStream">UnbufferedCharStream</seealso>
+        /// <summary>
+        /// Indicates whether
+        /// <see cref="CommonToken.Text(string)">CommonToken.Text(string)</see>
+        /// should be called after
+        /// constructing tokens to explicitly set the text. This is useful for cases
+        /// where the input stream might not be able to provide arbitrary substrings
+        /// of text from the input after the lexer creates a token (e.g. the
+        /// implementation of
+        /// <see cref="ICharStream.GetText(Antlr4.Runtime.Misc.Interval)">ICharStream.GetText(Antlr4.Runtime.Misc.Interval)</see>
+        /// in
+        /// <see cref="UnbufferedCharStream">UnbufferedCharStream</see>
+        /// throws an
+        /// <see cref="System.NotSupportedException">System.NotSupportedException</see>
+        /// ). Explicitly setting the token text
+        /// allows
+        /// <see cref="IToken.Text()">IToken.Text()</see>
+        /// to be called at any time regardless of the
+        /// input stream implementation.
+        /// <p>
+        /// The default value is
+        /// <code>false</code>
+        /// to avoid the performance and memory
+        /// overhead of copying text for every token unless explicitly requested.</p>
+        /// </summary>
         protected internal readonly bool copyText;
 
         /// <summary>
-        /// Create factory and indicate whether or not the factory copy
-        /// text out of the char stream.
+        /// Constructs a
+        /// <see cref="CommonTokenFactory">CommonTokenFactory</see>
+        /// with the specified value for
+        /// <see cref="copyText">copyText</see>
+        /// .
+        /// <p>
+        /// When
+        /// <code>copyText</code>
+        /// is
+        /// <code>false</code>
+        /// , the
+        /// <see cref="Default">Default</see>
+        /// instance
+        /// should be used instead of constructing a new instance.</p>
         /// </summary>
-        /// <remarks>
-        /// Create factory and indicate whether or not the factory copy
-        /// text out of the char stream.
-        /// </remarks>
+        /// <param name="copyText">
+        /// The value for
+        /// <see cref="copyText">copyText</see>
+        /// .
+        /// </param>
         public CommonTokenFactory(bool copyText)
         {
             this.copyText = copyText;
         }
 
+        /// <summary>
+        /// Constructs a
+        /// <see cref="CommonTokenFactory">CommonTokenFactory</see>
+        /// with
+        /// <see cref="copyText">copyText</see>
+        /// set to
+        /// <code>false</code>
+        /// .
+        /// <p>
+        /// The
+        /// <see cref="Default">Default</see>
+        /// instance should be used instead of calling this
+        /// directly.</p>
+        /// </summary>
         public CommonTokenFactory()
             : this(false)
         {

runtime/CSharp/Antlr4.Runtime/CommonTokenStream.cs

@@ -33,42 +33,88 @@ using Sharpen;
 namespace Antlr4.Runtime
 {
     /// <summary>
-    /// The most common stream of tokens where every token is buffered up
-    /// and tokens are filtered for a certain channel (the parser will only
-    /// see these tokens).
-    /// </summary>
-    /// <remarks>
-    /// The most common stream of tokens where every token is buffered up
-    /// and tokens are filtered for a certain channel (the parser will only
-    /// see these tokens).
-    /// Even though it buffers all of the tokens, this token stream pulls tokens
-    /// from the tokens source on demand. In other words, until you ask for a
-    /// token using consume(), LT(), etc. the stream does not pull from the lexer.
-    /// The only difference between this stream and
+    /// This class extends
     /// <see cref="BufferedTokenStream">BufferedTokenStream</see>
-    /// superclass
-    /// is that this stream knows how to ignore off channel tokens. There may be
-    /// a performance advantage to using the superclass if you don't pass
-    /// whitespace and comments etc. to the parser on a hidden channel (i.e.,
-    /// you set
-    /// <code>$channel</code>
-    /// instead of calling
-    /// <code>skip()</code>
-    /// in lexer rules.)
-    /// </remarks>
-    /// <seealso cref="UnbufferedTokenStream">UnbufferedTokenStream</seealso>
-    /// <seealso cref="BufferedTokenStream">BufferedTokenStream</seealso>
+    /// with functionality to filter
+    /// token streams to tokens on a particular channel (tokens where
+    /// <see cref="IToken.Channel()">IToken.Channel()</see>
+    /// returns a particular value).
+    /// <p>
+    /// This token stream provides access to all tokens by index or when calling
+    /// methods like
+    /// <see cref="BufferedTokenStream.GetText()">BufferedTokenStream.GetText()</see>
+    /// . The channel filtering is only used for code
+    /// accessing tokens via the lookahead methods
+    /// <see cref="BufferedTokenStream.La(int)">BufferedTokenStream.La(int)</see>
+    /// ,
+    /// <see cref="Lt(int)">Lt(int)</see>
+    /// , and
+    /// <see cref="Lb(int)">Lb(int)</see>
+    /// .</p>
+    /// <p>
+    /// By default, tokens are placed on the default channel
+    /// (
+    /// <see cref="IToken.DefaultChannel">IToken.DefaultChannel</see>
+    /// ), but may be reassigned by using the
+    /// <code>-&gt;channel(HIDDEN)</code>
+    /// lexer command, or by using an embedded action to
+    /// call
+    /// <see cref="Lexer.Channel(int)">Lexer.Channel(int)</see>
+    /// .
+    /// </p>
+    /// <p>
+    /// Note: lexer rules which use the
+    /// <code>-&gt;skip</code>
+    /// lexer command or call
+    /// <see cref="Lexer.Skip()">Lexer.Skip()</see>
+    /// do not produce tokens at all, so input text matched by
+    /// such a rule will not be available as part of the token stream, regardless of
+    /// channel.</p>
+    /// </summary>
     public class CommonTokenStream : BufferedTokenStream
     {
-        /// <summary>Skip tokens on any channel but this one; this is how we skip whitespace...</summary>
-        /// <remarks>Skip tokens on any channel but this one; this is how we skip whitespace...</remarks>
+        /// <summary>Specifies the channel to use for filtering tokens.</summary>
+        /// <remarks>
+        /// Specifies the channel to use for filtering tokens.
+        /// <p>
+        /// The default value is
+        /// <see cref="IToken.DefaultChannel">IToken.DefaultChannel</see>
+        /// , which matches the
+        /// default channel assigned to tokens created by the lexer.</p>
+        /// </remarks>
         protected internal int channel = TokenConstants.DefaultChannel;
 
+        /// <summary>
+        /// Constructs a new
+        /// <see cref="CommonTokenStream">CommonTokenStream</see>
+        /// using the specified token
+        /// source and the default token channel (
+        /// <see cref="IToken.DefaultChannel">IToken.DefaultChannel</see>
+        /// ).
+        /// </summary>
+        /// <param name="tokenSource">The token source.</param>
         public CommonTokenStream(ITokenSource tokenSource)
             : base(tokenSource)
         {
         }
 
+        /// <summary>
+        /// Constructs a new
+        /// <see cref="CommonTokenStream">CommonTokenStream</see>
+        /// using the specified token
+        /// source and filtering tokens to the specified channel. Only tokens whose
+        /// <see cref="IToken.Channel()">IToken.Channel()</see>
+        /// matches
+        /// <code>channel</code>
+        /// or have the
+        /// <see cref="IToken.Type()">IToken.Type()</see>
+        /// equal to
+        /// <see cref="IToken.Eof">IToken.Eof</see>
+        /// will be returned by the
+        /// token stream lookahead methods.
+        /// </summary>
+        /// <param name="tokenSource">The token source.</param>
+        /// <param name="channel">The channel to use for filtering tokens.</param>
         public CommonTokenStream(ITokenSource tokenSource, int channel)
             : this(tokenSource)
         {

runtime/CSharp/Antlr4.Runtime/ConsoleErrorListener.cs

@@ -35,8 +35,31 @@ namespace Antlr4.Runtime
     /// <author>Sam Harwell</author>
     public class ConsoleErrorListener : IAntlrErrorListener<object>
     {
+        /// <summary>
+        /// Provides a default instance of
+        /// <see cref="ConsoleErrorListener">ConsoleErrorListener</see>
+        /// .
+        /// </summary>
         public static readonly ConsoleErrorListener Instance = new ConsoleErrorListener();
 
+        /// <summary>
+        /// <inheritDoc></inheritDoc>
+        /// <p>
+        /// This implementation prints messages to
+        /// <see cref="System.Console.Error">System.Console.Error</see>
+        /// containing the
+        /// values of
+        /// <code>line</code>
+        /// ,
+        /// <code>charPositionInLine</code>
+        /// , and
+        /// <code>msg</code>
+        /// using
+        /// the following format.</p>
+        /// <pre>
+        /// line <em>line</em>:<em>charPositionInLine</em> <em>msg</em>
+        /// </pre>
+        /// </summary>
         public virtual void SyntaxError<T>(Recognizer<T, object> recognizer, T offendingSymbol, int line, int charPositionInLine, string msg, RecognitionException e)
         {
             System.Console.Error.WriteLine("line " + line + ":" + charPositionInLine + " " + msg);

runtime/CSharp/Antlr4.Runtime/DefaultErrorStrategy.cs

@@ -36,23 +36,21 @@ using Sharpen;
 namespace Antlr4.Runtime
 {
     /// <summary>
-    /// This is the default error handling mechanism for ANTLR parsers
-    /// and tree parsers.
+    /// This is the default implementation of
+    /// <see cref="IAntlrErrorStrategy">IAntlrErrorStrategy</see>
+    /// used for
+    /// error reporting and recovery in ANTLR parsers.
     /// </summary>
-    /// <remarks>
-    /// This is the default error handling mechanism for ANTLR parsers
-    /// and tree parsers.
-    /// </remarks>
     public class DefaultErrorStrategy : IAntlrErrorStrategy
     {
         /// <summary>
-        /// This is true after we see an error and before having successfully
-        /// matched a token.
+        /// Indicates whether the error strategy is currently "recovering from an
+        /// error".
         /// </summary>
         /// <remarks>
-        /// This is true after we see an error and before having successfully
-        /// matched a token. Prevents generation of more than one error message
-        /// per error.
+        /// Indicates whether the error strategy is currently "recovering from an
+        /// error". This is used to suppress reporting multiple error messages while
+        /// attempting to recover from a detected syntax error.
         /// </remarks>
         /// <seealso cref="InErrorRecoveryMode(Parser)">InErrorRecoveryMode(Parser)</seealso>
         protected internal bool errorRecoveryMode = false;
@@ -71,11 +69,10 @@ namespace Antlr4.Runtime
 
         /// <summary>
         /// <inheritDoc></inheritDoc>
-        /// <p/>
-        /// The default implementation simply calls
+        /// <p>The default implementation simply calls
         /// <see cref="EndErrorCondition(Parser)">EndErrorCondition(Parser)</see>
         /// to
-        /// ensure that the handler is not in error recovery mode.
+        /// ensure that the handler is not in error recovery mode.</p>
         /// </summary>
         public virtual void Reset(Parser recognizer)
         {
@@ -120,10 +117,9 @@ namespace Antlr4.Runtime
 
         /// <summary>
         /// <inheritDoc></inheritDoc>
-        /// <p/>
-        /// The default implementation simply calls
+        /// <p>The default implementation simply calls
         /// <see cref="EndErrorCondition(Parser)">EndErrorCondition(Parser)</see>
-        /// .
+        /// .</p>
         /// </summary>
         public virtual void ReportMatch(Parser recognizer)
         {
@@ -132,13 +128,12 @@ namespace Antlr4.Runtime
 
         /// <summary>
         /// <inheritDoc></inheritDoc>
-        /// <p/>
-        /// The default implementation returns immediately if the handler is already
+        /// <p>The default implementation returns immediately if the handler is already
         /// in error recovery mode. Otherwise, it calls
         /// <see cref="BeginErrorCondition(Parser)">BeginErrorCondition(Parser)</see>
         /// and dispatches the reporting task based on the runtime type of
         /// <code>e</code>
-        /// according to the following table.
+        /// according to the following table.</p>
         /// <ul>
         /// <li>
         /// <see cref="NoViableAltException">NoViableAltException</see>
@@ -204,10 +199,9 @@ namespace Antlr4.Runtime
 
         /// <summary>
         /// <inheritDoc></inheritDoc>
-        /// <p/>
-        /// The default implementation resynchronizes the parser by consuming tokens
+        /// <p>The default implementation resynchronizes the parser by consuming tokens
         /// until we find one in the resynchronization set--loosely the set of tokens
-        /// that can follow the current rule.
+        /// that can follow the current rule.</p>
         /// </summary>
         public virtual void Recover(Parser recognizer, RecognitionException e)
         {
@@ -244,9 +238,8 @@ namespace Antlr4.Runtime
         /// that the current lookahead symbol is consistent with what were expecting
         /// at this point in the ATN. You can call this anytime but ANTLR only
         /// generates code to check before subrules/loops and each iteration.
-        /// <p/>
-        /// Implements Jim Idle's magic sync mechanism in closures and optional
-        /// subrules. E.g.,
+        /// <p>Implements Jim Idle's magic sync mechanism in closures and optional
+        /// subrules. E.g.,</p>
         /// <pre>
         /// a : sync ( stuff sync )* ;
         /// sync : {consume to what can follow sync} ;
@@ -257,35 +250,30 @@ namespace Antlr4.Runtime
         /// token deletion, if possible. If it can't do that, it bails on the current
         /// rule and uses the default error recovery, which consumes until the
         /// resynchronization set of the current rule.
-        /// <p/>
-        /// If the sub rule is optional (
+        /// <p>If the sub rule is optional (
         /// <code>(...)?</code>
         /// ,
         /// <code>(...)*</code>
         /// , or block
         /// with an empty alternative), then the expected set includes what follows
-        /// the subrule.
-        /// <p/>
-        /// During loop iteration, it consumes until it sees a token that can start a
+        /// the subrule.</p>
+        /// <p>During loop iteration, it consumes until it sees a token that can start a
         /// sub rule or what follows loop. Yes, that is pretty aggressive. We opt to
-        /// stay in the loop as long as possible.
-        /// <p/>
-        /// <strong>ORIGINS</strong>
-        /// <p/>
-        /// Previous versions of ANTLR did a poor job of their recovery within loops.
+        /// stay in the loop as long as possible.</p>
+        /// <p><strong>ORIGINS</strong></p>
+        /// <p>Previous versions of ANTLR did a poor job of their recovery within loops.
         /// A single mismatch token or missing token would force the parser to bail
-        /// out of the entire rules surrounding the loop. So, for rule
+        /// out of the entire rules surrounding the loop. So, for rule</p>
         /// <pre>
         /// classDef : 'class' ID '{' member* '}'
         /// </pre>
         /// input with an extra token between members would force the parser to
         /// consume until it found the next class definition rather than the next
         /// member definition of the current class.
-        /// <p/>
-        /// This functionality cost a little bit of effort because the parser has to
+        /// <p>This functionality cost a little bit of effort because the parser has to
         /// compare token set at the start of the loop and at each iteration. If for
         /// some reason speed is suffering for you, you can turn off this
-        /// functionality by simply overriding this method as a blank { }.
+        /// functionality by simply overriding this method as a blank { }.</p>
         /// </summary>
         /// <exception cref="Antlr4.Runtime.RecognitionException"></exception>
         public virtual void Sync(Parser recognizer)
@@ -357,7 +345,7 @@ namespace Antlr4.Runtime
         {
             ITokenStream tokens = ((ITokenStream)recognizer.InputStream);
             string input;
-            if (tokens is ITokenStream)
+            if (tokens != null)
             {
                 if (e.GetStartToken().Type == TokenConstants.Eof)
                 {
@@ -422,20 +410,18 @@ namespace Antlr4.Runtime
         /// removed from the input stream. When this method returns,
         /// <code>recognizer</code>
         /// is in error recovery mode.
-        /// <p/>
-        /// This method is called when
+        /// <p>This method is called when
         /// <see cref="SingleTokenDeletion(Parser)">SingleTokenDeletion(Parser)</see>
         /// identifies
         /// single-token deletion as a viable recovery strategy for a mismatched
-        /// input error.
-        /// <p/>
-        /// The default implementation simply returns if the handler is already in
+        /// input error.</p>
+        /// <p>The default implementation simply returns if the handler is already in
         /// error recovery mode. Otherwise, it calls
         /// <see cref="BeginErrorCondition(Parser)">BeginErrorCondition(Parser)</see>
         /// to
         /// enter error recovery mode, followed by calling
         /// <see cref="Parser.NotifyErrorListeners(string)">Parser.NotifyErrorListeners(string)</see>
-        /// .
+        /// .</p>
         /// </remarks>
         /// <param name="recognizer">the parser instance</param>
         protected internal virtual void ReportUnwantedToken(Parser recognizer)
@@ -463,20 +449,18 @@ namespace Antlr4.Runtime
         /// method returns,
         /// <code>recognizer</code>
         /// is in error recovery mode.
-        /// <p/>
-        /// This method is called when
+        /// <p>This method is called when
         /// <see cref="SingleTokenInsertion(Parser)">SingleTokenInsertion(Parser)</see>
         /// identifies
         /// single-token insertion as a viable recovery strategy for a mismatched
-        /// input error.
-        /// <p/>
-        /// The default implementation simply returns if the handler is already in
+        /// input error.</p>
+        /// <p>The default implementation simply returns if the handler is already in
         /// error recovery mode. Otherwise, it calls
         /// <see cref="BeginErrorCondition(Parser)">BeginErrorCondition(Parser)</see>
         /// to
         /// enter error recovery mode, followed by calling
         /// <see cref="Parser.NotifyErrorListeners(string)">Parser.NotifyErrorListeners(string)</see>
-        /// .
+        /// .</p>
         /// </remarks>
         /// <param name="recognizer">the parser instance</param>
         protected internal virtual void ReportMissingToken(Parser recognizer)
@@ -494,15 +478,13 @@ namespace Antlr4.Runtime
 
         /// <summary>
         /// <inheritDoc></inheritDoc>
-        /// <p/>
-        /// The default implementation attempts to recover from the mismatched input
+        /// <p>The default implementation attempts to recover from the mismatched input
         /// by using single token insertion and deletion as described below. If the
         /// recovery attempt fails, this method throws an
         /// <see cref="InputMismatchException">InputMismatchException</see>
-        /// .
-        /// <p/>
-        /// <strong>EXTRA TOKEN</strong> (single token deletion)
-        /// <p/>
+        /// .</p>
+        /// <p><strong>EXTRA TOKEN</strong> (single token deletion)</p>
+        /// <p>
         /// <code>LA(1)</code>
         /// is not what we are looking for. If
         /// <code>LA(2)</code>
@@ -513,15 +495,12 @@ namespace Antlr4.Runtime
         /// token and delete it. Then consume and return the next token (which was
         /// the
         /// <code>LA(2)</code>
-        /// token) as the successful result of the match operation.
-        /// <p/>
-        /// This recovery strategy is implemented by
+        /// token) as the successful result of the match operation.</p>
+        /// <p>This recovery strategy is implemented by
         /// <see cref="SingleTokenDeletion(Parser)">SingleTokenDeletion(Parser)</see>
-        /// .
-        /// <p/>
-        /// <strong>MISSING TOKEN</strong> (single token insertion)
-        /// <p/>
-        /// If current token (at
+        /// .</p>
+        /// <p><strong>MISSING TOKEN</strong> (single token insertion)</p>
+        /// <p>If current token (at
         /// <code>LA(1)</code>
         /// ) is consistent with what could come
         /// after the expected
@@ -531,15 +510,12 @@ namespace Antlr4.Runtime
         /// <see cref="ITokenFactory">ITokenFactory</see>
         /// to create it on the fly. The
         /// "insertion" is performed by returning the created token as the successful
-        /// result of the match operation.
-        /// <p/>
-        /// This recovery strategy is implemented by
+        /// result of the match operation.</p>
+        /// <p>This recovery strategy is implemented by
         /// <see cref="SingleTokenInsertion(Parser)">SingleTokenInsertion(Parser)</see>
-        /// .
-        /// <p/>
-        /// <strong>EXAMPLE</strong>
-        /// <p/>
-        /// For example, Input
+        /// .</p>
+        /// <p><strong>EXAMPLE</strong></p>
+        /// <p>For example, Input
         /// <code>i=(3;</code>
         /// is clearly missing the
         /// <code>')'</code>
@@ -547,9 +523,9 @@ namespace Antlr4.Runtime
         /// the parser returns from the nested call to
         /// <code>expr</code>
         /// , it will have
-        /// call chain:
+        /// call chain:</p>
         /// <pre>
-        /// stat -&gt; expr -&gt; atom
+        /// stat &rarr; expr &rarr; atom
         /// </pre>
         /// and it will be trying to match the
         /// <code>')'</code>
@@ -614,8 +590,7 @@ namespace Antlr4.Runtime
         /// <code>recognizer</code>
         /// will be in error recovery
         /// mode.
-        /// <p/>
-        /// This method determines whether or not single-token insertion is viable by
+        /// <p>This method determines whether or not single-token insertion is viable by
         /// checking if the
         /// <code>LA(1)</code>
         /// input symbol could be successfully matched
@@ -624,7 +599,7 @@ namespace Antlr4.Runtime
         /// symbol. If this method returns
         /// <code>true</code>
         /// , the caller is responsible for creating and inserting a
-        /// token with the correct type to produce this behavior.
+        /// token with the correct type to produce this behavior.</p>
         /// </remarks>
         /// <param name="recognizer">the parser instance</param>
         /// <returns>
@@ -667,8 +642,7 @@ namespace Antlr4.Runtime
         /// <code>recognizer</code>
         /// will <em>not</em> be in error recovery mode since the
         /// returned token was a successful match.
-        /// <p/>
-        /// If the single-token deletion is successful, this method calls
+        /// <p>If the single-token deletion is successful, this method calls
         /// <see cref="ReportUnwantedToken(Parser)">ReportUnwantedToken(Parser)</see>
         /// to report the error, followed by
         /// <see cref="Parser.Consume()">Parser.Consume()</see>
@@ -676,7 +650,7 @@ namespace Antlr4.Runtime
         /// before returning
         /// <see cref="ReportMatch(Parser)">ReportMatch(Parser)</see>
         /// is called to signal a successful
-        /// match.
+        /// match.</p>
         /// </remarks>
         /// <param name="recognizer">the parser instance</param>
         /// <returns>

runtime/CSharp/Antlr4.Runtime/Dfa/DFA.cs

@@ -27,6 +27,7 @@
  *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
+using System;
 using System.Collections.Concurrent;
 using Antlr4.Runtime.Atn;
 using Antlr4.Runtime.Dfa;
@@ -63,6 +64,18 @@ namespace Antlr4.Runtime.Dfa
 
         private readonly AtomicInteger nextStateNumber = new AtomicInteger();
 
+        /// <summary>
+        /// <code>true</code>
+        /// if this DFA is for a precedence decision; otherwise,
+        /// <code>false</code>
+        /// . This is the backing field for
+        /// <see cref="IsPrecedenceDfa()">IsPrecedenceDfa()</see>
+        /// ,
+        /// <see cref="SetPrecedenceDfa(bool)">SetPrecedenceDfa(bool)</see>
+        /// .
+        /// </summary>
+        private volatile bool precedenceDfa;
+
         public DFA(ATNState atnStartState)
             : this(atnStartState, 0)
         {
@@ -74,13 +87,171 @@ namespace Antlr4.Runtime.Dfa
             this.decision = decision;
         }
 
+        /// <summary>Gets whether this DFA is a precedence DFA.</summary>
+        /// <remarks>
+        /// Gets whether this DFA is a precedence DFA. Precedence DFAs use a special
+        /// start state
+        /// <see cref="s0">s0</see>
+        /// which is not stored in
+        /// <see cref="states">states</see>
+        /// . The
+        /// <see cref="DFAState#edges">DFAState#edges</see>
+        /// array for this start state contains outgoing edges
+        /// supplying individual start states corresponding to specific precedence
+        /// values.
+        /// </remarks>
+        /// <returns>
+        /// 
+        /// <code>true</code>
+        /// if this is a precedence DFA; otherwise,
+        /// <code>false</code>
+        /// .
+        /// </returns>
+        /// <seealso cref="Antlr4.Runtime.Parser.GetPrecedence()">Antlr4.Runtime.Parser.GetPrecedence()</seealso>
+        public bool IsPrecedenceDfa()
+        {
+            return precedenceDfa;
+        }
+
+        /// <summary>Get the start state for a specific precedence value.</summary>
+        /// <remarks>Get the start state for a specific precedence value.</remarks>
+        /// <param name="precedence">The current precedence.</param>
+        /// <returns>
+        /// The start state corresponding to the specified precedence, or
+        /// <code>null</code>
+        /// if no start state exists for the specified precedence.
+        /// </returns>
+        /// <exception cref="System.InvalidOperationException">if this is not a precedence DFA.</exception>
+        /// <seealso cref="IsPrecedenceDfa()">IsPrecedenceDfa()</seealso>
+        public DFAState GetPrecedenceStartState(int precedence, bool fullContext)
+        {
+            if (!IsPrecedenceDfa())
+            {
+                throw new InvalidOperationException("Only precedence DFAs may contain a precedence start state.");
+            }
+            // s0.get() and s0full.get() are never null for a precedence DFA
+            if (fullContext)
+            {
+                return s0full.Get().GetTarget(precedence);
+            }
+            else
+            {
+                return s0.Get().GetTarget(precedence);
+            }
+        }
+
+        /// <summary>Set the start state for a specific precedence value.</summary>
+        /// <remarks>Set the start state for a specific precedence value.</remarks>
+        /// <param name="precedence">The current precedence.</param>
+        /// <param name="startState">
+        /// The start state corresponding to the specified
+        /// precedence.
+        /// </param>
+        /// <exception cref="System.InvalidOperationException">if this is not a precedence DFA.</exception>
+        /// <seealso cref="IsPrecedenceDfa()">IsPrecedenceDfa()</seealso>
+        public void SetPrecedenceStartState(int precedence, bool fullContext, DFAState startState)
+        {
+            if (!IsPrecedenceDfa())
+            {
+                throw new InvalidOperationException("Only precedence DFAs may contain a precedence start state.");
+            }
+            if (precedence < 0)
+            {
+                return;
+            }
+            if (fullContext)
+            {
+                lock (s0full)
+                {
+                    // s0full.get() is never null for a precedence DFA
+                    s0full.Get().SetTarget(precedence, startState);
+                }
+            }
+            else
+            {
+                lock (s0)
+                {
+                    // s0.get() is never null for a precedence DFA
+                    s0.Get().SetTarget(precedence, startState);
+                }
+            }
+        }
+
+        /// <summary>Sets whether this is a precedence DFA.</summary>
+        /// <remarks>
+        /// Sets whether this is a precedence DFA. If the specified value differs
+        /// from the current DFA configuration, the following actions are taken;
+        /// otherwise no changes are made to the current DFA.
+        /// <ul>
+        /// <li>The
+        /// <see cref="states">states</see>
+        /// map is cleared</li>
+        /// <li>If
+        /// <code>precedenceDfa</code>
+        /// is
+        /// <code>false</code>
+        /// , the initial state
+        /// <see cref="s0">s0</see>
+        /// is set to
+        /// <code>null</code>
+        /// ; otherwise, it is initialized to a new
+        /// <see cref="DFAState">DFAState</see>
+        /// with an empty outgoing
+        /// <see cref="DFAState#edges">DFAState#edges</see>
+        /// array to
+        /// store the start states for individual precedence values.</li>
+        /// <li>The
+        /// <see cref="precedenceDfa">precedenceDfa</see>
+        /// field is updated</li>
+        /// </ul>
+        /// </remarks>
+        /// <param name="precedenceDfa">
+        /// 
+        /// <code>true</code>
+        /// if this is a precedence DFA; otherwise,
+        /// <code>false</code>
+        /// </param>
+        public void SetPrecedenceDfa(bool precedenceDfa)
+        {
+            lock (this)
+            {
+                if (this.precedenceDfa != precedenceDfa)
+                {
+                    this.states.Clear();
+                    if (precedenceDfa)
+                    {
+                        DFAState precedenceState = new DFAState(new ATNConfigSet(), 0, 200);
+                        precedenceState.isAcceptState = false;
+                        this.s0.Set(precedenceState);
+                        DFAState fullContextPrecedenceState = new DFAState(new ATNConfigSet(), 0, 200);
+                        fullContextPrecedenceState.isAcceptState = false;
+                        this.s0full.Set(fullContextPrecedenceState);
+                    }
+                    else
+                    {
+                        this.s0.Set(null);
+                        this.s0full.Set(null);
+                    }
+                    this.precedenceDfa = precedenceDfa;
+                }
+            }
+        }
+
         public virtual bool IsEmpty()
         {
+            if (IsPrecedenceDfa())
+            {
+                return s0.Get().EdgeMap.IsEmpty() && s0full.Get().EdgeMap.IsEmpty();
+            }
             return s0.Get() == null && s0full.Get() == null;
         }
 
         public virtual bool IsContextSensitive()
         {
+            if (IsPrecedenceDfa())
+            {
+                return s0full.Get().EdgeMap.IsEmpty();
+            }
             return s0full.Get() != null;
         }
 

runtime/CSharp/Antlr4.Runtime/Dfa/DFASerializer.cs

@@ -122,6 +122,10 @@ namespace Antlr4.Runtime.Dfa
                 }
             }
             string output = buf.ToString();
+            if (output.Length == 0)
+            {
+                return null;
+            }
             //return Utils.sortLinesInString(output);
             return output;
         }

runtime/CSharp/Antlr4.Runtime/Dfa/DFAState.cs

@@ -46,7 +46,7 @@ namespace Antlr4.Runtime.Dfa
     /// input a1a2..an, the DFA is in a state that represents the
     /// subset T of the states of the ATN that are reachable from the
     /// ATN's start state along some path labeled a1a2..an."
-    /// In conventional NFA->DFA conversion, therefore, the subset T
+    /// In conventional NFA→DFA conversion, therefore, the subset T
     /// would be a bitset representing the set of states the
     /// ATN could be in.  We need to track the alt predicted by each
     /// state as well, however.  More importantly, we need to maintain
@@ -54,14 +54,12 @@ namespace Antlr4.Runtime.Dfa
     /// jump from rule to rule, emulating rule invocations (method calls).
     /// I have to add a stack to simulate the proper lookahead sequences for
     /// the underlying LL grammar from which the ATN was derived.
-    /// <p/>
-    /// I use a set of ATNConfig objects not simple states.  An ATNConfig
+    /// <p>I use a set of ATNConfig objects not simple states.  An ATNConfig
     /// is both a state (ala normal conversion) and a RuleContext describing
-    /// the chain of rules (if any) followed to arrive at that state.
-    /// <p/>
-    /// A DFA state may have multiple references to a particular state,
+    /// the chain of rules (if any) followed to arrive at that state.</p>
+    /// <p>A DFA state may have multiple references to a particular state,
     /// but with different ATN contexts (with same or different alts)
-    /// meaning that state was reached via a different set of rule invocations.
+    /// meaning that state was reached via a different set of rule invocations.</p>
     /// </remarks>
     public class DFAState
     {
@@ -94,9 +92,7 @@ namespace Antlr4.Runtime.Dfa
         /// </summary>
         public int prediction;
 
-        public int lexerRuleIndex = -1;
-
-        public int lexerActionIndex = -1;
+        public LexerActionExecutor lexerActionExecutor;
 
         /// <summary>These keys for these edges are the top level element of the global context.</summary>
         /// <remarks>These keys for these edges are the top level element of the global context.</remarks>
@@ -127,8 +123,6 @@ namespace Antlr4.Runtime.Dfa
 
             public PredPrediction(SemanticContext pred, int alt)
             {
-                // if accept, exec action in what rule?
-                // if accept, exec what action?
                 // never null; at least SemanticContext.NONE
                 this.alt = alt;
                 this.pred = pred;
@@ -302,17 +296,15 @@ namespace Antlr4.Runtime.Dfa
         /// <see cref="DFAState">DFAState</see>
         /// instances are equal if their ATN configuration sets
         /// are the same. This method is used to see if a state already exists.
-        /// <p/>
-        /// Because the number of alternatives and number of ATN configurations are
+        /// <p>Because the number of alternatives and number of ATN configurations are
         /// finite, there is a finite number of DFA states that can be processed.
-        /// This is necessary to show that the algorithm terminates.
-        /// <p/>
-        /// Cannot test the DFA state numbers here because in
+        /// This is necessary to show that the algorithm terminates.</p>
+        /// <p>Cannot test the DFA state numbers here because in
         /// <see cref="Antlr4.Runtime.Atn.ParserATNSimulator.AddDFAState(DFA, Antlr4.Runtime.Atn.ATNConfigSet, Antlr4.Runtime.Atn.PredictionContextCache)">Antlr4.Runtime.Atn.ParserATNSimulator.AddDFAState(DFA, Antlr4.Runtime.Atn.ATNConfigSet, Antlr4.Runtime.Atn.PredictionContextCache)</see>
         /// we need to know if any other state
         /// exists that has this exact set of ATN configurations. The
         /// <see cref="stateNumber">stateNumber</see>
-        /// is irrelevant.
+        /// is irrelevant.</p>
         /// </summary>
         public override bool Equals(object o)
         {

runtime/CSharp/Antlr4.Runtime/IAntlrErrorListener`1.cs

@@ -44,13 +44,12 @@ namespace Antlr4.Runtime
         /// specifies how to recover from syntax errors and how to compute error
         /// messages. This listener's job is simply to emit a computed message,
         /// though it has enough information to create its own message in many cases.
-        /// <p/>
-        /// The
+        /// <p>The
         /// <see cref="RecognitionException">RecognitionException</see>
         /// is non-null for all syntax errors except
         /// when we discover mismatched token errors that we can recover from
         /// in-line, without returning from the surrounding rule (via the single
-        /// token insertion and deletion mechanism).
+        /// token insertion and deletion mechanism).</p>
         /// </remarks>
         /// <param name="recognizer">
         /// What parser got the error. From this

runtime/CSharp/Antlr4.Runtime/IAntlrErrorStrategy.cs

@@ -50,8 +50,7 @@ namespace Antlr4.Runtime
     /// Implementations of this interface report syntax errors by calling
     /// <see cref="Parser.NotifyErrorListeners(string)">Parser.NotifyErrorListeners(string)</see>
     /// .
-    /// <p/>
-    /// TODO: what to do about lexers
+    /// <p>TODO: what to do about lexers</p>
     /// </remarks>
     public interface IAntlrErrorStrategy
     {
@@ -73,12 +72,11 @@ namespace Antlr4.Runtime
         /// <see cref="IToken">IToken</see>
         /// instance which should be treated as the
         /// successful result of the match.
-        /// <p/>
-        /// Note that the calling code will not report an error if this method
+        /// <p>Note that the calling code will not report an error if this method
         /// returns successfully. The error strategy implementation is responsible
         /// for calling
         /// <see cref="Parser.NotifyErrorListeners(string)">Parser.NotifyErrorListeners(string)</see>
-        /// as appropriate.
+        /// as appropriate.</p>
         /// </summary>
         /// <param name="recognizer">the parser instance</param>
         /// <exception cref="RecognitionException">
@@ -113,19 +111,17 @@ namespace Antlr4.Runtime
         /// syntactic or semantic errors in the input stream before they result in a
         /// <see cref="RecognitionException">RecognitionException</see>
         /// .
-        /// <p/>
-        /// The generated code currently contains calls to
+        /// <p>The generated code currently contains calls to
         /// <see cref="Sync(Parser)">Sync(Parser)</see>
         /// after
         /// entering the decision state of a closure block (
         /// <code>(...)*</code>
         /// or
         /// <code>(...)+</code>
-        /// ).
-        /// <p/>
-        /// For an implementation based on Jim Idle's "magic sync" mechanism, see
+        /// ).</p>
+        /// <p>For an implementation based on Jim Idle's "magic sync" mechanism, see
         /// <see cref="DefaultErrorStrategy.Sync(Parser)">DefaultErrorStrategy.Sync(Parser)</see>
-        /// .
+        /// .</p>
         /// </summary>
         /// <seealso cref="DefaultErrorStrategy.Sync(Parser)">DefaultErrorStrategy.Sync(Parser)</seealso>
         /// <param name="recognizer">the parser instance</param>

runtime/CSharp/Antlr4.Runtime/IIntStream.cs

@@ -40,10 +40,9 @@ namespace Antlr4.Runtime
     /// For more information on marked ranges, see
     /// <see cref="Mark()">Mark()</see>
     /// .
-    /// <p/>
-    /// <strong>Initializing Methods:</strong> Some methods in this interface have
+    /// <p><strong>Initializing Methods:</strong> Some methods in this interface have
     /// unspecified behavior if no call to an initializing method has occurred after
-    /// the stream was constructed. The following is a list of initializing methods:
+    /// the stream was constructed. The following is a list of initializing methods:</p>
     /// <ul>
     /// <li>
     /// <see cref="La(int)">La(int)</see>
@@ -110,8 +109,7 @@ namespace Antlr4.Runtime
         /// <code>i==0</code>
         /// , but the specific behavior is unspecified because this
         /// method is frequently called from performance-critical code.
-        /// <p/>
-        /// This method is guaranteed to succeed if any of the following are true:
+        /// <p>This method is guaranteed to succeed if any of the following are true:</p>
         /// <ul>
         /// <li>
         /// <code>i&gt;0</code>
@@ -138,21 +136,20 @@ namespace Antlr4.Runtime
         /// refers to a symbol consumed within a marked region
         /// that has not yet been released.</li>
         /// </ul>
-        /// If
+        /// <p>If
         /// <code>i</code>
         /// represents a position at or beyond the end of the stream,
         /// this method returns
         /// <see cref="Eof">Eof</see>
-        /// .
-        /// <p/>
-        /// The return value is unspecified if
+        /// .</p>
+        /// <p>The return value is unspecified if
         /// <code>i&lt;0</code>
         /// and fewer than
         /// <code>-i</code>
         /// calls to
         /// <see cref="Consume()">consume()</see>
         /// have occurred from the beginning of
-        /// the stream before calling this method.
+        /// the stream before calling this method.</p>
         /// </summary>
         /// <exception cref="System.NotSupportedException">
         /// if the stream does not support
@@ -171,8 +168,7 @@ namespace Antlr4.Runtime
         /// . This allows the use of
         /// streaming input sources by specifying the minimum buffering requirements
         /// to support arbitrary lookahead during prediction.
-        /// <p/>
-        /// The returned mark is an opaque handle (type
+        /// <p>The returned mark is an opaque handle (type
         /// <code>int</code>
         /// ) which is passed
         /// to
@@ -187,16 +183,13 @@ namespace Antlr4.Runtime
         /// used during performance-critical sections of prediction, the specific
         /// behavior of invalid usage is unspecified (i.e. a mark is not released, or
         /// a mark is released twice, or marks are not released in reverse order from
-        /// which they were created).
-        /// <p/>
-        /// The behavior of this method is unspecified if no call to an
+        /// which they were created).</p>
+        /// <p>The behavior of this method is unspecified if no call to an
         /// <see cref="IIntStream">initializing method</see>
         /// has occurred after this stream was
-        /// constructed.
-        /// <p/>
-        /// This method does not change the current position in the input stream.
-        /// <p/>
-        /// The following example shows the use of
+        /// constructed.</p>
+        /// <p>This method does not change the current position in the input stream.</p>
+        /// <p>The following example shows the use of
         /// <see cref="Mark()">mark()</see>
         /// ,
         /// <see cref="Release(int)">release(mark)</see>
@@ -206,7 +199,7 @@ namespace Antlr4.Runtime
         /// <see cref="Seek(int)">seek(index)</see>
         /// as part of an operation to safely work within a
         /// marked region, then restore the stream position to its original value and
-        /// release the mark.
+        /// release the mark.</p>
         /// <pre>
         /// IntStream stream = ...;
         /// int index = -1;
@@ -242,10 +235,9 @@ namespace Antlr4.Runtime
         /// corresponding calls to
         /// <code>mark()</code>
         /// , the behavior is unspecified.
-        /// <p/>
-        /// For more information and an example, see
+        /// <p>For more information and an example, see
         /// <see cref="Mark()">Mark()</see>
-        /// .
+        /// .</p>
         /// </summary>
         /// <param name="marker">
         /// A marker returned by a call to
@@ -259,11 +251,10 @@ namespace Antlr4.Runtime
         /// Return the index into the stream of the input symbol referred to by
         /// <code>LA(1)</code>
         /// .
-        /// <p/>
-        /// The behavior of this method is unspecified if no call to an
+        /// <p>The behavior of this method is unspecified if no call to an
         /// <see cref="IIntStream">initializing method</see>
         /// has occurred after this stream was
-        /// constructed.
+        /// constructed.</p>
         /// </summary>
         int Index
         {

runtime/CSharp/Antlr4.Runtime/ITokenSource.cs

@@ -28,79 +28,135 @@
  *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 using Antlr4.Runtime;
+using Antlr4.Runtime.Misc;
 using Sharpen;
 
 namespace Antlr4.Runtime
 {
     /// <summary>
-    /// A source of tokens must provide a sequence of tokens via nextToken()
-    /// and also must reveal it's source of characters; CommonToken's text is
-    /// computed from a CharStream; it only store indices into the char stream.
+    /// A source of tokens must provide a sequence of tokens via
+    /// <see cref="NextToken()">NextToken()</see>
+    /// and also must reveal it's source of characters;
+    /// <see cref="CommonToken">CommonToken</see>
+    /// 's text is
+    /// computed from a
+    /// <see cref="ICharStream">ICharStream</see>
+    /// ; it only store indices into the char
+    /// stream.
+    /// <p>Errors from the lexer are never passed to the parser. Either you want to keep
+    /// going or you do not upon token recognition error. If you do not want to
+    /// continue lexing then you do not want to continue parsing. Just throw an
+    /// exception not under
+    /// <see cref="RecognitionException">RecognitionException</see>
+    /// and Java will naturally toss
+    /// you all the way out of the recognizers. If you want to continue lexing then
+    /// you should not throw an exception to the parser--it has already requested a
+    /// token. Keep lexing until you get a valid one. Just report errors and keep
+    /// going, looking for a valid token.</p>
     /// </summary>
-    /// <remarks>
-    /// A source of tokens must provide a sequence of tokens via nextToken()
-    /// and also must reveal it's source of characters; CommonToken's text is
-    /// computed from a CharStream; it only store indices into the char stream.
-    /// Errors from the lexer are never passed to the parser.  Either you want
-    /// to keep going or you do not upon token recognition error.  If you do not
-    /// want to continue lexing then you do not want to continue parsing.  Just
-    /// throw an exception not under RecognitionException and Java will naturally
-    /// toss you all the way out of the recognizers.  If you want to continue
-    /// lexing then you should not throw an exception to the parser--it has already
-    /// requested a token.  Keep lexing until you get a valid one.  Just report
-    /// errors and keep going, looking for a valid token.
-    /// </remarks>
     public interface ITokenSource
     {
-        /// <summary>Return a Token object from your input stream (usually a CharStream).</summary>
-        /// <remarks>
-        /// Return a Token object from your input stream (usually a CharStream).
-        /// Do not fail/return upon lexing error; keep chewing on the characters
-        /// until you get a good one; errors are not passed through to the parser.
-        /// </remarks>
+        /// <summary>
+        /// Return a
+        /// <see cref="IToken">IToken</see>
+        /// object from your input stream (usually a
+        /// <see cref="ICharStream">ICharStream</see>
+        /// ). Do not fail/return upon lexing error; keep chewing
+        /// on the characters until you get a good one; errors are not passed through
+        /// to the parser.
+        /// </summary>
+        [NotNull]
         IToken NextToken();
 
+        /// <summary>Get the line number for the current position in the input stream.</summary>
+        /// <remarks>
+        /// Get the line number for the current position in the input stream. The
+        /// first line in the input is line 1.
+        /// </remarks>
+        /// <returns>
+        /// The line number for the current position in the input stream, or
+        /// 0 if the current token source does not track line numbers.
+        /// </returns>
         int Line
         {
             get;
         }
 
+        /// <summary>
+        /// Get the index into the current line for the current position in the input
+        /// stream.
+        /// </summary>
+        /// <remarks>
+        /// Get the index into the current line for the current position in the input
+        /// stream. The first character on a line has position 0.
+        /// </remarks>
+        /// <returns>
+        /// The line number for the current position in the input stream, or
+        /// -1 if the current token source does not track character positions.
+        /// </returns>
         int Column
         {
             get;
         }
 
         /// <summary>
-        /// From what character stream was this token created?  You don't have to
-        /// implement but it's nice to know where a Token comes from if you have
-        /// include files etc...
+        /// Get the
+        /// <see cref="ICharStream">ICharStream</see>
+        /// from which this token source is currently
+        /// providing tokens.
         /// </summary>
-        /// <remarks>
-        /// From what character stream was this token created?  You don't have to
-        /// implement but it's nice to know where a Token comes from if you have
-        /// include files etc... on the input.
-        /// </remarks>
+        /// <returns>
+        /// The
+        /// <see cref="ICharStream">ICharStream</see>
+        /// associated with the current position in
+        /// the input, or
+        /// <code>null</code>
+        /// if no input stream is available for the token
+        /// source.
+        /// </returns>
         ICharStream InputStream
         {
             get;
         }
 
-        /// <summary>
-        /// Where are you getting tokens from? normally the implication will simply
-        /// ask lexers input stream.
-        /// </summary>
+        /// <summary>Gets the name of the underlying input source.</summary>
         /// <remarks>
-        /// Where are you getting tokens from? normally the implication will simply
-        /// ask lexers input stream.
+        /// Gets the name of the underlying input source. This method returns a
+        /// non-null, non-empty string. If such a name is not known, this method
+        /// returns
+        /// <see cref="IIntStream.UnknownSourceName">IIntStream.UnknownSourceName</see>
+        /// .
         /// </remarks>
         string SourceName
         {
             get;
         }
 
-        /// <summary>Gets the factory used for constructing tokens.</summary>
-        /// <remarks>Gets the factory used for constructing tokens.</remarks>
-        /// <summary>Optional method that lets users set factory in lexer or other source</summary>
+        /// <summary>
+        /// Set the
+        /// <see cref="ITokenFactory">ITokenFactory</see>
+        /// this token source should use for creating
+        /// <see cref="IToken">IToken</see>
+        /// objects from the input.
+        /// </summary>
+        /// <value>
+        /// The
+        /// <see cref="ITokenFactory">ITokenFactory</see>
+        /// to use for creating tokens.
+        /// </value>
+        /// <summary>
+        /// Gets the
+        /// <see cref="ITokenFactory">ITokenFactory</see>
+        /// this token source is currently using for
+        /// creating
+        /// <see cref="IToken">IToken</see>
+        /// objects from the input.
+        /// </summary>
+        /// <returns>
+        /// The
+        /// <see cref="ITokenFactory">ITokenFactory</see>
+        /// currently used by this token source.
+        /// </returns>
         ITokenFactory TokenFactory
         {
             get;

runtime/CSharp/Antlr4.Runtime/ITokenStream.cs

@@ -65,8 +65,7 @@ namespace Antlr4.Runtime
         /// <code>index</code>
         /// in the stream. When
         /// the preconditions of this method are met, the return value is non-null.
-        /// <p/>
-        /// The preconditions for this method are the same as the preconditions of
+        /// <p>The preconditions for this method are the same as the preconditions of
         /// <see cref="IIntStream.Seek(int)">IIntStream.Seek(int)</see>
         /// . If the behavior of
         /// <code>seek(index)</code>
@@ -74,9 +73,8 @@ namespace Antlr4.Runtime
         /// unspecified for the current state and given
         /// <code>index</code>
         /// , then the
-        /// behavior of this method is also unspecified.
-        /// <p/>
-        /// The symbol referred to by
+        /// behavior of this method is also unspecified.</p>
+        /// <p>The symbol referred to by
         /// <code>index</code>
         /// differs from
         /// <code>seek()</code>
@@ -88,7 +86,7 @@ namespace Antlr4.Runtime
         /// <code>seek()</code>
         /// , this method does not adjust
         /// <code>index</code>
-        /// to point to a non-ignored symbol.
+        /// to point to a non-ignored symbol.</p>
         /// </summary>
         /// <exception cref="System.ArgumentException">if {code index} is less than 0</exception>
         /// <exception cref="System.NotSupportedException">
@@ -172,11 +170,10 @@ namespace Antlr4.Runtime
         /// <see cref="GetText(Antlr4.Runtime.Misc.Interval)">GetText(Antlr4.Runtime.Misc.Interval)</see>
         /// , but may be
         /// optimized by the specific implementation.
-        /// </p>
-        /// If
+        /// <p>If
         /// <code>ctx.getSourceInterval()</code>
         /// does not return a valid interval of
-        /// tokens provided by this stream, the behavior is unspecified.
+        /// tokens provided by this stream, the behavior is unspecified.</p>
         /// <pre>
         /// TokenStream stream = ...;
         /// String text = stream.getText(ctx.getSourceInterval());
@@ -200,8 +197,7 @@ namespace Antlr4.Runtime
         /// and
         /// <code>stop</code>
         /// (inclusive).
-        /// <p/>
-        /// If the specified
+        /// <p>If the specified
         /// <code>start</code>
         /// or
         /// <code>stop</code>
@@ -210,14 +206,13 @@ namespace Antlr4.Runtime
         /// <code>stop</code>
         /// occurred before the
         /// <code>start</code>
-        /// token, the behavior is unspecified.
-        /// <p/>
-        /// For streams which ensure that the
+        /// token, the behavior is unspecified.</p>
+        /// <p>For streams which ensure that the
         /// <see cref="IToken.TokenIndex()">IToken.TokenIndex()</see>
         /// method is
         /// accurate for all of its provided tokens, this method behaves like the
         /// following code. Other streams may implement this method in other ways
-        /// provided the behavior is consistent with this at a high level.
+        /// provided the behavior is consistent with this at a high level.</p>
         /// <pre>
         /// TokenStream stream = ...;
         /// String text = "";

runtime/CSharp/Antlr4.Runtime/InterpreterRuleContext.cs

@@ -0,0 +1,80 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using Antlr4.Runtime;
+using Sharpen;
+
+namespace Antlr4.Runtime
+{
+    /// <summary>
+    /// This class extends
+    /// <see cref="ParserRuleContext">ParserRuleContext</see>
+    /// by allowing the value of
+    /// <see cref="GetRuleIndex()">GetRuleIndex()</see>
+    /// to be explicitly set for the context.
+    /// <p>
+    /// <see cref="ParserRuleContext">ParserRuleContext</see>
+    /// does not include field storage for the rule index
+    /// since the context classes created by the code generator override the
+    /// <see cref="GetRuleIndex()">GetRuleIndex()</see>
+    /// method to return the correct value for that context.
+    /// Since the parser interpreter does not use the context classes generated for a
+    /// parser, this class (with slightly more memory overhead per node) is used to
+    /// provide equivalent functionality.</p>
+    /// </summary>
+    public class InterpreterRuleContext : ParserRuleContext
+    {
+        /// <summary>
+        /// This is the backing field for
+        /// <see cref="GetRuleIndex()">GetRuleIndex()</see>
+        /// .
+        /// </summary>
+        private readonly int ruleIndex;
+
+        /// <summary>
+        /// Constructs a new
+        /// <see cref="InterpreterRuleContext">InterpreterRuleContext</see>
+        /// with the specified
+        /// parent, invoking state, and rule index.
+        /// </summary>
+        /// <param name="parent">The parent context.</param>
+        /// <param name="invokingStateNumber">The invoking state number.</param>
+        /// <param name="ruleIndex">The rule index for the current context.</param>
+        public InterpreterRuleContext(ParserRuleContext parent, int invokingStateNumber, int ruleIndex)
+            : base(parent, invokingStateNumber)
+        {
+            this.ruleIndex = ruleIndex;
+        }
+
+        public override int GetRuleIndex()
+        {
+            return ruleIndex;
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/LexerInterpreter.cs

@@ -0,0 +1,105 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using System;
+using System.Collections.Generic;
+using Antlr4.Runtime;
+using Antlr4.Runtime.Atn;
+using Sharpen;
+
+namespace Antlr4.Runtime
+{
+    public class LexerInterpreter : Lexer
+    {
+        protected internal readonly string grammarFileName;
+
+        protected internal readonly ATN atn;
+
+        protected internal readonly string[] tokenNames;
+
+        protected internal readonly string[] ruleNames;
+
+        protected internal readonly string[] modeNames;
+
+        public LexerInterpreter(string grammarFileName, ICollection<string> tokenNames, ICollection<string> ruleNames, ICollection<string> modeNames, ATN atn, ICharStream input)
+            : base(input)
+        {
+            if (atn.grammarType != ATNType.Lexer)
+            {
+                throw new ArgumentException("The ATN must be a lexer ATN.");
+            }
+            this.grammarFileName = grammarFileName;
+            this.atn = atn;
+            this.tokenNames = Sharpen.Collections.ToArray(tokenNames, new string[tokenNames.Count]);
+            this.ruleNames = Sharpen.Collections.ToArray(ruleNames, new string[ruleNames.Count]);
+            this.modeNames = Sharpen.Collections.ToArray(modeNames, new string[modeNames.Count]);
+            this._interp = new LexerATNSimulator(this, atn);
+        }
+
+        public override ATN Atn
+        {
+            get
+            {
+                return atn;
+            }
+        }
+
+        public override string GrammarFileName
+        {
+            get
+            {
+                return grammarFileName;
+            }
+        }
+
+        public override string[] TokenNames
+        {
+            get
+            {
+                return tokenNames;
+            }
+        }
+
+        public override string[] RuleNames
+        {
+            get
+            {
+                return ruleNames;
+            }
+        }
+
+        public override string[] ModeNames
+        {
+            get
+            {
+                return modeNames;
+            }
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/ListTokenSource.cs

@@ -0,0 +1,349 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using System;
+using System.Collections.Generic;
+using Antlr4.Runtime;
+using Sharpen;
+
+namespace Antlr4.Runtime
+{
+    /// <summary>
+    /// Provides an implementation of
+    /// <see cref="ITokenSource">ITokenSource</see>
+    /// as a wrapper around a list
+    /// of
+    /// <see cref="IToken">IToken</see>
+    /// objects.
+    /// <p>If the final token in the list is an
+    /// <see cref="IToken.Eof">IToken.Eof</see>
+    /// token, it will be used
+    /// as the EOF token for every call to
+    /// <see cref="NextToken()">NextToken()</see>
+    /// after the end of the
+    /// list is reached. Otherwise, an EOF token will be created.</p>
+    /// </summary>
+    public class ListTokenSource : ITokenSource
+    {
+        /// <summary>
+        /// The wrapped collection of
+        /// <see cref="IToken">IToken</see>
+        /// objects to return.
+        /// </summary>
+        protected internal readonly IList<IToken> tokens;
+
+        /// <summary>The name of the input source.</summary>
+        /// <remarks>
+        /// The name of the input source. If this value is
+        /// <code>null</code>
+        /// , a call to
+        /// <see cref="SourceName()">SourceName()</see>
+        /// should return the source name used to create the
+        /// the next token in
+        /// <see cref="tokens">tokens</see>
+        /// (or the previous token if the end of
+        /// the input has been reached).
+        /// </remarks>
+        private readonly string sourceName;
+
+        /// <summary>
+        /// The index into
+        /// <see cref="tokens">tokens</see>
+        /// of token to return by the next call to
+        /// <see cref="NextToken()">NextToken()</see>
+        /// . The end of the input is indicated by this value
+        /// being greater than or equal to the number of items in
+        /// <see cref="tokens">tokens</see>
+        /// .
+        /// </summary>
+        protected internal int i;
+
+        /// <summary>This field caches the EOF token for the token source.</summary>
+        /// <remarks>This field caches the EOF token for the token source.</remarks>
+        protected internal IToken eofToken;
+
+        /// <summary>
+        /// This is the backing field for
+        /// <see cref="TokenFactory()">TokenFactory()</see>
+        /// and
+        /// <see cref="setTokenFactory">setTokenFactory</see>
+        /// .
+        /// </summary>
+        private ITokenFactory _factory = CommonTokenFactory.Default;
+
+        /// <summary>
+        /// Constructs a new
+        /// <see cref="ListTokenSource">ListTokenSource</see>
+        /// instance from the specified
+        /// collection of
+        /// <see cref="IToken">IToken</see>
+        /// objects.
+        /// </summary>
+        /// <param name="tokens">
+        /// The collection of
+        /// <see cref="IToken">IToken</see>
+        /// objects to provide as a
+        /// <see cref="ITokenSource">ITokenSource</see>
+        /// .
+        /// </param>
+        /// <exception>
+        /// NullPointerException
+        /// if
+        /// <code>tokens</code>
+        /// is
+        /// <code>null</code>
+        /// </exception>
+        public ListTokenSource(IList<IToken> tokens)
+            : this(tokens, null)
+        {
+        }
+
+        /// <summary>
+        /// Constructs a new
+        /// <see cref="ListTokenSource">ListTokenSource</see>
+        /// instance from the specified
+        /// collection of
+        /// <see cref="IToken">IToken</see>
+        /// objects and source name.
+        /// </summary>
+        /// <param name="tokens">
+        /// The collection of
+        /// <see cref="IToken">IToken</see>
+        /// objects to provide as a
+        /// <see cref="ITokenSource">ITokenSource</see>
+        /// .
+        /// </param>
+        /// <param name="sourceName">
+        /// The name of the
+        /// <see cref="ITokenSource">ITokenSource</see>
+        /// . If this value is
+        /// <code>null</code>
+        /// ,
+        /// <see cref="SourceName()">SourceName()</see>
+        /// will attempt to infer the name from
+        /// the next
+        /// <see cref="IToken">IToken</see>
+        /// (or the previous token if the end of the input has
+        /// been reached).
+        /// </param>
+        /// <exception>
+        /// NullPointerException
+        /// if
+        /// <code>tokens</code>
+        /// is
+        /// <code>null</code>
+        /// </exception>
+        public ListTokenSource(IList<IToken> tokens, string sourceName)
+        {
+            if (tokens == null)
+            {
+                throw new ArgumentNullException("tokens cannot be null");
+            }
+            this.tokens = tokens;
+            this.sourceName = sourceName;
+        }
+
+        /// <summary><inheritDoc></inheritDoc></summary>
+        public virtual int Column
+        {
+            get
+            {
+                if (i < tokens.Count)
+                {
+                    return tokens[i].Column;
+                }
+                else
+                {
+                    if (eofToken != null)
+                    {
+                        return eofToken.Column;
+                    }
+                    else
+                    {
+                        if (tokens.Count > 0)
+                        {
+                            // have to calculate the result from the line/column of the previous
+                            // token, along with the text of the token.
+                            IToken lastToken = tokens[tokens.Count - 1];
+                            string tokenText = lastToken.Text;
+                            if (tokenText != null)
+                            {
+                                int lastNewLine = tokenText.LastIndexOf('\n');
+                                if (lastNewLine >= 0)
+                                {
+                                    return tokenText.Length - lastNewLine - 1;
+                                }
+                            }
+                            return lastToken.Column + lastToken.StopIndex - lastToken.StartIndex + 1;
+                        }
+                    }
+                }
+                // only reach this if tokens is empty, meaning EOF occurs at the first
+                // position in the input
+                return 0;
+            }
+        }
+
+        /// <summary><inheritDoc></inheritDoc></summary>
+        public virtual IToken NextToken()
+        {
+            if (i >= tokens.Count)
+            {
+                if (eofToken == null)
+                {
+                    int start = -1;
+                    if (tokens.Count > 0)
+                    {
+                        int previousStop = tokens[tokens.Count - 1].StopIndex;
+                        if (previousStop != -1)
+                        {
+                            start = previousStop + 1;
+                        }
+                    }
+                    int stop = Math.Max(-1, start - 1);
+                    eofToken = _factory.Create(Tuple.Create(this, InputStream), TokenConstants.Eof, "EOF", TokenConstants.DefaultChannel, start, stop, Line, Column);
+                }
+                return eofToken;
+            }
+            IToken t = tokens[i];
+            if (i == tokens.Count - 1 && t.Type == TokenConstants.Eof)
+            {
+                eofToken = t;
+            }
+            i++;
+            return t;
+        }
+
+        /// <summary><inheritDoc></inheritDoc></summary>
+        public virtual int Line
+        {
+            get
+            {
+                if (i < tokens.Count)
+                {
+                    return tokens[i].Line;
+                }
+                else
+                {
+                    if (eofToken != null)
+                    {
+                        return eofToken.Line;
+                    }
+                    else
+                    {
+                        if (tokens.Count > 0)
+                        {
+                            // have to calculate the result from the line/column of the previous
+                            // token, along with the text of the token.
+                            IToken lastToken = tokens[tokens.Count - 1];
+                            int line = lastToken.Line;
+                            string tokenText = lastToken.Text;
+                            if (tokenText != null)
+                            {
+                                for (int i = 0; i < tokenText.Length; i++)
+                                {
+                                    if (tokenText[i] == '\n')
+                                    {
+                                        line++;
+                                    }
+                                }
+                            }
+                            // if no text is available, assume the token did not contain any newline characters.
+                            return line;
+                        }
+                    }
+                }
+                // only reach this if tokens is empty, meaning EOF occurs at the first
+                // position in the input
+                return 1;
+            }
+        }
+
+        /// <summary><inheritDoc></inheritDoc></summary>
+        public virtual ICharStream InputStream
+        {
+            get
+            {
+                if (i < tokens.Count)
+                {
+                    return tokens[i].InputStream;
+                }
+                else
+                {
+                    if (eofToken != null)
+                    {
+                        return eofToken.InputStream;
+                    }
+                    else
+                    {
+                        if (tokens.Count > 0)
+                        {
+                            return tokens[tokens.Count - 1].InputStream;
+                        }
+                    }
+                }
+                // no input stream information is available
+                return null;
+            }
+        }
+
+        /// <summary><inheritDoc></inheritDoc></summary>
+        public virtual string SourceName
+        {
+            get
+            {
+                if (sourceName != null)
+                {
+                    return sourceName;
+                }
+                ICharStream inputStream = InputStream;
+                if (inputStream != null)
+                {
+                    return inputStream.SourceName;
+                }
+                return "List";
+            }
+        }
+
+        /// <summary><inheritDoc></inheritDoc></summary>
+        /// <summary><inheritDoc></inheritDoc></summary>
+        public virtual ITokenFactory TokenFactory
+        {
+            get
+            {
+                return _factory;
+            }
+            set
+            {
+                ITokenFactory factory = value;
+                this._factory = factory;
+            }
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/Misc/Interval.cs

@@ -117,9 +117,9 @@ namespace Antlr4.Runtime.Misc
 
         public override int GetHashCode()
         {
-            int hash = 5;
-            hash = 37 * hash ^ this.a;
-            hash = 37 * hash ^ this.b;
+            int hash = 23;
+            hash = hash * 31 + a;
+            hash = hash * 31 + b;
             return hash;
         }
 

runtime/CSharp/Antlr4.Runtime/Misc/IntervalSet.cs

@@ -304,9 +304,9 @@ namespace Antlr4.Runtime.Misc
             return compl;
         }
 
-        /// <summary>Compute this-other via this&~other.</summary>
+        /// <summary>Compute this-other via this&amp;~other.</summary>
         /// <remarks>
-        /// Compute this-other via this&~other.
+        /// Compute this-other via this&amp;~other.
         /// Return a new set containing all elements in this but not in other.
         /// other is assumed to be a subset of this;
         /// anything that is in other but not in this will be ignored.

runtime/CSharp/Antlr4.Runtime/Misc/RuleDependencyChecker.cs

@@ -447,7 +447,7 @@ namespace Antlr4.Runtime.Misc
             {
                 return null;
             }
-            ATN atn = ATNSimulator.Deserialize(serializedATN.ToCharArray());
+            ATN atn = new ATNDeserializer().Deserialize(serializedATN.ToCharArray());
             RuleDependencyChecker.RuleRelations relations = new RuleDependencyChecker.RuleRelations(atn.ruleToStartState.Length);
             foreach (ATNState state in atn.states)
             {

runtime/CSharp/Antlr4.Runtime/Misc/RuleDependencyProcessor.cs

@@ -642,7 +642,7 @@ namespace Antlr4.Runtime.Misc
             {
                 return null;
             }
-            ATN atn = ATNSimulator.Deserialize(serializedATN.ToCharArray());
+            ATN atn = new ATNDeserializer().Deserialize(serializedATN.ToCharArray());
             RuleDependencyProcessor.RuleRelations relations = new RuleDependencyProcessor.RuleRelations(atn.ruleToStartState.Length);
             foreach (ATNState state in atn.states)
             {

runtime/CSharp/Antlr4.Runtime/Misc/Utils.cs

@@ -124,5 +124,34 @@ namespace Antlr4.Runtime.Misc
                 list.SubList(j, list.Count).Clear();
             }
         }
+
+        /// <summary>Convert array of strings to string&rarr;index map.</summary>
+        /// <remarks>
+        /// Convert array of strings to string&rarr;index map. Useful for
+        /// converting rulenames to name&rarr;ruleindex map.
+        /// </remarks>
+        public static IDictionary<string, int> ToMap(string[] keys)
+        {
+            IDictionary<string, int> m = new Dictionary<string, int>();
+            for (int i = 0; i < keys.Length; i++)
+            {
+                m.Put(keys[i], i);
+            }
+            return m;
+        }
+
+        public static char[] ToCharArray(List<int> data)
+        {
+            if (data == null)
+            {
+                return null;
+            }
+            char[] cdata = new char[data.Size()];
+            for (int i = 0; i < data.Size(); i++)
+            {
+                cdata[i] = (char)data.Get(i);
+            }
+            return cdata;
+        }
     }
 }

runtime/CSharp/Antlr4.Runtime/Parser.cs

@@ -35,6 +35,7 @@ using Antlr4.Runtime.Atn;
 using Antlr4.Runtime.Dfa;
 using Antlr4.Runtime.Misc;
 using Antlr4.Runtime.Tree;
+using Antlr4.Runtime.Tree.Pattern;
 using Sharpen;
 
 namespace Antlr4.Runtime
@@ -99,6 +100,15 @@ namespace Antlr4.Runtime
             }
         }
 
+        /// <summary>
+        /// This field maps from the serialized ATN string to the deserialized
+        /// <see cref="Antlr4.Runtime.Atn.ATN">Antlr4.Runtime.Atn.ATN</see>
+        /// with
+        /// bypass alternatives.
+        /// </summary>
+        /// <seealso cref="Antlr4.Runtime.Atn.ATNDeserializationOptions.IsGenerateRuleBypassTransitions()">Antlr4.Runtime.Atn.ATNDeserializationOptions.IsGenerateRuleBypassTransitions()</seealso>
+        private static readonly IDictionary<string, ATN> bypassAltsAtnCache = new WeakHashMap<string, ATN>();
+
         /// <summary>The error handling strategy for the parser.</summary>
         /// <remarks>
         /// The error handling strategy for the parser. The default value is a new
@@ -215,8 +225,7 @@ namespace Antlr4.Runtime
         /// <see cref="Consume()">Consume()</see>
         /// are
         /// called to complete the match process.
-        /// <p/>
-        /// If the symbol type does not match,
+        /// <p>If the symbol type does not match,
         /// <see cref="IAntlrErrorStrategy.RecoverInline(Parser)">IAntlrErrorStrategy.RecoverInline(Parser)</see>
         /// is called on the current error
         /// strategy to attempt recovery. If
@@ -228,7 +237,7 @@ namespace Antlr4.Runtime
         /// is -1, the symbol is added to
         /// the parse tree by calling
         /// <see cref="ParserRuleContext.AddErrorNode(IToken)">ParserRuleContext.AddErrorNode(IToken)</see>
-        /// .
+        /// .</p>
         /// </summary>
         /// <param name="ttype">the token type to match</param>
         /// <returns>the matched symbol</returns>
@@ -269,8 +278,7 @@ namespace Antlr4.Runtime
         /// and
         /// <see cref="Consume()">Consume()</see>
         /// are called to complete the match process.
-        /// <p/>
-        /// If the symbol type does not match,
+        /// <p>If the symbol type does not match,
         /// <see cref="IAntlrErrorStrategy.RecoverInline(Parser)">IAntlrErrorStrategy.RecoverInline(Parser)</see>
         /// is called on the current error
         /// strategy to attempt recovery. If
@@ -282,7 +290,7 @@ namespace Antlr4.Runtime
         /// is -1, the symbol is added to
         /// the parse tree by calling
         /// <see cref="ParserRuleContext.AddErrorNode(IToken)">ParserRuleContext.AddErrorNode(IToken)</see>
-        /// .
+        /// .</p>
         /// </remarks>
         /// <returns>the matched symbol</returns>
         /// <exception cref="RecognitionException">
@@ -324,16 +332,14 @@ namespace Antlr4.Runtime
         /// <see cref="ParserRuleContext">ParserRuleContext</see>
         /// returned from the start
         /// rule represents the root of the parse tree.
-        /// <p/>
-        /// Note that if we are not building parse trees, rule contexts only point
+        /// <p>Note that if we are not building parse trees, rule contexts only point
         /// upwards. When a rule exits, it returns the context but that gets garbage
         /// collected if nobody holds a reference. It points upwards but nobody
-        /// points at it.
-        /// <p/>
-        /// When we build parse trees, we are adding all of these contexts to
+        /// points at it.</p>
+        /// <p>When we build parse trees, we are adding all of these contexts to
         /// <see cref="ParserRuleContext.children">ParserRuleContext.children</see>
         /// list. Contexts are then not candidates
-        /// for garbage collection.
+        /// for garbage collection.</p>
         /// </summary>
         /// <summary>
         /// Gets whether or not a complete parse tree will be constructed while
@@ -430,8 +436,7 @@ namespace Antlr4.Runtime
         /// Registers
         /// <code>listener</code>
         /// to receive events during the parsing process.
-        /// <p/>
-        /// To support output-preserving grammar transformations (including but not
+        /// <p>To support output-preserving grammar transformations (including but not
         /// limited to left-recursion removal, automated left-factoring, and
         /// optimized code generation), calls to listener methods during the parse
         /// may differ substantially from calls made by
@@ -439,11 +444,10 @@ namespace Antlr4.Runtime
         /// used after the parse is complete. In
         /// particular, rule entry and exit events may occur in a different order
         /// during the parse than after the parser. In addition, calls to certain
-        /// rule entry methods may be omitted.
-        /// <p/>
-        /// With the following specific exceptions, calls to listener events are
+        /// rule entry methods may be omitted.</p>
+        /// <p>With the following specific exceptions, calls to listener events are
         /// <em>deterministic</em>, i.e. for identical input the calls to listener
-        /// methods will be the same.
+        /// methods will be the same.</p>
         /// <ul>
         /// <li>Alterations to the grammar used to generate code may change the
         /// behavior of the listener calls.</li>
@@ -477,13 +481,12 @@ namespace Antlr4.Runtime
         /// Remove
         /// <code>listener</code>
         /// from the list of parse listeners.
-        /// <p/>
-        /// If
+        /// <p>If
         /// <code>listener</code>
         /// is
         /// <code>null</code>
         /// or has not been added as a parse
-        /// listener, this method does nothing.
+        /// listener, this method does nothing.</p>
         /// </summary>
         /// <seealso cref="AddParseListener(Antlr4.Runtime.Tree.IParseTreeListener)">AddParseListener(Antlr4.Runtime.Tree.IParseTreeListener)</seealso>
         /// <param name="listener">the listener to remove</param>
@@ -556,6 +559,80 @@ namespace Antlr4.Runtime
             return _input.TokenSource.TokenFactory;
         }
 
+        /// <summary>
+        /// The ATN with bypass alternatives is expensive to create so we create it
+        /// lazily.
+        /// </summary>
+        /// <remarks>
+        /// The ATN with bypass alternatives is expensive to create so we create it
+        /// lazily.
+        /// </remarks>
+        /// <exception cref="System.NotSupportedException">
+        /// if the current parser does not
+        /// implement the
+        /// <see cref="Recognizer{Symbol, ATNInterpreter}.GetSerializedATN()">Recognizer&lt;Symbol, ATNInterpreter&gt;.GetSerializedATN()</see>
+        /// method.
+        /// </exception>
+        [NotNull]
+        public virtual ATN GetATNWithBypassAlts()
+        {
+            string serializedAtn = GetSerializedATN();
+            if (serializedAtn == null)
+            {
+                throw new NotSupportedException("The current parser does not support an ATN with bypass alternatives.");
+            }
+            lock (bypassAltsAtnCache)
+            {
+                ATN result = bypassAltsAtnCache.Get(serializedAtn);
+                if (result == null)
+                {
+                    ATNDeserializationOptions deserializationOptions = new ATNDeserializationOptions();
+                    deserializationOptions.SetGenerateRuleBypassTransitions(true);
+                    result = new ATNDeserializer(deserializationOptions).Deserialize(serializedAtn.ToCharArray());
+                    bypassAltsAtnCache.Put(serializedAtn, result);
+                }
+                return result;
+            }
+        }
+
+        /// <summary>The preferred method of getting a tree pattern.</summary>
+        /// <remarks>
+        /// The preferred method of getting a tree pattern. For example, here's a
+        /// sample use:
+        /// <pre>
+        /// ParseTree t = parser.expr();
+        /// ParseTreePattern p = parser.compileParseTreePattern("&lt;ID&gt;+0", MyParser.RULE_expr);
+        /// ParseTreeMatch m = p.match(t);
+        /// String id = m.get("ID");
+        /// </pre>
+        /// </remarks>
+        public virtual ParseTreePattern CompileParseTreePattern(string pattern, int patternRuleIndex)
+        {
+            if (((ITokenStream)InputStream) != null)
+            {
+                ITokenSource tokenSource = ((ITokenStream)InputStream).TokenSource;
+                if (tokenSource is Lexer)
+                {
+                    Lexer lexer = (Lexer)tokenSource;
+                    return CompileParseTreePattern(pattern, patternRuleIndex, lexer);
+                }
+            }
+            throw new NotSupportedException("Parser can't discover a lexer to use");
+        }
+
+        /// <summary>
+        /// The same as
+        /// <see cref="CompileParseTreePattern(string, int)">CompileParseTreePattern(string, int)</see>
+        /// but specify a
+        /// <see cref="Lexer">Lexer</see>
+        /// rather than trying to deduce it from this parser.
+        /// </summary>
+        public virtual ParseTreePattern CompileParseTreePattern(string pattern, int patternRuleIndex, Lexer lexer)
+        {
+            ParseTreePatternMatcher m = new ParseTreePatternMatcher(lexer, this);
+            return m.Compile(pattern, patternRuleIndex);
+        }
+
         public virtual IAntlrErrorStrategy ErrorHandler
         {
             get
@@ -628,15 +705,14 @@ namespace Antlr4.Runtime
         /// current symbol
         /// </linkplain>
         /// .
-        /// <p/>
-        /// E.g., given the following input with
+        /// <p>E.g., given the following input with
         /// <code>A</code>
         /// being the current
         /// lookahead symbol, this function moves the cursor to
         /// <code>B</code>
         /// and returns
         /// <code>A</code>
-        /// .
+        /// .</p>
         /// <pre>
         /// A B
         /// ^
@@ -772,8 +848,31 @@ namespace Antlr4.Runtime
             _ctx = localctx;
         }
 
-        public virtual void EnterRecursionRule(ParserRuleContext localctx, int ruleIndex, int precedence)
+        /// <summary>Get the precedence level for the top-most precedence rule.</summary>
+        /// <remarks>Get the precedence level for the top-most precedence rule.</remarks>
+        /// <returns>
+        /// The precedence level for the top-most precedence rule, or -1 if
+        /// the parser context is not nested within a precedence rule.
+        /// </returns>
+        public int GetPrecedence()
         {
+            if (_precedenceStack.IsEmpty())
+            {
+                return -1;
+            }
+            return _precedenceStack.Peek();
+        }
+
+        [Obsolete]
+        [System.ObsoleteAttribute(@"UseEnterRecursionRule(ParserRuleContext, int, int, int) instead.")]
+        public virtual void EnterRecursionRule(ParserRuleContext localctx, int ruleIndex)
+        {
+            EnterRecursionRule(localctx, Atn.ruleToStartState[ruleIndex].stateNumber, ruleIndex, 0);
+        }
+
+        public virtual void EnterRecursionRule(ParserRuleContext localctx, int state, int ruleIndex, int precedence)
+        {
+            State = state;
             _precedenceStack.Add(precedence);
             _ctx = localctx;
             _ctx.start = _input.Lt(1);
@@ -858,6 +957,11 @@ namespace Antlr4.Runtime
             }
         }
 
+        public virtual void SetContext(ParserRuleContext ctx)
+        {
+            _ctx = ctx;
+        }
+
         public override bool Precpred(RuleContext localctx, int precedence)
         {
             return precedence >= _precedenceStack.Peek();
@@ -954,19 +1058,25 @@ namespace Antlr4.Runtime
             return atn.NextTokens(s);
         }
 
+        /// <summary>
+        /// Get a rule's index (i.e.,
+        /// <code>RULE_ruleName</code>
+        /// field) or -1 if not found.
+        /// </summary>
+        public virtual int GetRuleIndex(string ruleName)
+        {
+            int ruleIndex = GetRuleIndexMap().Get(ruleName);
+            if (ruleIndex != null)
+            {
+                return ruleIndex;
+            }
+            return -1;
+        }
+
         public virtual ParserRuleContext RuleContext
         {
             get
             {
-                //	/** Compute the set of valid tokens reachable from the current
-                //	 *  position in the parse.
-                //	 */
-                //	public IntervalSet nextTokens(@NotNull RuleContext ctx) {
-                //		ATN atn = getInterpreter().atn;
-                //		ATNState s = atn.states.get(ctx.s);
-                //		if ( s == null ) return null;
-                //		return atn.nextTokens(s, ctx);
-                //	}
                 return _ctx;
             }
         }

runtime/CSharp/Antlr4.Runtime/ParserInterpreter.cs

@@ -0,0 +1,299 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using System;
+using System.Collections.Generic;
+using Antlr4.Runtime;
+using Antlr4.Runtime.Atn;
+using Sharpen;
+
+namespace Antlr4.Runtime
+{
+    /// <summary>
+    /// A parser simulator that mimics what ANTLR's generated
+    /// parser code does.
+    /// </summary>
+    /// <remarks>
+    /// A parser simulator that mimics what ANTLR's generated
+    /// parser code does. A ParserATNSimulator is used to make
+    /// predictions via adaptivePredict but this class moves a pointer through the
+    /// ATN to simulate parsing. ParserATNSimulator just
+    /// makes us efficient rather than having to backtrack, for example.
+    /// This properly creates parse trees even for left recursive rules.
+    /// We rely on the left recursive rule invocation and special predicate
+    /// transitions to make left recursive rules work.
+    /// See TestParserInterpreter for examples.
+    /// </remarks>
+    public class ParserInterpreter : Parser
+    {
+        protected internal readonly string grammarFileName;
+
+        protected internal readonly ATN atn;
+
+        protected internal readonly BitSet pushRecursionContextStates;
+
+        protected internal readonly string[] tokenNames;
+
+        protected internal readonly string[] ruleNames;
+
+        protected internal readonly IDeque<Tuple<ParserRuleContext, int>> _parentContextStack = new ArrayDeque<Tuple<ParserRuleContext, int>>();
+
+        public ParserInterpreter(string grammarFileName, ICollection<string> tokenNames, ICollection<string> ruleNames, ATN atn, ITokenStream input)
+            : base(input)
+        {
+            this.grammarFileName = grammarFileName;
+            this.atn = atn;
+            this.tokenNames = Sharpen.Collections.ToArray(tokenNames, new string[tokenNames.Count]);
+            this.ruleNames = Sharpen.Collections.ToArray(ruleNames, new string[ruleNames.Count]);
+            // identify the ATN states where pushNewRecursionContext must be called
+            this.pushRecursionContextStates = new BitSet(atn.states.Count);
+            foreach (ATNState state in atn.states)
+            {
+                if (!(state is StarLoopEntryState))
+                {
+                    continue;
+                }
+                if (((StarLoopEntryState)state).precedenceRuleDecision)
+                {
+                    this.pushRecursionContextStates.Set(state.stateNumber);
+                }
+            }
+            // get atn simulator that knows how to do predictions
+            Interpreter = new ParserATNSimulator(this, atn);
+        }
+
+        public override ATN Atn
+        {
+            get
+            {
+                return atn;
+            }
+        }
+
+        public override string[] TokenNames
+        {
+            get
+            {
+                return tokenNames;
+            }
+        }
+
+        public override string[] RuleNames
+        {
+            get
+            {
+                return ruleNames;
+            }
+        }
+
+        public override string GrammarFileName
+        {
+            get
+            {
+                return grammarFileName;
+            }
+        }
+
+        /// <summary>Begin parsing at startRuleIndex</summary>
+        public virtual ParserRuleContext Parse(int startRuleIndex)
+        {
+            RuleStartState startRuleStartState = atn.ruleToStartState[startRuleIndex];
+            InterpreterRuleContext rootContext = new InterpreterRuleContext(null, ATNState.InvalidStateNumber, startRuleIndex);
+            if (startRuleStartState.isPrecedenceRule)
+            {
+                EnterRecursionRule(rootContext, startRuleStartState.stateNumber, startRuleIndex, 0);
+            }
+            else
+            {
+                EnterRule(rootContext, startRuleStartState.stateNumber, startRuleIndex);
+            }
+            while (true)
+            {
+                ATNState p = GetATNState();
+                switch (p.StateType)
+                {
+                    case StateType.RuleStop:
+                    {
+                        // pop; return from rule
+                        if (_ctx.IsEmpty())
+                        {
+                            ExitRule();
+                            return rootContext;
+                        }
+                        VisitRuleStopState(p);
+                        break;
+                    }
+
+                    default:
+                    {
+                        try
+                        {
+                            VisitState(p);
+                        }
+                        catch (RecognitionException e)
+                        {
+                            State = atn.ruleToStopState[p.ruleIndex].stateNumber;
+                            Context.exception = e;
+                            ErrorHandler.ReportError(this, e);
+                            ErrorHandler.Recover(this, e);
+                        }
+                        break;
+                    }
+                }
+            }
+        }
+
+        public override void EnterRecursionRule(ParserRuleContext localctx, int state, int ruleIndex, int precedence)
+        {
+            _parentContextStack.Push(Tuple.Create(_ctx, localctx.invokingState));
+            base.EnterRecursionRule(localctx, state, ruleIndex, precedence);
+        }
+
+        protected internal virtual ATNState GetATNState()
+        {
+            return atn.states[State];
+        }
+
+        protected internal virtual void VisitState(ATNState p)
+        {
+            int edge;
+            if (p.NumberOfTransitions > 1)
+            {
+                ErrorHandler.Sync(this);
+                edge = Interpreter.AdaptivePredict(_input, ((DecisionState)p).decision, _ctx);
+            }
+            else
+            {
+                edge = 1;
+            }
+            Transition transition = p.Transition(edge - 1);
+            switch (transition.TransitionType)
+            {
+                case TransitionType.Epsilon:
+                {
+                    if (pushRecursionContextStates.Get(p.stateNumber) && !(transition.target is LoopEndState))
+                    {
+                        InterpreterRuleContext ctx = new InterpreterRuleContext(_parentContextStack.Peek().Item1, _parentContextStack.Peek().Item2, _ctx.GetRuleIndex());
+                        PushNewRecursionContext(ctx, atn.ruleToStartState[p.ruleIndex].stateNumber, _ctx.GetRuleIndex());
+                    }
+                    break;
+                }
+
+                case TransitionType.Atom:
+                {
+                    Match(((AtomTransition)transition).label);
+                    break;
+                }
+
+                case TransitionType.Range:
+                case TransitionType.Set:
+                case TransitionType.NotSet:
+                {
+                    if (!transition.Matches(_input.La(1), TokenConstants.MinUserTokenType, 65535))
+                    {
+                        _errHandler.RecoverInline(this);
+                    }
+                    MatchWildcard();
+                    break;
+                }
+
+                case TransitionType.Wildcard:
+                {
+                    MatchWildcard();
+                    break;
+                }
+
+                case TransitionType.Rule:
+                {
+                    RuleStartState ruleStartState = (RuleStartState)transition.target;
+                    int ruleIndex = ruleStartState.ruleIndex;
+                    InterpreterRuleContext ctx_1 = new InterpreterRuleContext(_ctx, p.stateNumber, ruleIndex);
+                    if (ruleStartState.isPrecedenceRule)
+                    {
+                        EnterRecursionRule(ctx_1, ruleStartState.stateNumber, ruleIndex, ((RuleTransition)transition).precedence);
+                    }
+                    else
+                    {
+                        EnterRule(ctx_1, transition.target.stateNumber, ruleIndex);
+                    }
+                    break;
+                }
+
+                case TransitionType.Predicate:
+                {
+                    PredicateTransition predicateTransition = (PredicateTransition)transition;
+                    if (!Sempred(_ctx, predicateTransition.ruleIndex, predicateTransition.predIndex))
+                    {
+                        throw new FailedPredicateException(this);
+                    }
+                    break;
+                }
+
+                case TransitionType.Action:
+                {
+                    ActionTransition actionTransition = (ActionTransition)transition;
+                    Action(_ctx, actionTransition.ruleIndex, actionTransition.actionIndex);
+                    break;
+                }
+
+                case TransitionType.Precedence:
+                {
+                    if (!Precpred(_ctx, ((PrecedencePredicateTransition)transition).precedence))
+                    {
+                        throw new FailedPredicateException(this, string.Format("precpred(_ctx, %d)", ((PrecedencePredicateTransition)transition).precedence));
+                    }
+                    break;
+                }
+
+                default:
+                {
+                    throw new NotSupportedException("Unrecognized ATN transition type.");
+                }
+            }
+            State = transition.target.stateNumber;
+        }
+
+        protected internal virtual void VisitRuleStopState(ATNState p)
+        {
+            RuleStartState ruleStartState = atn.ruleToStartState[p.ruleIndex];
+            if (ruleStartState.isPrecedenceRule)
+            {
+                Tuple<ParserRuleContext, int> parentContext = _parentContextStack.Pop();
+                UnrollRecursionContexts(parentContext.Item1);
+                State = parentContext.Item2;
+            }
+            else
+            {
+                ExitRule();
+            }
+            RuleTransition ruleTransition = (RuleTransition)atn.states[State].Transition(0);
+            State = ruleTransition.followState.stateNumber;
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/ParserRuleContext.cs

@@ -119,9 +119,9 @@ namespace Antlr4.Runtime
         /// </remarks>
         public IToken stop;
 
-        /// <summary>The exception which forced this rule to return.</summary>
+        /// <summary>The exception that forced this rule to return.</summary>
         /// <remarks>
-        /// The exception which forced this rule to return. If the rule successfully
+        /// The exception that forced this rule to return. If the rule successfully
         /// completed, this is
         /// <code>null</code>
         /// .

runtime/CSharp/Antlr4.Runtime/RecognitionException.cs

@@ -116,8 +116,7 @@ namespace Antlr4.Runtime
         /// <see cref="Antlr4.Runtime.Atn.DecisionState">Antlr4.Runtime.Atn.DecisionState</see>
         /// number. For others, it is the state whose outgoing
         /// edge we couldn't match.
-        /// <p/>
-        /// If the state number is not known, this method returns -1.
+        /// <p>If the state number is not known, this method returns -1.</p>
         /// </remarks>
         protected internal int OffendingState
         {
@@ -139,11 +138,10 @@ namespace Antlr4.Runtime
         /// <remarks>
         /// Gets the set of input symbols which could potentially follow the
         /// previously matched symbol at the time this exception was thrown.
-        /// <p/>
-        /// If the set of expected tokens is not known and could not be computed,
+        /// <p>If the set of expected tokens is not known and could not be computed,
         /// this method returns
         /// <code>null</code>
-        /// .
+        /// .</p>
         /// </remarks>
         /// <returns>
         /// The set of token types that could potentially follow the current
@@ -165,10 +163,9 @@ namespace Antlr4.Runtime
         /// Gets the
         /// <see cref="RuleContext">RuleContext</see>
         /// at the time this exception was thrown.
-        /// <p/>
-        /// If the context is not available, this method returns
+        /// <p>If the context is not available, this method returns
         /// <code>null</code>
-        /// .
+        /// .</p>
         /// </summary>
         /// <returns>
         /// The
@@ -193,10 +190,9 @@ namespace Antlr4.Runtime
         /// <remarks>
         /// Gets the input stream which is the symbol source for the recognizer where
         /// this exception was thrown.
-        /// <p/>
-        /// If the input stream is not available, this method returns
+        /// <p>If the input stream is not available, this method returns
         /// <code>null</code>
-        /// .
+        /// .</p>
         /// </remarks>
         /// <returns>
         /// The input stream which is the symbol source for the recognizer
@@ -230,10 +226,9 @@ namespace Antlr4.Runtime
         /// Gets the
         /// <see cref="Recognizer{Symbol, ATNInterpreter}">Recognizer&lt;Symbol, ATNInterpreter&gt;</see>
         /// where this exception occurred.
-        /// <p/>
-        /// If the recognizer is not available, this method returns
+        /// <p>If the recognizer is not available, this method returns
         /// <code>null</code>
-        /// .
+        /// .</p>
         /// </summary>
         /// <returns>
         /// The recognizer where this exception occurred, or

runtime/CSharp/Antlr4.Runtime/Recognizer`2.cs

@@ -27,6 +27,7 @@
  *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
+using System;
 using System.Collections.Generic;
 using Antlr4.Runtime;
 using Antlr4.Runtime.Atn;
@@ -40,9 +41,13 @@ namespace Antlr4.Runtime
     {
         public const int Eof = -1;
 
-        private sealed class _CopyOnWriteArrayList_49 : CopyOnWriteArrayList<IAntlrErrorListener<Symbol>>
+        private static readonly IDictionary<string[], IDictionary<string, int>> tokenTypeMapCache = new WeakHashMap<string[], IDictionary<string, int>>();
+
+        private static readonly IDictionary<string[], IDictionary<string, int>> ruleIndexMapCache = new WeakHashMap<string[], IDictionary<string, int>>();
+
+        private sealed class _CopyOnWriteArrayList_58 : CopyOnWriteArrayList<IAntlrErrorListener<Symbol>>
         {
-            public _CopyOnWriteArrayList_49()
+            public _CopyOnWriteArrayList_58()
             {
                 {
                     this.AddItem(ConsoleErrorListener.Instance);
@@ -51,7 +56,7 @@ namespace Antlr4.Runtime
         }
 
         [NotNull]
-        private IList<IAntlrErrorListener<Symbol>> _listeners = new _CopyOnWriteArrayList_49();
+        private IList<IAntlrErrorListener<Symbol>> _listeners = new _CopyOnWriteArrayList_58();
 
         protected internal ATNInterpreter _interp;
 
@@ -76,6 +81,84 @@ namespace Antlr4.Runtime
             get;
         }
 
+        /// <summary>Get a map from token names to token types.</summary>
+        /// <remarks>
+        /// Get a map from token names to token types.
+        /// <p>Used for XPath and tree pattern compilation.</p>
+        /// </remarks>
+        [NotNull]
+        public virtual IDictionary<string, int> GetTokenTypeMap()
+        {
+            string[] tokenNames = TokenNames;
+            if (tokenNames == null)
+            {
+                throw new NotSupportedException("The current recognizer does not provide a list of token names.");
+            }
+            lock (tokenTypeMapCache)
+            {
+                IDictionary<string, int> result = tokenTypeMapCache.Get(tokenNames);
+                if (result == null)
+                {
+                    result = Utils.ToMap(tokenNames);
+                    result.Put("EOF", TokenConstants.Eof);
+                    result = Sharpen.Collections.UnmodifiableMap(result);
+                    tokenTypeMapCache.Put(tokenNames, result);
+                }
+                return result;
+            }
+        }
+
+        /// <summary>Get a map from rule names to rule indexes.</summary>
+        /// <remarks>
+        /// Get a map from rule names to rule indexes.
+        /// <p>Used for XPath and tree pattern compilation.</p>
+        /// </remarks>
+        [NotNull]
+        public virtual IDictionary<string, int> GetRuleIndexMap()
+        {
+            string[] ruleNames = RuleNames;
+            if (ruleNames == null)
+            {
+                throw new NotSupportedException("The current recognizer does not provide a list of rule names.");
+            }
+            lock (ruleIndexMapCache)
+            {
+                IDictionary<string, int> result = ruleIndexMapCache.Get(ruleNames);
+                if (result == null)
+                {
+                    result = Sharpen.Collections.UnmodifiableMap(Utils.ToMap(ruleNames));
+                    ruleIndexMapCache.Put(ruleNames, result);
+                }
+                return result;
+            }
+        }
+
+        public virtual int GetTokenType(string tokenName)
+        {
+            int ttype = GetTokenTypeMap().Get(tokenName);
+            if (ttype != null)
+            {
+                return ttype;
+            }
+            return TokenConstants.InvalidType;
+        }
+
+        /// <summary>
+        /// If this recognizer was generated, it will have a serialized ATN
+        /// representation of the grammar.
+        /// </summary>
+        /// <remarks>
+        /// If this recognizer was generated, it will have a serialized ATN
+        /// representation of the grammar.
+        /// <p>For interpreters, we don't know their serialized ATN despite having
+        /// created the interpreter from it.</p>
+        /// </remarks>
+        [NotNull]
+        public virtual string GetSerializedATN()
+        {
+            throw new NotSupportedException("there is no serialized ATN");
+        }
+
         /// <summary>For debugging and other purposes, might want the grammar name.</summary>
         /// <remarks>
         /// For debugging and other purposes, might want the grammar name.
@@ -86,6 +169,16 @@ namespace Antlr4.Runtime
             get;
         }
 
+        /// <summary>
+        /// Get the
+        /// <see cref="Antlr4.Runtime.Atn.ATN">Antlr4.Runtime.Atn.ATN</see>
+        /// used by the recognizer for prediction.
+        /// </summary>
+        /// <returns>
+        /// The
+        /// <see cref="Antlr4.Runtime.Atn.ATN">Antlr4.Runtime.Atn.ATN</see>
+        /// used by the recognizer for prediction.
+        /// </returns>
         public virtual ATN Atn
         {
             get
@@ -94,6 +187,15 @@ namespace Antlr4.Runtime
             }
         }
 
+        /// <summary>Get the ATN interpreter used by the recognizer for prediction.</summary>
+        /// <remarks>Get the ATN interpreter used by the recognizer for prediction.</remarks>
+        /// <returns>The ATN interpreter used by the recognizer for prediction.</returns>
+        /// <summary>Set the ATN interpreter used by the recognizer for prediction.</summary>
+        /// <remarks>Set the ATN interpreter used by the recognizer for prediction.</remarks>
+        /// <value>
+        /// The ATN interpreter used by the recognizer for
+        /// prediction.
+        /// </value>
         public virtual ATNInterpreter Interpreter
         {
             get
@@ -108,6 +210,7 @@ namespace Antlr4.Runtime
         }
 
         /// <summary>What is the error header, normally line/character position information?</summary>
+        [NotNull]
         public virtual string GetErrorHeader(RecognitionException e)
         {
             int line = e.OffendingToken.Line;
@@ -153,7 +256,8 @@ namespace Antlr4.Runtime
             return "'" + s + "'";
         }
 
-        /// <exception cref="System.ArgumentNullException">
+        /// <exception>
+        /// NullPointerException
         /// if
         /// <code>listener</code>
         /// is

runtime/CSharp/Antlr4.Runtime/RuleContext.cs

@@ -51,7 +51,7 @@ namespace Antlr4.Runtime
     /// The parent contexts are useful for computing lookahead sets and
     /// getting error information.
     /// These objects are used during parsing and prediction.
-    /// For the special case of parsers and tree parsers, we use the subclass
+    /// For the special case of parsers, we use the subclass
     /// ParserRuleContext.
     /// </remarks>
     /// <seealso cref="ParserRuleContext">ParserRuleContext</seealso>

runtime/CSharp/Antlr4.Runtime/TokenStreamRewriter.cs

@@ -43,54 +43,84 @@ namespace Antlr4.Runtime
     /// <remarks>
     /// Useful for rewriting out a buffered input token stream after doing some
     /// augmentation or other manipulations on it.
-    /// You can insert stuff, replace, and delete chunks.  Note that the
-    /// operations are done lazily--only if you convert the buffer to a
-    /// String with getText(). This is very efficient because you are not moving
-    /// data around all the time.  As the buffer of tokens is converted to strings,
-    /// the getText() method(s) scan the input token stream and check
-    /// to see if there is an operation at the current index.
-    /// If so, the operation is done and then normal String
-    /// rendering continues on the buffer.  This is like having multiple Turing
-    /// machine instruction streams (programs) operating on a single input tape. :)
-    /// This rewriter makes no modifications to the token stream. It does not
-    /// ask the stream to fill itself up nor does it advance the input cursor.
-    /// The token stream index() will return the same value before and after
-    /// any getText() call.
-    /// The rewriter only works on tokens that you have in the buffer and
-    /// ignores the current input cursor. If you are buffering tokens on-demand,
-    /// calling getText() halfway through the input will only do rewrites
-    /// for those tokens in the first half of the file.
-    /// Since the operations are done lazily at getText-time, operations do not
-    /// screw up the token index values.  That is, an insert operation at token
-    /// index i does not change the index values for tokens i+1..n-1.
-    /// Because operations never actually alter the buffer, you may always get
-    /// the original token stream back without undoing anything.  Since
-    /// the instructions are queued up, you can easily simulate transactions and
-    /// roll back any changes if there is an error just by removing instructions.
-    /// For example,
+    /// <p>
+    /// You can insert stuff, replace, and delete chunks. Note that the operations
+    /// are done lazily--only if you convert the buffer to a
+    /// <see cref="string">string</see>
+    /// with
+    /// <see cref="ITokenStream.GetText()">ITokenStream.GetText()</see>
+    /// . This is very efficient because you are not
+    /// moving data around all the time. As the buffer of tokens is converted to
+    /// strings, the
+    /// <see cref="GetText()">GetText()</see>
+    /// method(s) scan the input token stream and
+    /// check to see if there is an operation at the current index. If so, the
+    /// operation is done and then normal
+    /// <see cref="string">string</see>
+    /// rendering continues on the
+    /// buffer. This is like having multiple Turing machine instruction streams
+    /// (programs) operating on a single input tape. :)</p>
+    /// <p>
+    /// This rewriter makes no modifications to the token stream. It does not ask the
+    /// stream to fill itself up nor does it advance the input cursor. The token
+    /// stream
+    /// <see cref="IIntStream.Index()">IIntStream.Index()</see>
+    /// will return the same value before and
+    /// after any
+    /// <see cref="GetText()">GetText()</see>
+    /// call.</p>
+    /// <p>
+    /// The rewriter only works on tokens that you have in the buffer and ignores the
+    /// current input cursor. If you are buffering tokens on-demand, calling
+    /// <see cref="GetText()">GetText()</see>
+    /// halfway through the input will only do rewrites for those
+    /// tokens in the first half of the file.</p>
+    /// <p>
+    /// Since the operations are done lazily at
+    /// <see cref="GetText()">GetText()</see>
+    /// -time, operations do
+    /// not screw up the token index values. That is, an insert operation at token
+    /// index
+    /// <code>i</code>
+    /// does not change the index values for tokens
+    /// <code>i</code>
+    /// +1..n-1.</p>
+    /// <p>
+    /// Because operations never actually alter the buffer, you may always get the
+    /// original token stream back without undoing anything. Since the instructions
+    /// are queued up, you can easily simulate transactions and roll back any changes
+    /// if there is an error just by removing instructions. For example,</p>
+    /// <pre>
     /// CharStream input = new ANTLRFileStream("input");
     /// TLexer lex = new TLexer(input);
     /// CommonTokenStream tokens = new CommonTokenStream(lex);
     /// T parser = new T(tokens);
     /// TokenStreamRewriter rewriter = new TokenStreamRewriter(tokens);
     /// parser.startRule();
-    /// Then in the rules, you can execute (assuming rewriter is visible):
+    /// </pre>
+    /// <p>
+    /// Then in the rules, you can execute (assuming rewriter is visible):</p>
+    /// <pre>
     /// Token t,u;
     /// ...
     /// rewriter.insertAfter(t, "text to put after t");}
     /// rewriter.insertAfter(u, "text after u");}
     /// System.out.println(tokens.toString());
-    /// You can also have multiple "instruction streams" and get multiple
-    /// rewrites from a single pass over the input.  Just name the instruction
-    /// streams and use that name again when printing the buffer.  This could be
-    /// useful for generating a C file and also its header file--all from the
-    /// same buffer:
+    /// </pre>
+    /// <p>
+    /// You can also have multiple "instruction streams" and get multiple rewrites
+    /// from a single pass over the input. Just name the instruction streams and use
+    /// that name again when printing the buffer. This could be useful for generating
+    /// a C file and also its header file--all from the same buffer:</p>
+    /// <pre>
     /// tokens.insertAfter("pass1", t, "text to put after t");}
     /// tokens.insertAfter("pass2", u, "text after u");}
     /// System.out.println(tokens.toString("pass1"));
     /// System.out.println(tokens.toString("pass2"));
-    /// If you don't use named rewrite streams, a "default" stream is used as
-    /// the first example shows.
+    /// </pre>
+    /// <p>
+    /// If you don't use named rewrite streams, a "default" stream is used as the
+    /// first example shows.</p>
     /// </remarks>
     public class TokenStreamRewriter
     {
@@ -208,11 +238,11 @@ namespace Antlr4.Runtime
         /// <remarks>
         /// You may have multiple, named streams of rewrite operations.
         /// I'm calling these things "programs."
-        /// Maps String (name) -&gt; rewrite (List)
+        /// Maps String (name) &rarr; rewrite (List)
         /// </remarks>
         protected internal readonly IDictionary<string, IList<TokenStreamRewriter.RewriteOperation>> programs;
 
-        /// <summary>Map String (program name) -&gt; Integer index</summary>
+        /// <summary>Map String (program name) &rarr; Integer index</summary>
         protected internal readonly IDictionary<string, int> lastRewriteTokenIndexes;
 
         public TokenStreamRewriter(ITokenStream tokens)
@@ -542,7 +572,7 @@ namespace Antlr4.Runtime
         /// insert with replace and delete this replace.
         /// 3. throw exception if index in same range as previous replace
         /// Don't actually delete; make op null in list. Easier to walk list.
-        /// Later we can throw as we add to index -&gt; op map.
+        /// Later we can throw as we add to index &rarr; op map.
         /// Note that I.2 R.2-2 will wipe out I.2 even though, technically, the
         /// inserted stuff would be before the replace range.  But, if you
         /// add tokens in front of a method body '{' and then delete the method

runtime/CSharp/Antlr4.Runtime/Tree/AbstractParseTreeVisitor`1.cs

@@ -36,11 +36,10 @@ namespace Antlr4.Runtime.Tree
     {
         /// <summary>
         /// <inheritDoc></inheritDoc>
-        /// <p/>
-        /// The default implementation calls
+        /// <p>The default implementation calls
         /// <see cref="IParseTree.Accept{T}(IParseTreeVisitor{Result})">IParseTree.Accept&lt;T&gt;(IParseTreeVisitor&lt;Result&gt;)</see>
         /// on the
-        /// specified tree.
+        /// specified tree.</p>
         /// </summary>
         public virtual Result Visit(IParseTree tree)
         {
@@ -49,8 +48,7 @@ namespace Antlr4.Runtime.Tree
 
         /// <summary>
         /// <inheritDoc></inheritDoc>
-        /// <p/>
-        /// The default implementation initializes the aggregate result to
+        /// <p>The default implementation initializes the aggregate result to
         /// <see cref="AbstractParseTreeVisitor{Result}.DefaultResult()">defaultResult()</see>
         /// . Before visiting each child, it
         /// calls
@@ -63,7 +61,10 @@ namespace Antlr4.Runtime.Tree
         /// updated by calling
         /// <see cref="AbstractParseTreeVisitor{Result}.AggregateResult(object, object)">aggregateResult</see>
         /// with the
-        /// previous aggregate result and the result of visiting the child.
+        /// previous aggregate result and the result of visiting the child.</p>
+        /// <p>The default implementation is not safe for use in visitors that modify
+        /// the tree structure. Visitors that modify the tree should override this
+        /// method to behave properly in respect to the specific algorithm in use.</p>
         /// </summary>
         public virtual Result VisitChildren(IRuleNode node)
         {
@@ -84,10 +85,9 @@ namespace Antlr4.Runtime.Tree
 
         /// <summary>
         /// <inheritDoc></inheritDoc>
-        /// <p/>
-        /// The default implementation returns the result of
+        /// <p>The default implementation returns the result of
         /// <see cref="AbstractParseTreeVisitor{Result}.DefaultResult()">defaultResult</see>
-        /// .
+        /// .</p>
         /// </summary>
         public virtual Result VisitTerminal(ITerminalNode node)
         {
@@ -96,10 +96,9 @@ namespace Antlr4.Runtime.Tree
 
         /// <summary>
         /// <inheritDoc></inheritDoc>
-        /// <p/>
-        /// The default implementation returns the result of
+        /// <p>The default implementation returns the result of
         /// <see cref="AbstractParseTreeVisitor{Result}.DefaultResult()">defaultResult</see>
-        /// .
+        /// .</p>
         /// </summary>
         public virtual Result VisitErrorNode(IErrorNode node)
         {
@@ -117,10 +116,9 @@ namespace Antlr4.Runtime.Tree
         /// The default implementation of
         /// <see cref="AbstractParseTreeVisitor{Result}.VisitChildren(IRuleNode)">visitChildren</see>
         /// initializes its aggregate result to this value.
-        /// <p/>
-        /// The base implementation returns
+        /// <p>The base implementation returns
         /// <code>null</code>
-        /// .
+        /// .</p>
         /// </remarks>
         /// <returns>The default value returned by visitor methods.</returns>
         protected internal virtual Result DefaultResult()
@@ -138,13 +136,12 @@ namespace Antlr4.Runtime.Tree
         /// , the aggregate value is returned as the result of
         /// <see cref="AbstractParseTreeVisitor{Result}.VisitChildren(IRuleNode)">AbstractParseTreeVisitor&lt;Result&gt;.VisitChildren(IRuleNode)</see>
         /// .
-        /// <p/>
-        /// The default implementation returns
+        /// <p>The default implementation returns
         /// <code>nextResult</code>
         /// , meaning
         /// <see cref="AbstractParseTreeVisitor{Result}.VisitChildren(IRuleNode)">AbstractParseTreeVisitor&lt;Result&gt;.VisitChildren(IRuleNode)</see>
         /// will return the result of the last child visited
-        /// (or return the initial value if the node has no children).
+        /// (or return the initial value if the node has no children).</p>
         /// </remarks>
         /// <param name="aggregate">
         /// The previous aggregate value. In the default
@@ -177,8 +174,7 @@ namespace Antlr4.Runtime.Tree
         /// <see cref="AbstractParseTreeVisitor{Result}.DefaultResult()">AbstractParseTreeVisitor&lt;Result&gt;.DefaultResult()</see>
         /// . This method is not called after the last
         /// child is visited.
-        /// <p/>
-        /// The default implementation always returns
+        /// <p>The default implementation always returns
         /// <code>true</code>
         /// , indicating that
         /// <code>visitChildren</code>
@@ -186,7 +182,7 @@ namespace Antlr4.Runtime.Tree
         /// One reason to override this method is to provide a "short circuit"
         /// evaluation option for situations where the result of visiting a single
         /// child has the potential to determine the result of the visit operation as
-        /// a whole.
+        /// a whole.</p>
         /// </summary>
         /// <param name="node">
         /// The

runtime/CSharp/Antlr4.Runtime/Tree/IParseTree.cs

@@ -40,12 +40,11 @@ namespace Antlr4.Runtime.Tree
     /// during a parse that makes the data structure look like a simple parse tree.
     /// This node represents both internal nodes, rule invocations,
     /// and leaf nodes, token matches.
-    /// <p/>
-    /// The payload is either a
+    /// <p>The payload is either a
     /// <see cref="Antlr4.Runtime.IToken">Antlr4.Runtime.IToken</see>
     /// or a
     /// <see cref="Antlr4.Runtime.RuleContext">Antlr4.Runtime.RuleContext</see>
-    /// object.
+    /// object.</p>
     /// </summary>
     public interface IParseTree : ISyntaxTree
     {

runtime/CSharp/Antlr4.Runtime/Tree/ISyntaxTree.cs

@@ -52,10 +52,9 @@ namespace Antlr4.Runtime.Tree
         /// of the first and last token associated with this
         /// subtree. If this node is a leaf, then the interval represents a single
         /// token.
-        /// <p/>
-        /// If source interval is unknown, this returns
+        /// <p>If source interval is unknown, this returns
         /// <see cref="Antlr4.Runtime.Misc.Interval.Invalid">Antlr4.Runtime.Misc.Interval.Invalid</see>
-        /// .
+        /// .</p>
         /// </summary>
         Interval SourceInterval
         {

runtime/CSharp/Antlr4.Runtime/Tree/Pattern/Chunk.cs

@@ -0,0 +1,61 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using Antlr4.Runtime.Tree.Pattern;
+using Sharpen;
+
+namespace Antlr4.Runtime.Tree.Pattern
+{
+    /// <summary>
+    /// A chunk is either a token tag, a rule tag, or a span of literal text within a
+    /// tree pattern.
+    /// </summary>
+    /// <remarks>
+    /// A chunk is either a token tag, a rule tag, or a span of literal text within a
+    /// tree pattern.
+    /// <p>The method
+    /// <see cref="ParseTreePatternMatcher.Split(string)">ParseTreePatternMatcher.Split(string)</see>
+    /// returns a list of
+    /// chunks in preparation for creating a token stream by
+    /// <see cref="ParseTreePatternMatcher.Tokenize(string)">ParseTreePatternMatcher.Tokenize(string)</see>
+    /// . From there, we get a parse
+    /// tree from with
+    /// <see cref="ParseTreePatternMatcher.Compile(string, int)">ParseTreePatternMatcher.Compile(string, int)</see>
+    /// . These
+    /// chunks are converted to
+    /// <see cref="RuleTagToken">RuleTagToken</see>
+    /// ,
+    /// <see cref="TokenTagToken">TokenTagToken</see>
+    /// , or the
+    /// regular tokens of the text surrounding the tags.</p>
+    /// </remarks>
+    internal abstract class Chunk
+    {
+    }
+}

runtime/CSharp/Antlr4.Runtime/Tree/Pattern/ParseTreeMatch.cs

@@ -0,0 +1,304 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using System;
+using System.Collections.Generic;
+using Antlr4.Runtime.Misc;
+using Antlr4.Runtime.Tree;
+using Antlr4.Runtime.Tree.Pattern;
+using Sharpen;
+
+namespace Antlr4.Runtime.Tree.Pattern
+{
+    /// <summary>
+    /// Represents the result of matching a
+    /// <see cref="Antlr4.Runtime.Tree.IParseTree">Antlr4.Runtime.Tree.IParseTree</see>
+    /// against a tree pattern.
+    /// </summary>
+    public class ParseTreeMatch
+    {
+        /// <summary>
+        /// This is the backing field for
+        /// <see cref="GetTree()">GetTree()</see>
+        /// .
+        /// </summary>
+        private readonly IParseTree tree;
+
+        /// <summary>
+        /// This is the backing field for
+        /// <see cref="GetPattern()">GetPattern()</see>
+        /// .
+        /// </summary>
+        private readonly ParseTreePattern pattern;
+
+        /// <summary>
+        /// This is the backing field for
+        /// <see cref="GetLabels()">GetLabels()</see>
+        /// .
+        /// </summary>
+        private readonly MultiMap<string, IParseTree> labels;
+
+        /// <summary>
+        /// This is the backing field for
+        /// <see cref="GetMismatchedNode()">GetMismatchedNode()</see>
+        /// .
+        /// </summary>
+        private readonly IParseTree mismatchedNode;
+
+        /// <summary>
+        /// Constructs a new instance of
+        /// <see cref="ParseTreeMatch">ParseTreeMatch</see>
+        /// from the specified
+        /// parse tree and pattern.
+        /// </summary>
+        /// <param name="tree">The parse tree to match against the pattern.</param>
+        /// <param name="pattern">The parse tree pattern.</param>
+        /// <param name="labels">
+        /// A mapping from label names to collections of
+        /// <see cref="Antlr4.Runtime.Tree.IParseTree">Antlr4.Runtime.Tree.IParseTree</see>
+        /// objects located by the tree pattern matching process.
+        /// </param>
+        /// <param name="mismatchedNode">
+        /// The first node which failed to match the tree
+        /// pattern during the matching process.
+        /// </param>
+        /// <exception>
+        /// IllegalArgumentException
+        /// if
+        /// <code>tree</code>
+        /// is
+        /// <code>null</code>
+        /// </exception>
+        /// <exception>
+        /// IllegalArgumentException
+        /// if
+        /// <code>pattern</code>
+        /// is
+        /// <code>null</code>
+        /// </exception>
+        /// <exception>
+        /// IllegalArgumentException
+        /// if
+        /// <code>labels</code>
+        /// is
+        /// <code>null</code>
+        /// </exception>
+        public ParseTreeMatch(IParseTree tree, ParseTreePattern pattern, MultiMap<string, IParseTree> labels, IParseTree mismatchedNode)
+        {
+            if (tree == null)
+            {
+                throw new ArgumentException("tree cannot be null");
+            }
+            if (pattern == null)
+            {
+                throw new ArgumentException("pattern cannot be null");
+            }
+            if (labels == null)
+            {
+                throw new ArgumentException("labels cannot be null");
+            }
+            this.tree = tree;
+            this.pattern = pattern;
+            this.labels = labels;
+            this.mismatchedNode = mismatchedNode;
+        }
+
+        /// <summary>
+        /// Get the last node associated with a specific
+        /// <code>label</code>
+        /// .
+        /// <p>For example, for pattern
+        /// <code><id:ID></code>
+        /// ,
+        /// <code>get("id")</code>
+        /// returns the
+        /// node matched for that
+        /// <code>ID</code>
+        /// . If more than one node
+        /// matched the specified label, only the last is returned. If there is
+        /// no node associated with the label, this returns
+        /// <code>null</code>
+        /// .</p>
+        /// <p>Pattern tags like
+        /// <code><ID></code>
+        /// and
+        /// <code><expr></code>
+        /// without labels are
+        /// considered to be labeled with
+        /// <code>ID</code>
+        /// and
+        /// <code>expr</code>
+        /// , respectively.</p>
+        /// </summary>
+        /// <param name="label">The label to check.</param>
+        /// <returns>
+        /// The last
+        /// <see cref="Antlr4.Runtime.Tree.IParseTree">Antlr4.Runtime.Tree.IParseTree</see>
+        /// to match a tag with the specified
+        /// label, or
+        /// <code>null</code>
+        /// if no parse tree matched a tag with the label.
+        /// </returns>
+        [Nullable]
+        public virtual IParseTree Get(string label)
+        {
+            IList<IParseTree> parseTrees = labels.Get(label);
+            if (parseTrees == null || parseTrees.Count == 0)
+            {
+                return null;
+            }
+            return parseTrees[parseTrees.Count - 1];
+        }
+
+        // return last if multiple
+        /// <summary>Return all nodes matching a rule or token tag with the specified label.</summary>
+        /// <remarks>
+        /// Return all nodes matching a rule or token tag with the specified label.
+        /// <p>If the
+        /// <code>label</code>
+        /// is the name of a parser rule or token in the
+        /// grammar, the resulting list will contain both the parse trees matching
+        /// rule or tags explicitly labeled with the label and the complete set of
+        /// parse trees matching the labeled and unlabeled tags in the pattern for
+        /// the parser rule or token. For example, if
+        /// <code>label</code>
+        /// is
+        /// <code>"foo"</code>
+        /// ,
+        /// the result will contain <em>all</em> of the following.</p>
+        /// <ul>
+        /// <li>Parse tree nodes matching tags of the form
+        /// <code><foo:anyRuleName></code>
+        /// and
+        /// <code><foo:AnyTokenName></code>
+        /// .</li>
+        /// <li>Parse tree nodes matching tags of the form
+        /// <code><anyLabel:foo></code>
+        /// .</li>
+        /// <li>Parse tree nodes matching tags of the form
+        /// <code><foo></code>
+        /// .</li>
+        /// </ul>
+        /// </remarks>
+        /// <param name="label">The label.</param>
+        /// <returns>
+        /// A collection of all
+        /// <see cref="Antlr4.Runtime.Tree.IParseTree">Antlr4.Runtime.Tree.IParseTree</see>
+        /// nodes matching tags with
+        /// the specified
+        /// <code>label</code>
+        /// . If no nodes matched the label, an empty list
+        /// is returned.
+        /// </returns>
+        [NotNull]
+        public virtual IList<IParseTree> GetAll(string label)
+        {
+            IList<IParseTree> nodes = labels.Get(label);
+            if (nodes == null)
+            {
+                return Sharpen.Collections.EmptyList();
+            }
+            return nodes;
+        }
+
+        /// <summary>Return a mapping from label &rarr; [list of nodes].</summary>
+        /// <remarks>
+        /// Return a mapping from label &rarr; [list of nodes].
+        /// <p>The map includes special entries corresponding to the names of rules and
+        /// tokens referenced in tags in the original pattern. For additional
+        /// information, see the description of
+        /// <see cref="GetAll(string)">GetAll(string)</see>
+        /// .</p>
+        /// </remarks>
+        /// <returns>
+        /// A mapping from labels to parse tree nodes. If the parse tree
+        /// pattern did not contain any rule or token tags, this map will be empty.
+        /// </returns>
+        [NotNull]
+        public virtual MultiMap<string, IParseTree> GetLabels()
+        {
+            return labels;
+        }
+
+        /// <summary>Get the node at which we first detected a mismatch.</summary>
+        /// <remarks>Get the node at which we first detected a mismatch.</remarks>
+        /// <returns>
+        /// the node at which we first detected a mismatch, or
+        /// <code>null</code>
+        /// if the match was successful.
+        /// </returns>
+        [Nullable]
+        public virtual IParseTree GetMismatchedNode()
+        {
+            return mismatchedNode;
+        }
+
+        /// <summary>Gets a value indicating whether the match operation succeeded.</summary>
+        /// <remarks>Gets a value indicating whether the match operation succeeded.</remarks>
+        /// <returns>
+        /// 
+        /// <code>true</code>
+        /// if the match operation succeeded; otherwise,
+        /// <code>false</code>
+        /// .
+        /// </returns>
+        public virtual bool Succeeded()
+        {
+            return mismatchedNode == null;
+        }
+
+        /// <summary>Get the tree pattern we are matching against.</summary>
+        /// <remarks>Get the tree pattern we are matching against.</remarks>
+        /// <returns>The tree pattern we are matching against.</returns>
+        [NotNull]
+        public virtual ParseTreePattern GetPattern()
+        {
+            return pattern;
+        }
+
+        /// <summary>Get the parse tree we are trying to match to a pattern.</summary>
+        /// <remarks>Get the parse tree we are trying to match to a pattern.</remarks>
+        /// <returns>
+        /// The
+        /// <see cref="Antlr4.Runtime.Tree.IParseTree">Antlr4.Runtime.Tree.IParseTree</see>
+        /// we are trying to match to a pattern.
+        /// </returns>
+        [NotNull]
+        public virtual IParseTree GetTree()
+        {
+            return tree;
+        }
+
+        /// <summary><inheritDoc></inheritDoc></summary>
+        public override string ToString()
+        {
+            return string.Format("Match %s; found %d labels", Succeeded() ? "succeeded" : "failed", GetLabels().Count);
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/Tree/Pattern/ParseTreePattern.cs

@@ -0,0 +1,248 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using System.Collections.Generic;
+using Antlr4.Runtime.Misc;
+using Antlr4.Runtime.Tree;
+using Antlr4.Runtime.Tree.Pattern;
+using Antlr4.Runtime.Tree.Xpath;
+using Sharpen;
+
+namespace Antlr4.Runtime.Tree.Pattern
+{
+    /// <summary>
+    /// A pattern like
+    /// <code><ID> = <expr>;</code>
+    /// converted to a
+    /// <see cref="Antlr4.Runtime.Tree.IParseTree">Antlr4.Runtime.Tree.IParseTree</see>
+    /// by
+    /// <see cref="ParseTreePatternMatcher.Compile(string, int)">ParseTreePatternMatcher.Compile(string, int)</see>
+    /// .
+    /// </summary>
+    public class ParseTreePattern
+    {
+        /// <summary>
+        /// This is the backing field for
+        /// <see cref="GetPatternRuleIndex()">GetPatternRuleIndex()</see>
+        /// .
+        /// </summary>
+        private readonly int patternRuleIndex;
+
+        /// <summary>
+        /// This is the backing field for
+        /// <see cref="GetPattern()">GetPattern()</see>
+        /// .
+        /// </summary>
+        [NotNull]
+        private readonly string pattern;
+
+        /// <summary>
+        /// This is the backing field for
+        /// <see cref="GetPatternTree()">GetPatternTree()</see>
+        /// .
+        /// </summary>
+        [NotNull]
+        private readonly IParseTree patternTree;
+
+        /// <summary>
+        /// This is the backing field for
+        /// <see cref="GetMatcher()">GetMatcher()</see>
+        /// .
+        /// </summary>
+        [NotNull]
+        private readonly ParseTreePatternMatcher matcher;
+
+        /// <summary>
+        /// Construct a new instance of the
+        /// <see cref="ParseTreePattern">ParseTreePattern</see>
+        /// class.
+        /// </summary>
+        /// <param name="matcher">
+        /// The
+        /// <see cref="ParseTreePatternMatcher">ParseTreePatternMatcher</see>
+        /// which created this
+        /// tree pattern.
+        /// </param>
+        /// <param name="pattern">The tree pattern in concrete syntax form.</param>
+        /// <param name="patternRuleIndex">
+        /// The parser rule which serves as the root of the
+        /// tree pattern.
+        /// </param>
+        /// <param name="patternTree">
+        /// The tree pattern in
+        /// <see cref="Antlr4.Runtime.Tree.IParseTree">Antlr4.Runtime.Tree.IParseTree</see>
+        /// form.
+        /// </param>
+        public ParseTreePattern(ParseTreePatternMatcher matcher, string pattern, int patternRuleIndex, IParseTree patternTree)
+        {
+            this.matcher = matcher;
+            this.patternRuleIndex = patternRuleIndex;
+            this.pattern = pattern;
+            this.patternTree = patternTree;
+        }
+
+        /// <summary>Match a specific parse tree against this tree pattern.</summary>
+        /// <remarks>Match a specific parse tree against this tree pattern.</remarks>
+        /// <param name="tree">The parse tree to match against this tree pattern.</param>
+        /// <returns>
+        /// A
+        /// <see cref="ParseTreeMatch">ParseTreeMatch</see>
+        /// object describing the result of the
+        /// match operation. The
+        /// <see cref="ParseTreeMatch.Succeeded()">ParseTreeMatch.Succeeded()</see>
+        /// method can be
+        /// used to determine whether or not the match was successful.
+        /// </returns>
+        [NotNull]
+        public virtual ParseTreeMatch Match(IParseTree tree)
+        {
+            return matcher.Match(tree, this);
+        }
+
+        /// <summary>Determine whether or not a parse tree matches this tree pattern.</summary>
+        /// <remarks>Determine whether or not a parse tree matches this tree pattern.</remarks>
+        /// <param name="tree">The parse tree to match against this tree pattern.</param>
+        /// <returns>
+        /// 
+        /// <code>true</code>
+        /// if
+        /// <code>tree</code>
+        /// is a match for the current tree
+        /// pattern; otherwise,
+        /// <code>false</code>
+        /// .
+        /// </returns>
+        public virtual bool Matches(IParseTree tree)
+        {
+            return matcher.Match(tree, this).Succeeded();
+        }
+
+        /// <summary>
+        /// Find all nodes using XPath and then try to match those subtrees against
+        /// this tree pattern.
+        /// </summary>
+        /// <remarks>
+        /// Find all nodes using XPath and then try to match those subtrees against
+        /// this tree pattern.
+        /// </remarks>
+        /// <param name="tree">
+        /// The
+        /// <see cref="Antlr4.Runtime.Tree.IParseTree">Antlr4.Runtime.Tree.IParseTree</see>
+        /// to match against this pattern.
+        /// </param>
+        /// <param name="xpath">An expression matching the nodes</param>
+        /// <returns>
+        /// A collection of
+        /// <see cref="ParseTreeMatch">ParseTreeMatch</see>
+        /// objects describing the
+        /// successful matches. Unsuccessful matches are omitted from the result,
+        /// regardless of the reason for the failure.
+        /// </returns>
+        [NotNull]
+        public virtual IList<ParseTreeMatch> FindAll(IParseTree tree, string xpath)
+        {
+            ICollection<IParseTree> subtrees = XPath.FindAll(tree, xpath, matcher.GetParser());
+            IList<ParseTreeMatch> matches = new List<ParseTreeMatch>();
+            foreach (IParseTree t in subtrees)
+            {
+                ParseTreeMatch match = Match(t);
+                if (match.Succeeded())
+                {
+                    matches.AddItem(match);
+                }
+            }
+            return matches;
+        }
+
+        /// <summary>
+        /// Get the
+        /// <see cref="ParseTreePatternMatcher">ParseTreePatternMatcher</see>
+        /// which created this tree pattern.
+        /// </summary>
+        /// <returns>
+        /// The
+        /// <see cref="ParseTreePatternMatcher">ParseTreePatternMatcher</see>
+        /// which created this tree
+        /// pattern.
+        /// </returns>
+        [NotNull]
+        public virtual ParseTreePatternMatcher GetMatcher()
+        {
+            return matcher;
+        }
+
+        /// <summary>Get the tree pattern in concrete syntax form.</summary>
+        /// <remarks>Get the tree pattern in concrete syntax form.</remarks>
+        /// <returns>The tree pattern in concrete syntax form.</returns>
+        [NotNull]
+        public virtual string GetPattern()
+        {
+            return pattern;
+        }
+
+        /// <summary>
+        /// Get the parser rule which serves as the outermost rule for the tree
+        /// pattern.
+        /// </summary>
+        /// <remarks>
+        /// Get the parser rule which serves as the outermost rule for the tree
+        /// pattern.
+        /// </remarks>
+        /// <returns>
+        /// The parser rule which serves as the outermost rule for the tree
+        /// pattern.
+        /// </returns>
+        public virtual int GetPatternRuleIndex()
+        {
+            return patternRuleIndex;
+        }
+
+        /// <summary>
+        /// Get the tree pattern as a
+        /// <see cref="Antlr4.Runtime.Tree.IParseTree">Antlr4.Runtime.Tree.IParseTree</see>
+        /// . The rule and token tags from
+        /// the pattern are present in the parse tree as terminal nodes with a symbol
+        /// of type
+        /// <see cref="RuleTagToken">RuleTagToken</see>
+        /// or
+        /// <see cref="TokenTagToken">TokenTagToken</see>
+        /// .
+        /// </summary>
+        /// <returns>
+        /// The tree pattern as a
+        /// <see cref="Antlr4.Runtime.Tree.IParseTree">Antlr4.Runtime.Tree.IParseTree</see>
+        /// .
+        /// </returns>
+        [NotNull]
+        public virtual IParseTree GetPatternTree()
+        {
+            return patternTree;
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/Tree/Pattern/ParseTreePatternMatcher.cs

@@ -0,0 +1,723 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Antlr4.Runtime;
+using Antlr4.Runtime.Misc;
+using Antlr4.Runtime.Tree;
+using Antlr4.Runtime.Tree.Pattern;
+using Sharpen;
+
+namespace Antlr4.Runtime.Tree.Pattern
+{
+    /// <summary>
+    /// A tree pattern matching mechanism for ANTLR
+    /// <see cref="Antlr4.Runtime.Tree.IParseTree">Antlr4.Runtime.Tree.IParseTree</see>
+    /// s.
+    /// <p>Patterns are strings of source input text with special tags representing
+    /// token or rule references such as:</p>
+    /// <p>
+    /// <code><ID> = <expr>;</code>
+    /// </p>
+    /// <p>Given a pattern start rule such as
+    /// <code>statement</code>
+    /// , this object constructs
+    /// a
+    /// <see cref="Antlr4.Runtime.Tree.IParseTree">Antlr4.Runtime.Tree.IParseTree</see>
+    /// with placeholders for the
+    /// <code>ID</code>
+    /// and
+    /// <code>expr</code>
+    /// subtree. Then the
+    /// <see cref="Match(Antlr4.Runtime.Tree.IParseTree, ParseTreePattern)">Match(Antlr4.Runtime.Tree.IParseTree, ParseTreePattern)</see>
+    /// routines can compare an actual
+    /// <see cref="Antlr4.Runtime.Tree.IParseTree">Antlr4.Runtime.Tree.IParseTree</see>
+    /// from a parse with this pattern. Tag
+    /// <code><ID></code>
+    /// matches
+    /// any
+    /// <code>ID</code>
+    /// token and tag
+    /// <code><expr></code>
+    /// references the result of the
+    /// <code>expr</code>
+    /// rule (generally an instance of
+    /// <code>ExprContext</code>
+    /// .</p>
+    /// <p>Pattern
+    /// <code>x = 0;</code>
+    /// is a similar pattern that matches the same pattern
+    /// except that it requires the identifier to be
+    /// <code>x</code>
+    /// and the expression to
+    /// be
+    /// <code>0</code>
+    /// .</p>
+    /// <p>The
+    /// <see cref="Matches(Antlr4.Runtime.Tree.IParseTree, ParseTreePattern)">Matches(Antlr4.Runtime.Tree.IParseTree, ParseTreePattern)</see>
+    /// routines return
+    /// <code>true</code>
+    /// or
+    /// <code>false</code>
+    /// based
+    /// upon a match for the tree rooted at the parameter sent in. The
+    /// <see cref="Match(Antlr4.Runtime.Tree.IParseTree, ParseTreePattern)">Match(Antlr4.Runtime.Tree.IParseTree, ParseTreePattern)</see>
+    /// routines return a
+    /// <see cref="ParseTreeMatch">ParseTreeMatch</see>
+    /// object that
+    /// contains the parse tree, the parse tree pattern, and a map from tag name to
+    /// matched nodes (more below). A subtree that fails to match, returns with
+    /// <see cref="ParseTreeMatch#mismatchedNode">ParseTreeMatch#mismatchedNode</see>
+    /// set to the first tree node that did not
+    /// match.</p>
+    /// <p>For efficiency, you can compile a tree pattern in string form to a
+    /// <see cref="ParseTreePattern">ParseTreePattern</see>
+    /// object.</p>
+    /// <p>See
+    /// <code>TestParseTreeMatcher</code>
+    /// for lots of examples.
+    /// <see cref="ParseTreePattern">ParseTreePattern</see>
+    /// has two static helper methods:
+    /// <see cref="ParseTreePattern.FindAll(Antlr4.Runtime.Tree.IParseTree, string)">ParseTreePattern.FindAll(Antlr4.Runtime.Tree.IParseTree, string)</see>
+    /// and
+    /// <see cref="ParseTreePattern.Match(Antlr4.Runtime.Tree.IParseTree)">ParseTreePattern.Match(Antlr4.Runtime.Tree.IParseTree)</see>
+    /// that
+    /// are easy to use but not super efficient because they create new
+    /// <see cref="ParseTreePatternMatcher">ParseTreePatternMatcher</see>
+    /// objects each time and have to compile the
+    /// pattern in string form before using it.</p>
+    /// <p>The lexer and parser that you pass into the
+    /// <see cref="ParseTreePatternMatcher">ParseTreePatternMatcher</see>
+    /// constructor are used to parse the pattern in string form. The lexer converts
+    /// the
+    /// <code><ID> = <expr>;</code>
+    /// into a sequence of four tokens (assuming lexer
+    /// throws out whitespace or puts it on a hidden channel). Be aware that the
+    /// input stream is reset for the lexer (but not the parser; a
+    /// <see cref="Antlr4.Runtime.ParserInterpreter">Antlr4.Runtime.ParserInterpreter</see>
+    /// is created to parse the input.). Any user-defined
+    /// fields you have put into the lexer might get changed when this mechanism asks
+    /// it to scan the pattern string.</p>
+    /// <p>Normally a parser does not accept token
+    /// <code><expr></code>
+    /// as a valid
+    /// <code>expr</code>
+    /// but, from the parser passed in, we create a special version of
+    /// the underlying grammar representation (an
+    /// <see cref="Antlr4.Runtime.Atn.ATN">Antlr4.Runtime.Atn.ATN</see>
+    /// ) that allows imaginary
+    /// tokens representing rules (
+    /// <code><expr></code>
+    /// ) to match entire rules. We call
+    /// these <em>bypass alternatives</em>.</p>
+    /// <p>Delimiters are
+    /// <code>&lt;</code>
+    /// and
+    /// <code>&gt;</code>
+    /// , with
+    /// <code>\</code>
+    /// as the escape string
+    /// by default, but you can set them to whatever you want using
+    /// <see cref="SetDelimiters(string, string, string)">SetDelimiters(string, string, string)</see>
+    /// . You must escape both start and stop strings
+    /// <code>\&lt;</code>
+    /// and
+    /// <code>\&gt;</code>
+    /// .</p>
+    /// </summary>
+    public class ParseTreePatternMatcher
+    {
+        [System.Serializable]
+        public class CannotInvokeStartRule : RuntimeException
+        {
+            public CannotInvokeStartRule(Exception e)
+                : base(e)
+            {
+            }
+        }
+
+        [System.Serializable]
+        public class StartRuleDoesNotConsumeFullPattern : RuntimeException
+        {
+            // Fixes https://github.com/antlr/antlr4/issues/413
+            // "Tree pattern compilation doesn't check for a complete parse"
+        }
+
+        /// <summary>
+        /// This is the backing field for
+        /// <see cref="GetLexer()">GetLexer()</see>
+        /// .
+        /// </summary>
+        private readonly Lexer lexer;
+
+        /// <summary>
+        /// This is the backing field for
+        /// <see cref="GetParser()">GetParser()</see>
+        /// .
+        /// </summary>
+        private readonly Parser parser;
+
+        protected internal string start = "<";
+
+        protected internal string stop = ">";
+
+        protected internal string escape = "\\";
+
+        /// <summary>
+        /// Constructs a
+        /// <see cref="ParseTreePatternMatcher">ParseTreePatternMatcher</see>
+        /// or from a
+        /// <see cref="Antlr4.Runtime.Lexer">Antlr4.Runtime.Lexer</see>
+        /// and
+        /// <see cref="Antlr4.Runtime.Parser">Antlr4.Runtime.Parser</see>
+        /// object. The lexer input stream is altered for tokenizing
+        /// the tree patterns. The parser is used as a convenient mechanism to get
+        /// the grammar name, plus token, rule names.
+        /// </summary>
+        public ParseTreePatternMatcher(Lexer lexer, Parser parser)
+        {
+            // e.g., \< and \> must escape BOTH!
+            this.lexer = lexer;
+            this.parser = parser;
+        }
+
+        /// <summary>
+        /// Set the delimiters used for marking rule and token tags within concrete
+        /// syntax used by the tree pattern parser.
+        /// </summary>
+        /// <remarks>
+        /// Set the delimiters used for marking rule and token tags within concrete
+        /// syntax used by the tree pattern parser.
+        /// </remarks>
+        /// <param name="start">The start delimiter.</param>
+        /// <param name="stop">The stop delimiter.</param>
+        /// <param name="escapeLeft">The escape sequence to use for escaping a start or stop delimiter.</param>
+        /// <exception>
+        /// IllegalArgumentException
+        /// if
+        /// <code>start</code>
+        /// is
+        /// <code>null</code>
+        /// or empty.
+        /// </exception>
+        /// <exception>
+        /// IllegalArgumentException
+        /// if
+        /// <code>stop</code>
+        /// is
+        /// <code>null</code>
+        /// or empty.
+        /// </exception>
+        public virtual void SetDelimiters(string start, string stop, string escapeLeft)
+        {
+            if (start == null || start.IsEmpty())
+            {
+                throw new ArgumentException("start cannot be null or empty");
+            }
+            if (stop == null || stop.IsEmpty())
+            {
+                throw new ArgumentException("stop cannot be null or empty");
+            }
+            this.start = start;
+            this.stop = stop;
+            this.escape = escapeLeft;
+        }
+
+        /// <summary>
+        /// Does
+        /// <code>pattern</code>
+        /// matched as rule
+        /// <code>patternRuleIndex</code>
+        /// match
+        /// <code>tree</code>
+        /// ?
+        /// </summary>
+        public virtual bool Matches(IParseTree tree, string pattern, int patternRuleIndex)
+        {
+            ParseTreePattern p = Compile(pattern, patternRuleIndex);
+            return Matches(tree, p);
+        }
+
+        /// <summary>
+        /// Does
+        /// <code>pattern</code>
+        /// matched as rule patternRuleIndex match tree? Pass in a
+        /// compiled pattern instead of a string representation of a tree pattern.
+        /// </summary>
+        public virtual bool Matches(IParseTree tree, ParseTreePattern pattern)
+        {
+            MultiMap<string, IParseTree> labels = new MultiMap<string, IParseTree>();
+            IParseTree mismatchedNode = MatchImpl(tree, pattern.GetPatternTree(), labels);
+            return mismatchedNode == null;
+        }
+
+        /// <summary>
+        /// Compare
+        /// <code>pattern</code>
+        /// matched as rule
+        /// <code>patternRuleIndex</code>
+        /// against
+        /// <code>tree</code>
+        /// and return a
+        /// <see cref="ParseTreeMatch">ParseTreeMatch</see>
+        /// object that contains the
+        /// matched elements, or the node at which the match failed.
+        /// </summary>
+        public virtual ParseTreeMatch Match(IParseTree tree, string pattern, int patternRuleIndex)
+        {
+            ParseTreePattern p = Compile(pattern, patternRuleIndex);
+            return Match(tree, p);
+        }
+
+        /// <summary>
+        /// Compare
+        /// <code>pattern</code>
+        /// matched against
+        /// <code>tree</code>
+        /// and return a
+        /// <see cref="ParseTreeMatch">ParseTreeMatch</see>
+        /// object that contains the matched elements, or the
+        /// node at which the match failed. Pass in a compiled pattern instead of a
+        /// string representation of a tree pattern.
+        /// </summary>
+        [NotNull]
+        public virtual ParseTreeMatch Match(IParseTree tree, ParseTreePattern pattern)
+        {
+            MultiMap<string, IParseTree> labels = new MultiMap<string, IParseTree>();
+            IParseTree mismatchedNode = MatchImpl(tree, pattern.GetPatternTree(), labels);
+            return new ParseTreeMatch(tree, pattern, labels, mismatchedNode);
+        }
+
+        /// <summary>
+        /// For repeated use of a tree pattern, compile it to a
+        /// <see cref="ParseTreePattern">ParseTreePattern</see>
+        /// using this method.
+        /// </summary>
+        public virtual ParseTreePattern Compile(string pattern, int patternRuleIndex)
+        {
+            IList<IToken> tokenList = Tokenize(pattern);
+            ListTokenSource tokenSrc = new ListTokenSource(tokenList);
+            CommonTokenStream tokens = new CommonTokenStream(tokenSrc);
+            ParserInterpreter parserInterp = new ParserInterpreter(parser.GrammarFileName, Arrays.AsList(parser.TokenNames), Arrays.AsList(parser.RuleNames), parser.GetATNWithBypassAlts(), tokens);
+            IParseTree tree = null;
+            try
+            {
+                parserInterp.ErrorHandler = new BailErrorStrategy();
+                tree = parserInterp.Parse(patternRuleIndex);
+            }
+            catch (ParseCanceledException e)
+            {
+                //			System.out.println("pattern tree = "+tree.toStringTree(parserInterp));
+                throw (RecognitionException)e.InnerException;
+            }
+            catch (RecognitionException re)
+            {
+                throw;
+            }
+            catch (Exception e)
+            {
+                throw new ParseTreePatternMatcher.CannotInvokeStartRule(e);
+            }
+            // Make sure tree pattern compilation checks for a complete parse
+            if (tokens.La(1) != TokenConstants.Eof)
+            {
+                throw new ParseTreePatternMatcher.StartRuleDoesNotConsumeFullPattern();
+            }
+            return new ParseTreePattern(this, pattern, patternRuleIndex, tree);
+        }
+
+        /// <summary>Used to convert the tree pattern string into a series of tokens.</summary>
+        /// <remarks>
+        /// Used to convert the tree pattern string into a series of tokens. The
+        /// input stream is reset.
+        /// </remarks>
+        [NotNull]
+        public virtual Lexer GetLexer()
+        {
+            return lexer;
+        }
+
+        /// <summary>
+        /// Used to collect to the grammar file name, token names, rule names for
+        /// used to parse the pattern into a parse tree.
+        /// </summary>
+        /// <remarks>
+        /// Used to collect to the grammar file name, token names, rule names for
+        /// used to parse the pattern into a parse tree.
+        /// </remarks>
+        [NotNull]
+        public virtual Parser GetParser()
+        {
+            return parser;
+        }
+
+        // ---- SUPPORT CODE ----
+        /// <summary>
+        /// Recursively walk
+        /// <code>tree</code>
+        /// against
+        /// <code>patternTree</code>
+        /// , filling
+        /// <code>match.</code>
+        /// <see cref="ParseTreeMatch#labels">labels</see>
+        /// .
+        /// </summary>
+        /// <returns>
+        /// the first node encountered in
+        /// <code>tree</code>
+        /// which does not match
+        /// a corresponding node in
+        /// <code>patternTree</code>
+        /// , or
+        /// <code>null</code>
+        /// if the match
+        /// was successful. The specific node returned depends on the matching
+        /// algorithm used by the implementation, and may be overridden.
+        /// </returns>
+        [Nullable]
+        protected internal virtual IParseTree MatchImpl(IParseTree tree, IParseTree patternTree, MultiMap<string, IParseTree> labels)
+        {
+            if (tree == null)
+            {
+                throw new ArgumentException("tree cannot be null");
+            }
+            if (patternTree == null)
+            {
+                throw new ArgumentException("patternTree cannot be null");
+            }
+            // x and <ID>, x and y, or x and x; or could be mismatched types
+            if (tree is ITerminalNode && patternTree is ITerminalNode)
+            {
+                ITerminalNode t1 = (ITerminalNode)tree;
+                ITerminalNode t2 = (ITerminalNode)patternTree;
+                IParseTree mismatchedNode = null;
+                // both are tokens and they have same type
+                if (t1.Symbol.Type == t2.Symbol.Type)
+                {
+                    if (t2.Symbol is TokenTagToken)
+                    {
+                        // x and <ID>
+                        TokenTagToken tokenTagToken = (TokenTagToken)t2.Symbol;
+                        // track label->list-of-nodes for both token name and label (if any)
+                        labels.Map(tokenTagToken.GetTokenName(), tree);
+                        if (tokenTagToken.GetLabel() != null)
+                        {
+                            labels.Map(tokenTagToken.GetLabel(), tree);
+                        }
+                    }
+                    else
+                    {
+                        if (t1.GetText().Equals(t2.GetText()))
+                        {
+                        }
+                        else
+                        {
+                            // x and x
+                            // x and y
+                            if (mismatchedNode == null)
+                            {
+                                mismatchedNode = t1;
+                            }
+                        }
+                    }
+                }
+                else
+                {
+                    if (mismatchedNode == null)
+                    {
+                        mismatchedNode = t1;
+                    }
+                }
+                return mismatchedNode;
+            }
+            if (tree is ParserRuleContext && patternTree is ParserRuleContext)
+            {
+                ParserRuleContext r1 = (ParserRuleContext)tree;
+                ParserRuleContext r2 = (ParserRuleContext)patternTree;
+                IParseTree mismatchedNode = null;
+                // (expr ...) and <expr>
+                RuleTagToken ruleTagToken = GetRuleTagToken(r2);
+                if (ruleTagToken != null)
+                {
+                    ParseTreeMatch m = null;
+                    if (r1.RuleContext.GetRuleIndex() == r2.RuleContext.GetRuleIndex())
+                    {
+                        // track label->list-of-nodes for both rule name and label (if any)
+                        labels.Map(ruleTagToken.GetRuleName(), tree);
+                        if (ruleTagToken.GetLabel() != null)
+                        {
+                            labels.Map(ruleTagToken.GetLabel(), tree);
+                        }
+                    }
+                    else
+                    {
+                        if (mismatchedNode == null)
+                        {
+                            mismatchedNode = r1;
+                        }
+                    }
+                    return mismatchedNode;
+                }
+                // (expr ...) and (expr ...)
+                if (r1.ChildCount != r2.ChildCount)
+                {
+                    if (mismatchedNode == null)
+                    {
+                        mismatchedNode = r1;
+                    }
+                    return mismatchedNode;
+                }
+                int n = r1.ChildCount;
+                for (int i = 0; i < n; i++)
+                {
+                    IParseTree childMatch = MatchImpl(r1.GetChild(i), patternTree.GetChild(i), labels);
+                    if (childMatch != null)
+                    {
+                        return childMatch;
+                    }
+                }
+                return mismatchedNode;
+            }
+            // if nodes aren't both tokens or both rule nodes, can't match
+            return tree;
+        }
+
+        /// <summary>
+        /// Is
+        /// <code>t</code>
+        /// 
+        /// <code>(expr <expr>)</code>
+        /// subtree?
+        /// </summary>
+        protected internal virtual RuleTagToken GetRuleTagToken(IParseTree t)
+        {
+            if (t is IRuleNode)
+            {
+                IRuleNode r = (IRuleNode)t;
+                if (r.ChildCount == 1 && r.GetChild(0) is ITerminalNode)
+                {
+                    ITerminalNode c = (ITerminalNode)r.GetChild(0);
+                    if (c.Symbol is RuleTagToken)
+                    {
+                        //					System.out.println("rule tag subtree "+t.toStringTree(parser));
+                        return (RuleTagToken)c.Symbol;
+                    }
+                }
+            }
+            return null;
+        }
+
+        public virtual IList<IToken> Tokenize(string pattern)
+        {
+            // split pattern into chunks: sea (raw input) and islands (<ID>, <expr>)
+            IList<Chunk> chunks = Split(pattern);
+            // create token stream from text and tags
+            IList<IToken> tokens = new List<IToken>();
+            foreach (Chunk chunk in chunks)
+            {
+                if (chunk is TagChunk)
+                {
+                    TagChunk tagChunk = (TagChunk)chunk;
+                    // add special rule token or conjure up new token from name
+                    if (System.Char.IsUpper(tagChunk.GetTag()[0]))
+                    {
+                        int ttype = parser.GetTokenType(tagChunk.GetTag());
+                        if (ttype == TokenConstants.InvalidType)
+                        {
+                            throw new ArgumentException("Unknown token " + tagChunk.GetTag() + " in pattern: " + pattern);
+                        }
+                        TokenTagToken t = new TokenTagToken(tagChunk.GetTag(), ttype, tagChunk.GetLabel());
+                        tokens.AddItem(t);
+                    }
+                    else
+                    {
+                        if (System.Char.IsLower(tagChunk.GetTag()[0]))
+                        {
+                            int ruleIndex = parser.GetRuleIndex(tagChunk.GetTag());
+                            if (ruleIndex == -1)
+                            {
+                                throw new ArgumentException("Unknown rule " + tagChunk.GetTag() + " in pattern: " + pattern);
+                            }
+                            int ruleImaginaryTokenType = parser.GetATNWithBypassAlts().ruleToTokenType[ruleIndex];
+                            tokens.AddItem(new RuleTagToken(tagChunk.GetTag(), ruleImaginaryTokenType, tagChunk.GetLabel()));
+                        }
+                        else
+                        {
+                            throw new ArgumentException("invalid tag: " + tagChunk.GetTag() + " in pattern: " + pattern);
+                        }
+                    }
+                }
+                else
+                {
+                    TextChunk textChunk = (TextChunk)chunk;
+                    AntlrInputStream @in = new AntlrInputStream(textChunk.GetText());
+                    lexer.SetInputStream(@in);
+                    IToken t = lexer.NextToken();
+                    while (t.Type != TokenConstants.Eof)
+                    {
+                        tokens.AddItem(t);
+                        t = lexer.NextToken();
+                    }
+                }
+            }
+            //		System.out.println("tokens="+tokens);
+            return tokens;
+        }
+
+        /// <summary>
+        /// Split
+        /// <code><ID> = <e:expr> ;</code>
+        /// into 4 chunks for tokenizing by
+        /// <see cref="Tokenize(string)">Tokenize(string)</see>
+        /// .
+        /// </summary>
+        public virtual IList<Chunk> Split(string pattern)
+        {
+            int p = 0;
+            int n = pattern.Length;
+            IList<Chunk> chunks = new List<Chunk>();
+            StringBuilder buf = new StringBuilder();
+            // find all start and stop indexes first, then collect
+            IList<int> starts = new List<int>();
+            IList<int> stops = new List<int>();
+            while (p < n)
+            {
+                if (p == pattern.IndexOf(escape + start, p))
+                {
+                    p += escape.Length + start.Length;
+                }
+                else
+                {
+                    if (p == pattern.IndexOf(escape + stop, p))
+                    {
+                        p += escape.Length + stop.Length;
+                    }
+                    else
+                    {
+                        if (p == pattern.IndexOf(start, p))
+                        {
+                            starts.AddItem(p);
+                            p += start.Length;
+                        }
+                        else
+                        {
+                            if (p == pattern.IndexOf(stop, p))
+                            {
+                                stops.AddItem(p);
+                                p += stop.Length;
+                            }
+                            else
+                            {
+                                p++;
+                            }
+                        }
+                    }
+                }
+            }
+            //		System.out.println("");
+            //		System.out.println(starts);
+            //		System.out.println(stops);
+            if (starts.Count > stops.Count)
+            {
+                throw new ArgumentException("unterminated tag in pattern: " + pattern);
+            }
+            if (starts.Count < stops.Count)
+            {
+                throw new ArgumentException("missing start tag in pattern: " + pattern);
+            }
+            int ntags = starts.Count;
+            for (int i = 0; i < ntags; i++)
+            {
+                if (starts[i] >= stops[i])
+                {
+                    throw new ArgumentException("tag delimiters out of order in pattern: " + pattern);
+                }
+            }
+            // collect into chunks now
+            if (ntags == 0)
+            {
+                string text = Sharpen.Runtime.Substring(pattern, 0, n);
+                chunks.AddItem(new TextChunk(text));
+            }
+            if (ntags > 0 && starts[0] > 0)
+            {
+                // copy text up to first tag into chunks
+                string text = Sharpen.Runtime.Substring(pattern, 0, starts[0]);
+                chunks.AddItem(new TextChunk(text));
+            }
+            for (int i_1 = 0; i_1 < ntags; i_1++)
+            {
+                // copy inside of <tag>
+                string tag = Sharpen.Runtime.Substring(pattern, starts[i_1] + start.Length, stops[i_1]);
+                string ruleOrToken = tag;
+                string label = null;
+                int colon = tag.IndexOf(':');
+                if (colon >= 0)
+                {
+                    label = Sharpen.Runtime.Substring(tag, 0, colon);
+                    ruleOrToken = Sharpen.Runtime.Substring(tag, colon + 1, tag.Length);
+                }
+                chunks.AddItem(new TagChunk(label, ruleOrToken));
+                if (i_1 + 1 < ntags)
+                {
+                    // copy from end of <tag> to start of next
+                    string text = Sharpen.Runtime.Substring(pattern, stops[i_1] + stop.Length, starts[i_1 + 1]);
+                    chunks.AddItem(new TextChunk(text));
+                }
+            }
+            if (ntags > 0)
+            {
+                int afterLastTag = stops[ntags - 1] + stop.Length;
+                if (afterLastTag < n)
+                {
+                    // copy text from end of last tag to end
+                    string text = Sharpen.Runtime.Substring(pattern, afterLastTag, n);
+                    chunks.AddItem(new TextChunk(text));
+                }
+            }
+            // strip out the escape sequences from text chunks but not tags
+            for (int i_2 = 0; i_2 < chunks.Count; i_2++)
+            {
+                Chunk c = chunks[i_2];
+                if (c is TextChunk)
+                {
+                    TextChunk tc = (TextChunk)c;
+                    string unescaped = tc.GetText().Replace(escape, string.Empty);
+                    if (unescaped.Length < tc.GetText().Length)
+                    {
+                        chunks.Set(i_2, new TextChunk(unescaped));
+                    }
+                }
+            }
+            return chunks;
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/Tree/Pattern/RuleTagToken.cs

@@ -0,0 +1,308 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using System;
+using Antlr4.Runtime;
+using Antlr4.Runtime.Misc;
+using Sharpen;
+
+namespace Antlr4.Runtime.Tree.Pattern
+{
+    /// <summary>
+    /// A
+    /// <see cref="Antlr4.Runtime.IToken">Antlr4.Runtime.IToken</see>
+    /// object representing an entire subtree matched by a parser
+    /// rule; e.g.,
+    /// <code><expr></code>
+    /// . These tokens are created for
+    /// <see cref="TagChunk">TagChunk</see>
+    /// chunks where the tag corresponds to a parser rule.
+    /// </summary>
+    public class RuleTagToken : IToken
+    {
+        /// <summary>
+        /// This is the backing field for
+        /// <see cref="GetRuleName()">GetRuleName()</see>
+        /// .
+        /// </summary>
+        private readonly string ruleName;
+
+        /// <summary>The token type for the current token.</summary>
+        /// <remarks>
+        /// The token type for the current token. This is the token type assigned to
+        /// the bypass alternative for the rule during ATN deserialization.
+        /// </remarks>
+        private readonly int bypassTokenType;
+
+        /// <summary>
+        /// This is the backing field for
+        /// <see cref="GetLabel()">GetLabel()</see>
+        /// .
+        /// </summary>
+        private readonly string label;
+
+        /// <summary>
+        /// Constructs a new instance of
+        /// <see cref="RuleTagToken">RuleTagToken</see>
+        /// with the specified rule
+        /// name and bypass token type and no label.
+        /// </summary>
+        /// <param name="ruleName">The name of the parser rule this rule tag matches.</param>
+        /// <param name="bypassTokenType">The bypass token type assigned to the parser rule.</param>
+        /// <exception>
+        /// IllegalArgumentException
+        /// if
+        /// <code>ruleName</code>
+        /// is
+        /// <code>null</code>
+        /// or empty.
+        /// </exception>
+        public RuleTagToken(string ruleName, int bypassTokenType)
+            : this(ruleName, bypassTokenType, null)
+        {
+        }
+
+        /// <summary>
+        /// Constructs a new instance of
+        /// <see cref="RuleTagToken">RuleTagToken</see>
+        /// with the specified rule
+        /// name, bypass token type, and label.
+        /// </summary>
+        /// <param name="ruleName">The name of the parser rule this rule tag matches.</param>
+        /// <param name="bypassTokenType">The bypass token type assigned to the parser rule.</param>
+        /// <param name="label">
+        /// The label associated with the rule tag, or
+        /// <code>null</code>
+        /// if
+        /// the rule tag is unlabeled.
+        /// </param>
+        /// <exception>
+        /// IllegalArgumentException
+        /// if
+        /// <code>ruleName</code>
+        /// is
+        /// <code>null</code>
+        /// or empty.
+        /// </exception>
+        public RuleTagToken(string ruleName, int bypassTokenType, string label)
+        {
+            if (ruleName == null || ruleName.IsEmpty())
+            {
+                throw new ArgumentException("ruleName cannot be null or empty.");
+            }
+            this.ruleName = ruleName;
+            this.bypassTokenType = bypassTokenType;
+            this.label = label;
+        }
+
+        /// <summary>Gets the name of the rule associated with this rule tag.</summary>
+        /// <remarks>Gets the name of the rule associated with this rule tag.</remarks>
+        /// <returns>The name of the parser rule associated with this rule tag.</returns>
+        [NotNull]
+        public string GetRuleName()
+        {
+            return ruleName;
+        }
+
+        /// <summary>Gets the label associated with the rule tag.</summary>
+        /// <remarks>Gets the label associated with the rule tag.</remarks>
+        /// <returns>
+        /// The name of the label associated with the rule tag, or
+        /// <code>null</code>
+        /// if this is an unlabeled rule tag.
+        /// </returns>
+        [Nullable]
+        public string GetLabel()
+        {
+            return label;
+        }
+
+        /// <summary>
+        /// <inheritDoc></inheritDoc>
+        /// <p>Rule tag tokens are always placed on the
+        /// <see cref="Antlr4.Runtime.IToken.DefaultChannel">Antlr4.Runtime.IToken.DefaultChannel</see>
+        /// .</p>
+        /// </summary>
+        public virtual int Channel
+        {
+            get
+            {
+                return TokenConstants.DefaultChannel;
+            }
+        }
+
+        /// <summary>
+        /// <inheritDoc></inheritDoc>
+        /// <p>This method returns the rule tag formatted with
+        /// <code>&lt;</code>
+        /// and
+        /// <code>&gt;</code>
+        /// delimiters.</p>
+        /// </summary>
+        public virtual string Text
+        {
+            get
+            {
+                if (label != null)
+                {
+                    return "<" + label + ":" + ruleName + ">";
+                }
+                return "<" + ruleName + ">";
+            }
+        }
+
+        /// <summary>
+        /// <inheritDoc></inheritDoc>
+        /// <p>Rule tag tokens have types assigned according to the rule bypass
+        /// transitions created during ATN deserialization.</p>
+        /// </summary>
+        public virtual int Type
+        {
+            get
+            {
+                return bypassTokenType;
+            }
+        }
+
+        /// <summary>
+        /// <inheritDoc></inheritDoc>
+        /// <p>The implementation for
+        /// <see cref="RuleTagToken">RuleTagToken</see>
+        /// always returns 0.</p>
+        /// </summary>
+        public virtual int Line
+        {
+            get
+            {
+                return 0;
+            }
+        }
+
+        /// <summary>
+        /// <inheritDoc></inheritDoc>
+        /// <p>The implementation for
+        /// <see cref="RuleTagToken">RuleTagToken</see>
+        /// always returns -1.</p>
+        /// </summary>
+        public virtual int Column
+        {
+            get
+            {
+                return -1;
+            }
+        }
+
+        /// <summary>
+        /// <inheritDoc></inheritDoc>
+        /// <p>The implementation for
+        /// <see cref="RuleTagToken">RuleTagToken</see>
+        /// always returns -1.</p>
+        /// </summary>
+        public virtual int TokenIndex
+        {
+            get
+            {
+                return -1;
+            }
+        }
+
+        /// <summary>
+        /// <inheritDoc></inheritDoc>
+        /// <p>The implementation for
+        /// <see cref="RuleTagToken">RuleTagToken</see>
+        /// always returns -1.</p>
+        /// </summary>
+        public virtual int StartIndex
+        {
+            get
+            {
+                return -1;
+            }
+        }
+
+        /// <summary>
+        /// <inheritDoc></inheritDoc>
+        /// <p>The implementation for
+        /// <see cref="RuleTagToken">RuleTagToken</see>
+        /// always returns -1.</p>
+        /// </summary>
+        public virtual int StopIndex
+        {
+            get
+            {
+                return -1;
+            }
+        }
+
+        /// <summary>
+        /// <inheritDoc></inheritDoc>
+        /// <p>The implementation for
+        /// <see cref="RuleTagToken">RuleTagToken</see>
+        /// always returns
+        /// <code>null</code>
+        /// .</p>
+        /// </summary>
+        public virtual ITokenSource TokenSource
+        {
+            get
+            {
+                return null;
+            }
+        }
+
+        /// <summary>
+        /// <inheritDoc></inheritDoc>
+        /// <p>The implementation for
+        /// <see cref="RuleTagToken">RuleTagToken</see>
+        /// always returns
+        /// <code>null</code>
+        /// .</p>
+        /// </summary>
+        public virtual ICharStream InputStream
+        {
+            get
+            {
+                return null;
+            }
+        }
+
+        /// <summary>
+        /// <inheritDoc></inheritDoc>
+        /// <p>The implementation for
+        /// <see cref="RuleTagToken">RuleTagToken</see>
+        /// returns a string of the form
+        /// <code>ruleName:bypassTokenType</code>
+        /// .</p>
+        /// </summary>
+        public override string ToString()
+        {
+            return ruleName + ":" + bypassTokenType;
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/Tree/Pattern/TagChunk.cs

@@ -0,0 +1,182 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using System;
+using Antlr4.Runtime.Misc;
+using Antlr4.Runtime.Tree.Pattern;
+using Sharpen;
+
+namespace Antlr4.Runtime.Tree.Pattern
+{
+    /// <summary>Represents a placeholder tag in a tree pattern.</summary>
+    /// <remarks>
+    /// Represents a placeholder tag in a tree pattern. A tag can have any of the
+    /// following forms.
+    /// <ul>
+    /// <li>
+    /// <code>expr</code>
+    /// : An unlabeled placeholder for a parser rule
+    /// <code>expr</code>
+    /// .</li>
+    /// <li>
+    /// <code>ID</code>
+    /// : An unlabeled placeholder for a token of type
+    /// <code>ID</code>
+    /// .</li>
+    /// <li>
+    /// <code>e:expr</code>
+    /// : A labeled placeholder for a parser rule
+    /// <code>expr</code>
+    /// .</li>
+    /// <li>
+    /// <code>id:ID</code>
+    /// : A labeled placeholder for a token of type
+    /// <code>ID</code>
+    /// .</li>
+    /// </ul>
+    /// This class does not perform any validation on the tag or label names aside
+    /// from ensuring that the tag is a non-null, non-empty string.
+    /// </remarks>
+    internal class TagChunk : Chunk
+    {
+        /// <summary>
+        /// This is the backing field for
+        /// <see cref="GetTag()">GetTag()</see>
+        /// .
+        /// </summary>
+        private readonly string tag;
+
+        /// <summary>
+        /// This is the backing field for
+        /// <see cref="GetLabel()">GetLabel()</see>
+        /// .
+        /// </summary>
+        private readonly string label;
+
+        /// <summary>
+        /// Construct a new instance of
+        /// <see cref="TagChunk">TagChunk</see>
+        /// using the specified tag and
+        /// no label.
+        /// </summary>
+        /// <param name="tag">
+        /// The tag, which should be the name of a parser rule or token
+        /// type.
+        /// </param>
+        /// <exception>
+        /// IllegalArgumentException
+        /// if
+        /// <code>tag</code>
+        /// is
+        /// <code>null</code>
+        /// or
+        /// empty.
+        /// </exception>
+        public TagChunk(string tag)
+            : this(null, tag)
+        {
+        }
+
+        /// <summary>
+        /// Construct a new instance of
+        /// <see cref="TagChunk">TagChunk</see>
+        /// using the specified label
+        /// and tag.
+        /// </summary>
+        /// <param name="label">
+        /// The label for the tag. If this is
+        /// <code>null</code>
+        /// , the
+        /// <see cref="TagChunk">TagChunk</see>
+        /// represents an unlabeled tag.
+        /// </param>
+        /// <param name="tag">
+        /// The tag, which should be the name of a parser rule or token
+        /// type.
+        /// </param>
+        /// <exception>
+        /// IllegalArgumentException
+        /// if
+        /// <code>tag</code>
+        /// is
+        /// <code>null</code>
+        /// or
+        /// empty.
+        /// </exception>
+        public TagChunk(string label, string tag)
+        {
+            if (tag == null || tag.IsEmpty())
+            {
+                throw new ArgumentException("tag cannot be null or empty");
+            }
+            this.label = label;
+            this.tag = tag;
+        }
+
+        /// <summary>Get the tag for this chunk.</summary>
+        /// <remarks>Get the tag for this chunk.</remarks>
+        /// <returns>The tag for the chunk.</returns>
+        [NotNull]
+        public string GetTag()
+        {
+            return tag;
+        }
+
+        /// <summary>Get the label, if any, assigned to this chunk.</summary>
+        /// <remarks>Get the label, if any, assigned to this chunk.</remarks>
+        /// <returns>
+        /// The label assigned to this chunk, or
+        /// <code>null</code>
+        /// if no label is
+        /// assigned to the chunk.
+        /// </returns>
+        [Nullable]
+        public string GetLabel()
+        {
+            return label;
+        }
+
+        /// <summary>This method returns a text representation of the tag chunk.</summary>
+        /// <remarks>
+        /// This method returns a text representation of the tag chunk. Labeled tags
+        /// are returned in the form
+        /// <code>label:tag</code>
+        /// , and unlabeled tags are
+        /// returned as just the tag name.
+        /// </remarks>
+        public override string ToString()
+        {
+            if (label != null)
+            {
+                return label + ":" + tag;
+            }
+            return tag;
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/Tree/Pattern/TextChunk.cs

@@ -0,0 +1,100 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using System;
+using Antlr4.Runtime.Misc;
+using Antlr4.Runtime.Tree.Pattern;
+using Sharpen;
+
+namespace Antlr4.Runtime.Tree.Pattern
+{
+    /// <summary>
+    /// Represents a span of raw text (concrete syntax) between tags in a tree
+    /// pattern string.
+    /// </summary>
+    /// <remarks>
+    /// Represents a span of raw text (concrete syntax) between tags in a tree
+    /// pattern string.
+    /// </remarks>
+    internal class TextChunk : Chunk
+    {
+        /// <summary>
+        /// This is the backing field for
+        /// <see cref="GetText()">GetText()</see>
+        /// .
+        /// </summary>
+        [NotNull]
+        private readonly string text;
+
+        /// <summary>
+        /// Constructs a new instance of
+        /// <see cref="TextChunk">TextChunk</see>
+        /// with the specified text.
+        /// </summary>
+        /// <param name="text">The text of this chunk.</param>
+        /// <exception>
+        /// IllegalArgumentException
+        /// if
+        /// <code>text</code>
+        /// is
+        /// <code>null</code>
+        /// .
+        /// </exception>
+        public TextChunk(string text)
+        {
+            if (text == null)
+            {
+                throw new ArgumentException("text cannot be null");
+            }
+            this.text = text;
+        }
+
+        /// <summary>Gets the raw text of this chunk.</summary>
+        /// <remarks>Gets the raw text of this chunk.</remarks>
+        /// <returns>The text of the chunk.</returns>
+        [NotNull]
+        public string GetText()
+        {
+            return text;
+        }
+
+        /// <summary>
+        /// <inheritDoc></inheritDoc>
+        /// <p>The implementation for
+        /// <see cref="TextChunk">TextChunk</see>
+        /// returns the result of
+        /// <see cref="GetText()">GetText()</see>
+        /// in single quotes.</p>
+        /// </summary>
+        public override string ToString()
+        {
+            return "'" + text + "'";
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/Tree/Pattern/TokenTagToken.cs

@@ -0,0 +1,157 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using Antlr4.Runtime;
+using Antlr4.Runtime.Misc;
+using Sharpen;
+
+namespace Antlr4.Runtime.Tree.Pattern
+{
+    /// <summary>
+    /// A
+    /// <see cref="Antlr4.Runtime.IToken">Antlr4.Runtime.IToken</see>
+    /// object representing a token of a particular type; e.g.,
+    /// <code><ID></code>
+    /// . These tokens are created for
+    /// <see cref="TagChunk">TagChunk</see>
+    /// chunks where the
+    /// tag corresponds to a lexer rule or token type.
+    /// </summary>
+    [System.Serializable]
+    public class TokenTagToken : CommonToken
+    {
+        /// <summary>
+        /// This is the backing field for
+        /// <see cref="GetTokenName()">GetTokenName()</see>
+        /// .
+        /// </summary>
+        [NotNull]
+        private readonly string tokenName;
+
+        /// <summary>
+        /// This is the backing field for
+        /// <see cref="GetLabel()">GetLabel()</see>
+        /// .
+        /// </summary>
+        [Nullable]
+        private readonly string label;
+
+        /// <summary>
+        /// Constructs a new instance of
+        /// <see cref="TokenTagToken">TokenTagToken</see>
+        /// for an unlabeled tag
+        /// with the specified token name and type.
+        /// </summary>
+        /// <param name="tokenName">The token name.</param>
+        /// <param name="type">The token type.</param>
+        public TokenTagToken(string tokenName, int type)
+            : this(tokenName, type, null)
+        {
+        }
+
+        /// <summary>
+        /// Constructs a new instance of
+        /// <see cref="TokenTagToken">TokenTagToken</see>
+        /// with the specified
+        /// token name, type, and label.
+        /// </summary>
+        /// <param name="tokenName">The token name.</param>
+        /// <param name="type">The token type.</param>
+        /// <param name="label">
+        /// The label associated with the token tag, or
+        /// <code>null</code>
+        /// if
+        /// the token tag is unlabeled.
+        /// </param>
+        public TokenTagToken(string tokenName, int type, string label)
+            : base(type)
+        {
+            this.tokenName = tokenName;
+            this.label = label;
+        }
+
+        /// <summary>Gets the token name.</summary>
+        /// <remarks>Gets the token name.</remarks>
+        /// <returns>The token name.</returns>
+        [NotNull]
+        public string GetTokenName()
+        {
+            return tokenName;
+        }
+
+        /// <summary>Gets the label associated with the rule tag.</summary>
+        /// <remarks>Gets the label associated with the rule tag.</remarks>
+        /// <returns>
+        /// The name of the label associated with the rule tag, or
+        /// <code>null</code>
+        /// if this is an unlabeled rule tag.
+        /// </returns>
+        [Nullable]
+        public string GetLabel()
+        {
+            return label;
+        }
+
+        /// <summary>
+        /// <inheritDoc></inheritDoc>
+        /// <p>The implementation for
+        /// <see cref="TokenTagToken">TokenTagToken</see>
+        /// returns the token tag
+        /// formatted with
+        /// <code>&lt;</code>
+        /// and
+        /// <code>&gt;</code>
+        /// delimiters.</p>
+        /// </summary>
+        public override string Text
+        {
+            get
+            {
+                if (label != null)
+                {
+                    return "<" + label + ":" + tokenName + ">";
+                }
+                return "<" + tokenName + ">";
+            }
+        }
+
+        /// <summary>
+        /// <inheritDoc></inheritDoc>
+        /// <p>The implementation for
+        /// <see cref="TokenTagToken">TokenTagToken</see>
+        /// returns a string of the form
+        /// <code>tokenName:type</code>
+        /// .</p>
+        /// </summary>
+        public override string ToString()
+        {
+            return tokenName + ":" + type;
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/Tree/Trees.cs

@@ -146,6 +146,17 @@ namespace Antlr4.Runtime.Tree
             return t.Payload.ToString();
         }
 
+        /// <summary>Return ordered list of all children of this node</summary>
+        public static IList<ITree> GetChildren(ITree t)
+        {
+            IList<ITree> kids = new List<ITree>();
+            for (int i = 0; i < t.ChildCount; i++)
+            {
+                kids.AddItem(t.GetChild(i));
+            }
+            return kids;
+        }
+
         /// <summary>Return a list of all ancestors of this node.</summary>
         /// <remarks>
         /// Return a list of all ancestors of this node.  The first node of
@@ -169,6 +180,64 @@ namespace Antlr4.Runtime.Tree
             return ancestors;
         }
 
+        public static ICollection<IParseTree> FindAllTokenNodes(IParseTree t, int ttype)
+        {
+            return FindAllNodes(t, ttype, true);
+        }
+
+        public static ICollection<IParseTree> FindAllRuleNodes(IParseTree t, int ruleIndex)
+        {
+            return FindAllNodes(t, ruleIndex, false);
+        }
+
+        public static IList<IParseTree> FindAllNodes(IParseTree t, int index, bool findTokens)
+        {
+            IList<IParseTree> nodes = new List<IParseTree>();
+            _findAllNodes(t, index, findTokens, nodes);
+            return nodes;
+        }
+
+        public static void _findAllNodes<_T0>(IParseTree t, int index, bool findTokens, IList<_T0> nodes)
+        {
+            // check this node (the root) first
+            if (findTokens && t is ITerminalNode)
+            {
+                ITerminalNode tnode = (ITerminalNode)t;
+                if (tnode.Symbol.Type == index)
+                {
+                    nodes.AddItem(t);
+                }
+            }
+            else
+            {
+                if (!findTokens && t is ParserRuleContext)
+                {
+                    ParserRuleContext ctx = (ParserRuleContext)t;
+                    if (ctx.GetRuleIndex() == index)
+                    {
+                        nodes.AddItem(t);
+                    }
+                }
+            }
+            // check children
+            for (int i = 0; i < t.ChildCount; i++)
+            {
+                _findAllNodes(t.GetChild(i), index, findTokens, nodes);
+            }
+        }
+
+        public static IList<IParseTree> Descendants(IParseTree t)
+        {
+            IList<IParseTree> nodes = new List<IParseTree>();
+            nodes.AddItem(t);
+            int n = t.ChildCount;
+            for (int i = 0; i < n; i++)
+            {
+                Sharpen.Collections.AddAll(nodes, Descendants(t.GetChild(i)));
+            }
+            return nodes;
+        }
+
         private Trees()
         {
         }

runtime/CSharp/Antlr4.Runtime/Tree/Xpath/XPath.cs

@@ -0,0 +1,305 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using System;
+using System.Collections.Generic;
+using System.IO;
+using Antlr4.Runtime;
+using Antlr4.Runtime.Tree;
+using Antlr4.Runtime.Tree.Xpath;
+using Sharpen;
+
+namespace Antlr4.Runtime.Tree.Xpath
+{
+    /// <summary>
+    /// Represent a subset of XPath XML path syntax for use in identifying nodes in
+    /// parse trees.
+    /// </summary>
+    /// <remarks>
+    /// Represent a subset of XPath XML path syntax for use in identifying nodes in
+    /// parse trees.
+    /// <p>
+    /// Split path into words and separators
+    /// <code>/</code>
+    /// and
+    /// <code>//</code>
+    /// via ANTLR
+    /// itself then walk path elements from left to right. At each separator-word
+    /// pair, find set of nodes. Next stage uses those as work list.</p>
+    /// <p>
+    /// The basic interface is
+    /// <see cref="FindAll(Antlr4.Runtime.Tree.IParseTree, string, Antlr4.Runtime.Parser)">ParseTree.findAll</see>
+    /// <code>(tree, pathString, parser)</code>
+    /// .
+    /// But that is just shorthand for:</p>
+    /// <pre>
+    /// <see cref="XPath">XPath</see>
+    /// p = new
+    /// <see cref="XPath(Antlr4.Runtime.Parser, string)">XPath</see>
+    /// (parser, pathString);
+    /// return p.
+    /// <see cref="Evaluate(Antlr4.Runtime.Tree.IParseTree)">evaluate</see>
+    /// (tree);
+    /// </pre>
+    /// <p>
+    /// See
+    /// <code>org.antlr.v4.test.TestXPath</code>
+    /// for descriptions. In short, this
+    /// allows operators:</p>
+    /// <dl>
+    /// <dt>/</dt> <dd>root</dd>
+    /// <dt>//</dt> <dd>anywhere</dd>
+    /// <dt>!</dt> <dd>invert; this must appear directly after root or anywhere
+    /// operator</dd>
+    /// </dl>
+    /// <p>
+    /// and path elements:</p>
+    /// <dl>
+    /// <dt>ID</dt> <dd>token name</dd>
+    /// <dt>'string'</dt> <dd>any string literal token from the grammar</dd>
+    /// <dt>expr</dt> <dd>rule name</dd>
+    /// <dt>*</dt> <dd>wildcard matching any node</dd>
+    /// </dl>
+    /// <p>
+    /// Whitespace is not allowed.</p>
+    /// </remarks>
+    public class XPath
+    {
+        public const string Wildcard = "*";
+
+        public const string Not = "!";
+
+        protected internal string path;
+
+        protected internal XPathElement[] elements;
+
+        protected internal Parser parser;
+
+        public XPath(Parser parser, string path)
+        {
+            // word not operator/separator
+            // word for invert operator
+            this.parser = parser;
+            this.path = path;
+            elements = Split(path);
+        }
+
+        //		System.out.println(Arrays.toString(elements));
+        // TODO: check for invalid token/rule names, bad syntax
+        public virtual XPathElement[] Split(string path)
+        {
+            AntlrInputStream @in;
+            try
+            {
+                @in = new AntlrInputStream(new StringReader(path));
+            }
+            catch (IOException ioe)
+            {
+                throw new ArgumentException("Could not read path: " + path, ioe);
+            }
+            XPathLexer lexer = new _XPathLexer_87(@in);
+            lexer.RemoveErrorListeners();
+            lexer.AddErrorListener(new XPathLexerErrorListener());
+            CommonTokenStream tokenStream = new CommonTokenStream(lexer);
+            try
+            {
+                tokenStream.Fill();
+            }
+            catch (LexerNoViableAltException e)
+            {
+                int pos = lexer.Column;
+                string msg = "Invalid tokens or characters at index " + pos + " in path '" + path + "'";
+                throw new ArgumentException(msg, e);
+            }
+            IList<IToken> tokens = tokenStream.GetTokens();
+            //		System.out.println("path="+path+"=>"+tokens);
+            IList<XPathElement> elements = new List<XPathElement>();
+            int n = tokens.Count;
+            int i = 0;
+            while (i < n)
+            {
+                IToken el = tokens[i];
+                IToken next = null;
+                switch (el.Type)
+                {
+                    case XPathLexer.Root:
+                    case XPathLexer.Anywhere:
+                    {
+                        bool anywhere = el.Type == XPathLexer.Anywhere;
+                        i++;
+                        next = tokens[i];
+                        bool invert = next.Type == XPathLexer.Bang;
+                        if (invert)
+                        {
+                            i++;
+                            next = tokens[i];
+                        }
+                        XPathElement pathElement = GetXPathElement(next, anywhere);
+                        pathElement.invert = invert;
+                        elements.AddItem(pathElement);
+                        i++;
+                        break;
+                    }
+
+                    case XPathLexer.TokenRef:
+                    case XPathLexer.RuleRef:
+                    case XPathLexer.Wildcard:
+                    {
+                        elements.AddItem(GetXPathElement(el, false));
+                        i++;
+                        break;
+                    }
+
+                    case TokenConstants.Eof:
+                    {
+                        goto loop_break;
+                    }
+
+                    default:
+                    {
+                        throw new ArgumentException("Unknowth path element " + el);
+                    }
+                }
+loop_continue: ;
+            }
+loop_break: ;
+            return Sharpen.Collections.ToArray(elements, new XPathElement[0]);
+        }
+
+        private sealed class _XPathLexer_87 : XPathLexer
+        {
+            public _XPathLexer_87(ICharStream baseArg1)
+                : base(baseArg1)
+            {
+            }
+
+            public override void Recover(LexerNoViableAltException e)
+            {
+                throw e;
+            }
+        }
+
+        /// <summary>
+        /// Convert word like
+        /// <code>*</code>
+        /// or
+        /// <code>ID</code>
+        /// or
+        /// <code>expr</code>
+        /// to a path
+        /// element.
+        /// <code>anywhere</code>
+        /// is
+        /// <code>true</code>
+        /// if
+        /// <code>//</code>
+        /// precedes the
+        /// word.
+        /// </summary>
+        protected internal virtual XPathElement GetXPathElement(IToken wordToken, bool anywhere)
+        {
+            if (wordToken.Type == TokenConstants.Eof)
+            {
+                throw new ArgumentException("Missing path element at end of path");
+            }
+            string word = wordToken.Text;
+            int ttype = parser.GetTokenType(word);
+            int ruleIndex = parser.GetRuleIndex(word);
+            switch (wordToken.Type)
+            {
+                case XPathLexer.Wildcard:
+                {
+                    return anywhere ? new XPathWildcardAnywhereElement() : new XPathWildcardElement();
+                }
+
+                case XPathLexer.TokenRef:
+                case XPathLexer.String:
+                {
+                    if (ttype == TokenConstants.InvalidType)
+                    {
+                        throw new ArgumentException(word + " at index " + wordToken.StartIndex + " isn't a valid token name");
+                    }
+                    return anywhere ? new XPathTokenAnywhereElement(word, ttype) : new XPathTokenElement(word, ttype);
+                }
+
+                default:
+                {
+                    if (ruleIndex == -1)
+                    {
+                        throw new ArgumentException(word + " at index " + wordToken.StartIndex + " isn't a valid rule name");
+                    }
+                    return anywhere ? new XPathRuleAnywhereElement(word, ruleIndex) : new XPathRuleElement(word, ruleIndex);
+                }
+            }
+        }
+
+        public static ICollection<IParseTree> FindAll(IParseTree tree, string xpath, Parser parser)
+        {
+            Antlr4.Runtime.Tree.Xpath.XPath p = new Antlr4.Runtime.Tree.Xpath.XPath(parser, xpath);
+            return p.Evaluate(tree);
+        }
+
+        /// <summary>
+        /// Return a list of all nodes starting at
+        /// <code>t</code>
+        /// as root that satisfy the
+        /// path. The root
+        /// <code>/</code>
+        /// is relative to the node passed to
+        /// <see cref="Evaluate(Antlr4.Runtime.Tree.IParseTree)">Evaluate(Antlr4.Runtime.Tree.IParseTree)</see>
+        /// .
+        /// </summary>
+        public virtual ICollection<IParseTree> Evaluate(IParseTree t)
+        {
+            ParserRuleContext dummyRoot = new ParserRuleContext();
+            dummyRoot.children = Sharpen.Collections.SingletonList(t);
+            // don't set t's parent.
+            ICollection<IParseTree> work = Sharpen.Collections.Singleton<IParseTree>(dummyRoot);
+            int i = 0;
+            while (i < elements.Length)
+            {
+                ICollection<IParseTree> next = new LinkedHashSet<IParseTree>();
+                foreach (IParseTree node in work)
+                {
+                    if (node.ChildCount > 0)
+                    {
+                        // only try to match next element if it has children
+                        // e.g., //func/*/stat might have a token node for which
+                        // we can't go looking for stat nodes.
+                        ICollection<IParseTree> matching = elements[i].Evaluate(node);
+                        Sharpen.Collections.AddAll(next, matching);
+                    }
+                }
+                i++;
+                work = next;
+            }
+            return work;
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/Tree/Xpath/XPathElement.cs

@@ -0,0 +1,71 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using System.Collections.Generic;
+using Antlr4.Runtime.Tree;
+using Sharpen;
+
+namespace Antlr4.Runtime.Tree.Xpath
+{
+    public abstract class XPathElement
+    {
+        protected internal string nodeName;
+
+        protected internal bool invert;
+
+        /// <summary>
+        /// Construct element like
+        /// <code>/ID</code>
+        /// or
+        /// <code>ID</code>
+        /// or
+        /// <code>/*</code>
+        /// etc...
+        /// op is null if just node
+        /// </summary>
+        public XPathElement(string nodeName)
+        {
+            this.nodeName = nodeName;
+        }
+
+        /// <summary>
+        /// Given tree rooted at
+        /// <code>t</code>
+        /// return all nodes matched by this path
+        /// element.
+        /// </summary>
+        public abstract ICollection<IParseTree> Evaluate(IParseTree t);
+
+        public override string ToString()
+        {
+            string inv = invert ? "!" : string.Empty;
+            return GetType().Name + "[" + inv + nodeName + "]";
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/Tree/Xpath/XPathLexerErrorListener.cs

@@ -0,0 +1,43 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using Antlr4.Runtime;
+using Antlr4.Runtime.Tree.Xpath;
+using Sharpen;
+
+namespace Antlr4.Runtime.Tree.Xpath
+{
+    public class XPathLexerErrorListener : IAntlrErrorListener<int>
+    {
+        public virtual void SyntaxError<T>(Recognizer<T, object> recognizer, T offendingSymbol, int line, int charPositionInLine, string msg, RecognitionException e)
+            where T : int
+        {
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/Tree/Xpath/XPathRuleAnywhereElement.cs

@@ -0,0 +1,59 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using System.Collections.Generic;
+using Antlr4.Runtime.Tree;
+using Antlr4.Runtime.Tree.Xpath;
+using Sharpen;
+
+namespace Antlr4.Runtime.Tree.Xpath
+{
+    /// <summary>
+    /// Either
+    /// <code>ID</code>
+    /// at start of path or
+    /// <code>...//ID</code>
+    /// in middle of path.
+    /// </summary>
+    public class XPathRuleAnywhereElement : XPathElement
+    {
+        protected internal int ruleIndex;
+
+        public XPathRuleAnywhereElement(string ruleName, int ruleIndex)
+            : base(ruleName)
+        {
+            this.ruleIndex = ruleIndex;
+        }
+
+        public override ICollection<IParseTree> Evaluate(IParseTree t)
+        {
+            return Trees.FindAllRuleNodes(t, ruleIndex);
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/Tree/Xpath/XPathRuleElement.cs

@@ -0,0 +1,66 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using System.Collections.Generic;
+using Antlr4.Runtime;
+using Antlr4.Runtime.Tree;
+using Antlr4.Runtime.Tree.Xpath;
+using Sharpen;
+
+namespace Antlr4.Runtime.Tree.Xpath
+{
+    public class XPathRuleElement : XPathElement
+    {
+        protected internal int ruleIndex;
+
+        public XPathRuleElement(string ruleName, int ruleIndex)
+            : base(ruleName)
+        {
+            this.ruleIndex = ruleIndex;
+        }
+
+        public override ICollection<IParseTree> Evaluate(IParseTree t)
+        {
+            // return all children of t that match nodeName
+            IList<IParseTree> nodes = new List<IParseTree>();
+            foreach (ITree c in Trees.GetChildren(t))
+            {
+                if (c is ParserRuleContext)
+                {
+                    ParserRuleContext ctx = (ParserRuleContext)c;
+                    if ((ctx.GetRuleIndex() == ruleIndex && !invert) || (ctx.GetRuleIndex() != ruleIndex && invert))
+                    {
+                        nodes.AddItem(ctx);
+                    }
+                }
+            }
+            return nodes;
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/Tree/Xpath/XPathTokenAnywhereElement.cs

@@ -0,0 +1,52 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using System.Collections.Generic;
+using Antlr4.Runtime.Tree;
+using Antlr4.Runtime.Tree.Xpath;
+using Sharpen;
+
+namespace Antlr4.Runtime.Tree.Xpath
+{
+    public class XPathTokenAnywhereElement : XPathElement
+    {
+        protected internal int tokenType;
+
+        public XPathTokenAnywhereElement(string tokenName, int tokenType)
+            : base(tokenName)
+        {
+            this.tokenType = tokenType;
+        }
+
+        public override ICollection<IParseTree> Evaluate(IParseTree t)
+        {
+            return Trees.FindAllTokenNodes(t, tokenType);
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/Tree/Xpath/XPathTokenElement.cs

@@ -0,0 +1,65 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using System.Collections.Generic;
+using Antlr4.Runtime.Tree;
+using Antlr4.Runtime.Tree.Xpath;
+using Sharpen;
+
+namespace Antlr4.Runtime.Tree.Xpath
+{
+    public class XPathTokenElement : XPathElement
+    {
+        protected internal int tokenType;
+
+        public XPathTokenElement(string tokenName, int tokenType)
+            : base(tokenName)
+        {
+            this.tokenType = tokenType;
+        }
+
+        public override ICollection<IParseTree> Evaluate(IParseTree t)
+        {
+            // return all children of t that match nodeName
+            IList<IParseTree> nodes = new List<IParseTree>();
+            foreach (ITree c in Trees.GetChildren(t))
+            {
+                if (c is ITerminalNode)
+                {
+                    ITerminalNode tnode = (ITerminalNode)c;
+                    if ((tnode.Symbol.Type == tokenType && !invert) || (tnode.Symbol.Type != tokenType && invert))
+                    {
+                        nodes.AddItem(tnode);
+                    }
+                }
+            }
+            return nodes;
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/Tree/Xpath/XPathWildcardAnywhereElement.cs

@@ -0,0 +1,54 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using System.Collections.Generic;
+using Antlr4.Runtime.Tree;
+using Antlr4.Runtime.Tree.Xpath;
+using Sharpen;
+
+namespace Antlr4.Runtime.Tree.Xpath
+{
+    public class XPathWildcardAnywhereElement : XPathElement
+    {
+        public XPathWildcardAnywhereElement()
+            : base(XPath.Wildcard)
+        {
+        }
+
+        public override ICollection<IParseTree> Evaluate(IParseTree t)
+        {
+            if (invert)
+            {
+                return new List<IParseTree>();
+            }
+            // !* is weird but valid (empty)
+            return Trees.Descendants(t);
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/Tree/Xpath/XPathWildcardElement.cs

@@ -0,0 +1,59 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+using System.Collections.Generic;
+using Antlr4.Runtime.Tree;
+using Antlr4.Runtime.Tree.Xpath;
+using Sharpen;
+
+namespace Antlr4.Runtime.Tree.Xpath
+{
+    public class XPathWildcardElement : XPathElement
+    {
+        public XPathWildcardElement()
+            : base(XPath.Wildcard)
+        {
+        }
+
+        public override ICollection<IParseTree> Evaluate(IParseTree t)
+        {
+            if (invert)
+            {
+                return new List<IParseTree>();
+            }
+            // !* is weird but valid (empty)
+            IList<IParseTree> kids = new List<IParseTree>();
+            foreach (ITree c in Trees.GetChildren(t))
+            {
+                kids.AddItem((IParseTree)c);
+            }
+            return kids;
+        }
+    }
+}

runtime/CSharp/Antlr4.Runtime/UnbufferedCharStream.cs

@@ -58,10 +58,9 @@ namespace Antlr4.Runtime
         /// The number of characters currently in
         /// <see cref="data">data</see>
         /// .
-        /// <p/>
-        /// This is not the buffer capacity, that's
+        /// <p>This is not the buffer capacity, that's
         /// <code>data.length</code>
-        /// .
+        /// .</p>
         /// </summary>
         protected internal int n;
 
@@ -69,15 +68,14 @@ namespace Antlr4.Runtime
         /// 0..n-1 index into
         /// <see cref="data">data</see>
         /// of next character.
-        /// <p/>
-        /// The
+        /// <p>The
         /// <code>LA(1)</code>
         /// character is
         /// <code>data[p]</code>
         /// . If
         /// <code>p == n</code>
         /// , we are
-        /// out of buffered characters.
+        /// out of buffered characters.</p>
         /// </summary>
         protected internal int p = 0;
 
@@ -303,13 +301,12 @@ namespace Antlr4.Runtime
         /// <summary>Return a marker that we can release later.</summary>
         /// <remarks>
         /// Return a marker that we can release later.
-        /// <p/>
-        /// The specific marker value used for this class allows for some level of
+        /// <p>The specific marker value used for this class allows for some level of
         /// protection against misuse where
         /// <code>seek()</code>
         /// is called on a mark or
         /// <code>release()</code>
-        /// is called in the wrong order.
+        /// is called in the wrong order.</p>
         /// </remarks>
         public virtual int Mark()
         {

runtime/CSharp/Antlr4.Runtime/UnbufferedTokenStream.cs

@@ -53,10 +53,9 @@ namespace Antlr4.Runtime
         /// The number of tokens currently in
         /// <see cref="tokens">tokens</see>
         /// .
-        /// <p/>
-        /// This is not the buffer capacity, that's
+        /// <p>This is not the buffer capacity, that's
         /// <code>tokens.length</code>
-        /// .
+        /// .</p>
         /// </summary>
         protected internal int n;
 
@@ -64,15 +63,14 @@ namespace Antlr4.Runtime
         /// 0..n-1 index into
         /// <see cref="tokens">tokens</see>
         /// of next token.
-        /// <p/>
-        /// The
+        /// <p>The
         /// <code>LT(1)</code>
         /// token is
         /// <code>tokens[p]</code>
         /// . If
         /// <code>p == n</code>
         /// , we are
-        /// out of buffered tokens.
+        /// out of buffered tokens.</p>
         /// </summary>
         protected internal int p = 0;
 
@@ -120,11 +118,10 @@ namespace Antlr4.Runtime
         /// <code>LT(1)</code>
         /// . Goes from 0 to the number of tokens in the entire stream,
         /// although the stream size is unknown before the end is reached.
-        /// <p/>
-        /// This value is used to set the token indexes if the stream provides tokens
+        /// <p>This value is used to set the token indexes if the stream provides tokens
         /// that implement
         /// <see cref="IWritableToken">IWritableToken</see>
-        /// .
+        /// .</p>
         /// </remarks>
         protected internal int currentTokenIndex = 0;
 
@@ -296,13 +293,12 @@ namespace Antlr4.Runtime
         /// <summary>Return a marker that we can release later.</summary>
         /// <remarks>
         /// Return a marker that we can release later.
-        /// <p/>
-        /// The specific marker value used for this class allows for some level of
+        /// <p>The specific marker value used for this class allows for some level of
         /// protection against misuse where
         /// <code>seek()</code>
         /// is called on a mark or
         /// <code>release()</code>
-        /// is called in the wrong order.
+        /// is called in the wrong order.</p>
         /// </remarks>
         public virtual int Mark()
         {