jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

all tests pass!

This commit is contained in:
Eric Vergnaud 2016-12-12 21:36:41 +08:00
parent 0170c9a064
commit 2fc33d97cd
11 changed files with 100 additions and 261 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/CSharp.test.stg
View File

@ -76,7 +76,7 @@ GetExpectedTokenNames() ::= "this.GetExpectedTokens().ToString(this.Vocabulary)"
RuleInvocationStack() ::= "GetRuleInvocationStackAsString()"
LL_EXACT_AMBIG_DETECTION() ::= <<Interpreter.PredictionMode = PredictionMode.LlExactAmbigDetection;>>
LL_EXACT_AMBIG_DETECTION() ::= <<Interpreter.PredictionMode = PredictionMode.LL_EXACT_AMBIG_DETECTION;>>
ParserToken(parser, token) ::= <%<parser>.<token>%>

59
runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseCSharpTest.java
View File

@ -13,7 +13,6 @@ import org.antlr.v4.runtime.WritableToken;
import org.antlr.v4.runtime.misc.Utils;
import org.antlr.v4.test.runtime.ErrorQueue;
import org.antlr.v4.test.runtime.RuntimeTestSupport;
import org.antlr.v4.test.runtime.SpecialRuntimeTestAssert;
import org.antlr.v4.tool.ANTLRMessage;
import org.antlr.v4.tool.GrammarSemanticsMessage;
import org.junit.rules.TestRule;
@ -54,7 +53,7 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
public class BaseCSharpTest implements RuntimeTestSupport, SpecialRuntimeTestAssert {
public class BaseCSharpTest implements RuntimeTestSupport /*, SpecialRuntimeTestAssert*/ {
public static final String newline = System.getProperty("line.separator");
public static final String pathSep = System.getProperty("path.separator");
@ -788,64 +787,8 @@ public class BaseCSharpTest implements RuntimeTestSupport, SpecialRuntimeTestAss
org.junit.Assert.assertEquals(msg, a, b);
}
@Override
public void assertEqualStrings(String a, String b) {
assertEquals(a, b);
}
protected static void assertEquals(String a, String b) {
a = absorbExpectedDifferences(a);
b = absorbActualDifferences(b);
org.junit.Assert.assertEquals(a, b);
}
protected static void assertNull(String a) {
a = absorbActualDifferences(a);
org.junit.Assert.assertNull(a);
}
private static String absorbExpectedDifferences(String a) {
if(a==null)
return a;
// work around the lack of requiresFullContext field in DFAState
if(a.startsWith("Decision"))
a = a.replaceAll("\\^", "");
// work around the algo difference for full context
a = stripOutUnwantedLinesWith(a, "reportAttemptingFullContext","reportContextSensitivity", "reportAmbiguity");
if(a.isEmpty()) {
a = null;
}
return a;
}
private static String absorbActualDifferences(String a) {
if(a==null) return a;
// work around the algo difference for full context
// work around the algo difference for semantic predicates
a = stripOutUnwantedLinesWith(a, "reportContextSensitivity","eval=false");
if(a.isEmpty()) {
a = null;
}
return a;
}
private static String stripOutUnwantedLinesWith(String a, String ... unwanteds) {
String[] lines = a.split("\n");
StringBuilder sb = new StringBuilder();
for(String line : lines) {
boolean wanted = true;
for(String unwanted : unwanteds) {
if(line.contains(unwanted) ) {
wanted = false;
break;
}
}
if(!wanted)
continue;
sb.append(line);
sb.append("\n");
}
return sb.toString();
}
}

9
runtime-testsuite/test/org/antlr/v4/test/runtime/descriptors/PerformanceDescriptors.java
View File

@ -107,8 +107,7 @@ public class PerformanceDescriptors {
@Override
public boolean ignore(String targetName) {
// return !Arrays.asList("Java", "Python2", "Python3", "Node").contains(targetName);
return !Arrays.asList("Java").contains(targetName);
return !Arrays.asList("Java", "CSharp", "Python2", "Python3", "Node").contains(targetName);
}
}
@ -190,6 +189,12 @@ public class PerformanceDescriptors {
*/
@CommentHasStringValue
public String input;
@Override
public boolean ignore(String targetName) {
// passes, but still too slow in Python and JavaScript
return !Arrays.asList("Java", "CSharp").contains(targetName);
}
}
public static class DropLoopEntryBranchInLRRule_5 extends DropLoopEntryBranchInLRRule {

5
runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Atn/ATN.cs
View File

@ -139,9 +139,8 @@ namespace Antlr4.Runtime.Atn
/// 's rule.
/// </summary>
[return: NotNull]
public virtual IntervalSet NextTokens(ATNState s, PredictionContext ctx)
public virtual IntervalSet NextTokens(ATNState s, RuleContext ctx)
{
Args.NotNull("ctx", ctx);
LL1Analyzer anal = new LL1Analyzer(this);
IntervalSet next = anal.Look(s, ctx);
return next;
@ -163,7 +162,7 @@ namespace Antlr4.Runtime.Atn
{
return s.nextTokenWithinRule;
}
s.nextTokenWithinRule = NextTokens(s, PredictionContext.EMPTY);
s.nextTokenWithinRule = NextTokens(s, null);
s.nextTokenWithinRule.SetReadonly(true);
return s.nextTokenWithinRule;
}

75
runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Atn/LL1Analyzer.cs
View File

@ -138,9 +138,9 @@ namespace Antlr4.Runtime.Atn
/// .
/// </returns>
[return: NotNull]
public virtual IntervalSet Look(ATNState s, PredictionContext ctx)
public virtual IntervalSet Look(ATNState s, RuleContext ctx)
{
return Look(s, s.atn.ruleToStopState[s.ruleIndex], ctx);
return Look(s, null, ctx);
}
/// <summary>
@ -189,13 +189,12 @@ namespace Antlr4.Runtime.Atn
/// .
/// </returns>
[return: NotNull]
public virtual IntervalSet Look(ATNState s, ATNState stopState, PredictionContext ctx)
public virtual IntervalSet Look(ATNState s, ATNState stopState, RuleContext ctx)
{
IntervalSet r = new IntervalSet();
bool seeThruPreds = true;
// ignore preds; get all lookahead
bool addEOF = true;
Look(s, stopState, ctx, r, new HashSet<ATNConfig>(), new BitSet(), seeThruPreds, addEOF);
PredictionContext lookContext = ctx != null ? PredictionContext.FromRuleContext(s.atn, ctx) : null;
Look(s, stopState, lookContext, r, new HashSet<ATNConfig>(), new BitSet(), seeThruPreds, true);
return r;
}
@ -301,36 +300,37 @@ namespace Antlr4.Runtime.Atn
}
if (s is RuleStopState)
{
if (ctx.IsEmpty && !ctx.IsEmpty)
if (ctx == null)
{
look.Add(TokenConstants.EPSILON);
return;
}
else if (ctx.IsEmpty && addEOF)
{
if (addEOF)
{
look.Add(TokenConstants.EOF);
}
look.Add(TokenConstants.EOF);
return;
}
bool removed = calledRuleStack.Get(s.ruleIndex);
try
{
calledRuleStack.Clear(s.ruleIndex);
for (int i = 0; i < ctx.Size; i++)
{
if (ctx.GetReturnState(i) == PredictionContext.EMPTY_RETURN_STATE)
{
continue;
}
ATNState returnState = atn.states[ctx.GetReturnState(i)];
// System.out.println("popping back to "+retState);
Look(returnState, stopState, ctx.GetParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF);
}
}
finally
{
if (removed)
{
calledRuleStack.Set(s.ruleIndex);
}
}
if (ctx != PredictionContext.EMPTY)
{
for (int i = 0; i < ctx.Size; i++)
{
ATNState returnState = atn.states[ctx.GetReturnState(i)];
bool removed = calledRuleStack.Get(returnState.ruleIndex);
try
{
calledRuleStack.Clear(returnState.ruleIndex);
Look(returnState, stopState, ctx.GetParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF);
}
finally
{
if (removed)
{
calledRuleStack.Set(returnState.ruleIndex);
}
}
}
return;
}
}
int n = s.NumberOfTransitions;
for (int i_1 = 0; i_1 < n; i_1++)
@ -343,15 +343,15 @@ namespace Antlr4.Runtime.Atn
{
continue;
}
PredictionContext newContext = ctx.GetChild(ruleTransition.followState.stateNumber);
PredictionContext newContext = SingletonPredictionContext.Create(ctx, ruleTransition.followState.stateNumber);
try
{
calledRuleStack.Set(ruleTransition.ruleIndex);
calledRuleStack.Set(ruleTransition.target.ruleIndex);
Look(t.target, stopState, newContext, look, lookBusy, calledRuleStack, seeThruPreds, addEOF);
}
finally
{
calledRuleStack.Clear(ruleTransition.ruleIndex);
calledRuleStack.Clear(ruleTransition.target.ruleIndex);
}
}
else
@ -375,13 +375,12 @@ namespace Antlr4.Runtime.Atn
}
else
{
if (t.GetType() == typeof(WildcardTransition))
if (t is WildcardTransition)
{
look.AddAll(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType));
}
else
{
// System.out.println("adding "+ t);
IntervalSet set = t.Label;
if (set != null)
{

22
runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Atn/MergeCache.cs
View File

@ -1,16 +1,34 @@
using System;
using System.Collections.Generic;
namespace Antlr4.Runtime.Atn
{
public class MergeCache
{
Dictionary<PredictionContext, Dictionary<PredictionContext, PredictionContext>> data = new Dictionary<PredictionContext, Dictionary<PredictionContext, PredictionContext>>();
public PredictionContext Get(PredictionContext a, PredictionContext b)
{
throw new NotImplementedException();
Dictionary<PredictionContext, PredictionContext> first;
if (!data.TryGetValue(a, out first))
return null;
PredictionContext value;
if (first.TryGetValue(b, out value))
return value;
else
return null;
}
public void Put(PredictionContext a, PredictionContext b, PredictionContext value)
{
throw new NotImplementedException();
Dictionary<PredictionContext, PredictionContext> first;
if (!data.TryGetValue(a, out first))
{
first = new Dictionary<PredictionContext, PredictionContext>();
data[a] = first;
}
first[b] = value;
}
}
}

39
runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Atn/ParserATNSimulator.cs
View File

@ -1230,11 +1230,14 @@ namespace Antlr4.Runtime.Atn
* filter the prediction context for alternatives predicting alt>1
* (basically a graph subtraction algorithm).
*/
PredictionContext context = statesFromAlt1[config.state.stateNumber];
if (context != null && context.Equals(config.context))
PredictionContext ctx;
if (statesFromAlt1.TryGetValue(config.state.stateNumber, out ctx))
{
// eliminated
continue;
if (ctx != null && ctx.Equals(config.context))
{
// eliminated
continue;
}
}
}
@ -1275,7 +1278,7 @@ namespace Antlr4.Runtime.Atn
{
if (ambigAlts[c.alt])
{
altToPred[c.alt] = SemanticContext.Or(altToPred[c.alt], c.semanticContext);
altToPred[c.alt] = SemanticContext.OrOp(altToPred[c.alt], c.semanticContext);
}
}
@ -1556,7 +1559,8 @@ namespace Antlr4.Runtime.Atn
int depth,
bool treatEofAsEpsilon)
{
if (debug) Console.WriteLine("closure(" + config.ToString(parser, true) + ")");
if (debug)
Console.WriteLine("closure(" + config.ToString(parser, true) + ")");
if (config.state is RuleStopState)
{
@ -1960,7 +1964,7 @@ namespace Antlr4.Runtime.Atn
}
}
else {
SemanticContext newSemCtx = SemanticContext.And(config.semanticContext, pt.Predicate);
SemanticContext newSemCtx = SemanticContext.AndOp(config.semanticContext, pt.Predicate);
c = new ATNConfig(config, pt.target, newSemCtx);
}
}
@ -2011,7 +2015,7 @@ namespace Antlr4.Runtime.Atn
}
}
else {
SemanticContext newSemCtx = SemanticContext.And(config.semanticContext, pt.Predicate);
SemanticContext newSemCtx = SemanticContext.AndOp(config.semanticContext, pt.Predicate);
c = new ATNConfig(config, pt.target, newSemCtx);
}
}
@ -2324,20 +2328,19 @@ namespace Antlr4.Runtime.Atn
exact, ambigAlts, configs);
}
public void SetPredictionMode(PredictionMode mode)
public PredictionMode PredictionMode
{
this.mode = mode;
get
{
return this.mode;
}
set
{
this.mode = value;
}
}
public PredictionMode GetPredictionMode()
{
return mode;
}
/**
* @since 4.3
*/
public Parser getParser()
{
return parser;

139
runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Atn/SemanticContext.cs
View File

@ -30,99 +30,19 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Antlr4.Runtime;
using Antlr4.Runtime.Atn;
using Antlr4.Runtime.Misc;
using Antlr4.Runtime.Sharpen;
namespace Antlr4.Runtime.Atn
{
/// <summary>
/// A tree structure used to record the semantic context in which
/// an ATN configuration is valid.
/// </summary>
/// <remarks>
/// A tree structure used to record the semantic context in which
/// an ATN configuration is valid. It's either a single predicate,
/// a conjunction
/// <c>p1&amp;&amp;p2</c>
/// , or a sum of products
/// <c>p1||p2</c>
/// .
/// <p>I have scoped the
/// <see cref="AND"/>
/// ,
/// <see cref="OR"/>
/// , and
/// <see cref="Predicate"/>
/// subclasses of
/// <see cref="SemanticContext"/>
/// within the scope of this outer class.</p>
/// </remarks>
public abstract class SemanticContext
{
/// <summary>
/// The default
/// <see cref="SemanticContext"/>
/// , which is semantically equivalent to
/// a predicate of the form
/// <c/>
///
/// true}?}.
/// </summary>
public static readonly SemanticContext NONE = new SemanticContext.Predicate();
/// <summary>
/// For context independent predicates, we evaluate them without a local
/// context (i.e., null context).
/// </summary>
/// <remarks>
/// For context independent predicates, we evaluate them without a local
/// context (i.e., null context). That way, we can evaluate them without
/// having to create proper rule-specific context during prediction (as
/// opposed to the parser, which creates them naturally). In a practical
/// sense, this avoids a cast exception from RuleContext to myruleContext.
/// <p>For context dependent predicates, we must pass in a local context so that
/// references such as $arg evaluate properly as _localctx.arg. We only
/// capture context dependent predicates in the context in which we begin
/// prediction, so we passed in the outer context here in case of context
/// dependent predicate evaluation.</p>
/// </remarks>
public abstract bool Eval<Symbol, ATNInterpreter>(Recognizer<Symbol, ATNInterpreter> parser, RuleContext parserCallStack)
where ATNInterpreter : ATNSimulator;
/// <summary>Evaluate the precedence predicates for the context and reduce the result.</summary>
/// <remarks>Evaluate the precedence predicates for the context and reduce the result.</remarks>
/// <param name="parser">The parser instance.</param>
/// <param name="parserCallStack"/>
/// <returns>
/// The simplified semantic context after precedence predicates are
/// evaluated, which will be one of the following values.
/// <ul>
/// <li>
/// <see cref="NONE"/>
/// : if the predicate simplifies to
/// <see langword="true"/>
/// after
/// precedence predicates are evaluated.</li>
/// <li>
/// <see langword="null"/>
/// : if the predicate simplifies to
/// <see langword="false"/>
/// after
/// precedence predicates are evaluated.</li>
/// <li>
/// <c>this</c>
/// : if the semantic context is not changed as a result of
/// precedence predicate evaluation.</li>
/// <li>A non-
/// <see langword="null"/>
///
/// <see cref="SemanticContext"/>
/// : the new simplified
/// semantic context after precedence predicates are evaluated.</li>
/// </ul>
/// </returns>
public virtual SemanticContext EvalPrecedence<Symbol, ATNInterpreter>(Recognizer<Symbol, ATNInterpreter> parser, RuleContext parserCallStack)
public virtual SemanticContext EvalPrecedence<Symbol, ATNInterpreter>(Recognizer<Symbol, ATNInterpreter> parser, RuleContext parserCallStack)
where ATNInterpreter : ATNSimulator
{
return this;
@ -251,26 +171,8 @@ namespace Antlr4.Runtime.Atn
}
}
/// <summary>
/// This is the base class for semantic context "operators", which operate on
/// a collection of semantic context "operands".
/// </summary>
/// <remarks>
/// This is the base class for semantic context "operators", which operate on
/// a collection of semantic context "operands".
/// </remarks>
/// <since>4.3</since>
public abstract class Operator : SemanticContext
{
/// <summary>Gets the operands for the semantic context operator.</summary>
/// <remarks>Gets the operands for the semantic context operator.</remarks>
/// <returns>
/// a collection of
/// <see cref="SemanticContext"/>
/// operands for the
/// operator.
/// </returns>
/// <since>4.3</since>
[NotNull]
public abstract ICollection<SemanticContext> Operands
{
@ -278,14 +180,6 @@ namespace Antlr4.Runtime.Atn
}
}
/// <summary>
/// A semantic context which is true whenever none of the contained contexts
/// is false.
/// </summary>
/// <remarks>
/// A semantic context which is true whenever none of the contained contexts
/// is false.
/// </remarks>
public class AND : SemanticContext.Operator
{
[NotNull]
@ -347,12 +241,6 @@ namespace Antlr4.Runtime.Atn
return MurmurHash.HashCode(opnds, typeof(SemanticContext.AND).GetHashCode());
}
/// <summary>
/// <inheritDoc/>
/// <p>
/// The evaluation of predicates by this context is short-circuiting, but
/// unordered.</p>
/// </summary>
public override bool Eval<Symbol, ATNInterpreter>(Recognizer<Symbol, ATNInterpreter> parser, RuleContext parserCallStack)
{
foreach (SemanticContext opnd in opnds)
@ -399,7 +287,7 @@ namespace Antlr4.Runtime.Atn
SemanticContext result = operands[0];
for (int i = 1; i < operands.Count; i++)
{
result = SemanticContext.And(result, operands[i]);
result = SemanticContext.AndOp(result, operands[i]);
}
return result;
}
@ -410,14 +298,6 @@ namespace Antlr4.Runtime.Atn
}
}
/// <summary>
/// A semantic context which is true whenever at least one of the contained
/// contexts is true.
/// </summary>
/// <remarks>
/// A semantic context which is true whenever at least one of the contained
/// contexts is true.
/// </remarks>
public class OR : SemanticContext.Operator
{
[NotNull]
@ -479,12 +359,6 @@ namespace Antlr4.Runtime.Atn
return MurmurHash.HashCode(opnds, typeof(SemanticContext.OR).GetHashCode());
}
/// <summary>
/// <inheritDoc/>
/// <p>
/// The evaluation of predicates by this context is short-circuiting, but
/// unordered.</p>
/// </summary>
public override bool Eval<Symbol, ATNInterpreter>(Recognizer<Symbol, ATNInterpreter> parser, RuleContext parserCallStack)
{
foreach (SemanticContext opnd in opnds)
@ -531,7 +405,7 @@ namespace Antlr4.Runtime.Atn
SemanticContext result = operands[0];
for (int i = 1; i < operands.Count; i++)
{
result = SemanticContext.Or(result, operands[i]);
result = SemanticContext.OrOp(result, operands[i]);
}
return result;
}
@ -542,7 +416,7 @@ namespace Antlr4.Runtime.Atn
}
}
public static SemanticContext And(SemanticContext a, SemanticContext b)
public static SemanticContext AndOp(SemanticContext a, SemanticContext b)
{
if (a == null || a == NONE)
{
@ -560,8 +434,7 @@ namespace Antlr4.Runtime.Atn
return result;
}
/// <seealso cref="ParserATNSimulator.GetPredsForAmbigAlts(Antlr4.Runtime.Sharpen.BitSet, ATNConfigSet, int)"/>
public static SemanticContext Or(SemanticContext a, SemanticContext b)
public static SemanticContext OrOp(SemanticContext a, SemanticContext b)
{
if (a == null)
{

3
runtime/CSharp/runtime/CSharp/Antlr4.Runtime/DefaultErrorStrategy.cs
View File

@ -620,8 +620,7 @@ namespace Antlr4.Runtime
ATNState currentState = recognizer.Interpreter.atn.states[recognizer.State];
ATNState next = currentState.Transition(0).target;
ATN atn = recognizer.Interpreter.atn;
IntervalSet expectingAtLL2 = atn.NextTokens(next, PredictionContext.FromRuleContext(atn, recognizer.RuleContext));
// System.out.println("LT(2) set="+expectingAtLL2.toString(recognizer.getTokenNames()));
IntervalSet expectingAtLL2 = atn.NextTokens(next, recognizer.RuleContext);
if (expectingAtLL2.Contains(currentSymbolType))
{
ReportMissingToken(recognizer);

4
runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Dfa/DFASerializer.cs
View File

@ -132,7 +132,7 @@ namespace Antlr4.Runtime.Dfa
protected internal virtual string GetEdgeLabel(int i)
{
return vocabulary.GetDisplayName(i);
return vocabulary.GetDisplayName(i - 1);
}
internal virtual string GetStateString(DFAState s)
@ -143,7 +143,7 @@ namespace Antlr4.Runtime.Dfa
}
int n = s.stateNumber;
string baseStateStr = (s.isAcceptState ? ":" : "") + "s" + n ;
string baseStateStr = (s.isAcceptState ? ":" : "") + "s" + n + (s.requiresFullContext ? "^" : "");
if ( s.isAcceptState ) {
if ( s.predicates!=null ) {
return baseStateStr + "=>" + Arrays.ToString(s.predicates);

4
runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Parser.cs
View File

@ -1163,9 +1163,9 @@ namespace Antlr4.Runtime
public virtual void DumpDFA()
{
bool seenOne = false;
for (int d = 0; d < Interpreter.atn.decisionToDFA.Length; d++)
for (int d = 0; d < Interpreter.decisionToDFA.Length; d++)
{
DFA dfa = Interpreter.atn.decisionToDFA[d];
DFA dfa = Interpreter.decisionToDFA[d];
if (dfa.states.Count>0)
{
if (seenOne)