jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

RuleFunction: process STRING_LITERAL along with RULE_REF and TOKEN_REF. Add getRuleTokens method

This commit is contained in:
Ivan Kochurkin 2018-09-09 11:54:15 +03:00
parent 0a6dbc37b7
commit d3745c7b93
1 changed files with 50 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
tool/src/org/antlr/v4/codegen/model/RuleFunction.java
View File

@ -7,6 +7,7 @@
package org.antlr.v4.codegen.model;
import org.antlr.runtime.RecognitionException;
import org.antlr.runtime.tree.CommonTree;
import org.antlr.runtime.tree.CommonTreeNodeStream;
import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.codegen.model.decl.AltLabelStructDecl;
@ -32,6 +33,7 @@ import org.antlr.v4.tool.Rule;
import org.antlr.v4.tool.ast.ActionAST;
import org.antlr.v4.tool.ast.AltAST;
import org.antlr.v4.tool.ast.GrammarAST;
import org.antlr.v4.tool.ast.PredAST;
import java.util.ArrayList;
import java.util.Collection;
@ -43,6 +45,7 @@ import java.util.Map;
import java.util.Set;
import static org.antlr.v4.parse.ANTLRParser.RULE_REF;
import static org.antlr.v4.parse.ANTLRParser.STRING_LITERAL;
import static org.antlr.v4.parse.ANTLRParser.TOKEN_REF;
/** */
@ -166,7 +169,7 @@ public class RuleFunction extends OutputModelObject {
}
/** for all alts, find which ref X or r needs List
Must see across alts. If any alt needs X or r as list, then
Must see across alts. If any alt needs X or r as list, then
define as list.
*/
public Set<Decl> getDeclsForAllElements(List<AltAST> altASTs) {
@ -174,21 +177,24 @@ public class RuleFunction extends OutputModelObject {
Set<String> nonOptional = new HashSet<String>();
List<GrammarAST> allRefs = new ArrayList<GrammarAST>();
boolean firstAlt = true;
IntervalSet reftypes = new IntervalSet(RULE_REF, TOKEN_REF, STRING_LITERAL);
for (AltAST ast : altASTs) {
IntervalSet reftypes = new IntervalSet(RULE_REF, TOKEN_REF);
List<GrammarAST> refs = ast.getNodesWithType(reftypes);
List<GrammarAST> refs = getRuleTokens(ast.getNodesWithType(reftypes));
allRefs.addAll(refs);
Pair<FrequencySet<String>, FrequencySet<String>> minAndAltFreq = getElementFrequenciesForAlt(ast);
FrequencySet<String> minFreq = minAndAltFreq.a;
FrequencySet<String> altFreq = minAndAltFreq.b;
for (GrammarAST t : refs) {
String refLabelName = t.getText();
if ( altFreq.count(refLabelName)>1 ) {
needsList.add(refLabelName);
}
String refLabelName = getName(t);
if (firstAlt && minFreq.count(refLabelName) != 0) {
nonOptional.add(refLabelName);
if (refLabelName != null) {
if (altFreq.count(refLabelName) > 1) {
needsList.add(refLabelName);
}
if (firstAlt && minFreq.count(refLabelName) != 0) {
nonOptional.add(refLabelName);
}
}
}
@ -202,7 +208,12 @@ public class RuleFunction extends OutputModelObject {
}
Set<Decl> decls = new LinkedHashSet<Decl>();
for (GrammarAST t : allRefs) {
String refLabelName = t.getText();
String refLabelName = getName(t);
if (refLabelName == null) {
continue;
}
List<Decl> d = getDeclForAltElement(t,
refLabelName,
needsList.contains(refLabelName),
@ -212,6 +223,35 @@ public class RuleFunction extends OutputModelObject {
return decls;
}
private List<GrammarAST> getRuleTokens(List<GrammarAST> refs) {
List<GrammarAST> result = new ArrayList<>(refs.size());
for (GrammarAST ref : refs) {
CommonTree r = ref;
boolean ignore = false;
while (r != null) {
// Ignore string literals in predicates
if (r instanceof PredAST) {
ignore = true;
break;
}
r = r.parent;
}
if (!ignore) {
result.add(ref);
}
}
return result;
}
private String getName(GrammarAST token) {
String tokenText = token.getText();
String tokenName = token.getType() != STRING_LITERAL ? tokenText : token.g.getTokenName(tokenText);
return tokenName == null || tokenName.startsWith("T__") ? null : tokenName; // Do not include tokens with auto generated names
}
/** Given list of X and r refs in alt, compute how many of each there are */
protected Pair<FrequencySet<String>, FrequencySet<String>> getElementFrequenciesForAlt(AltAST ast) {
try {