jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

A few more adjustments to make the merged changes work with this revision of antlr.

This commit is contained in:
Mike Lischke 2016-03-18 15:27:17 +01:00
parent 1672bc0739
commit 11571fa092
8 changed files with 101 additions and 275 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
runtime/Cpp/demo/TLexer.g4
View File

@ -23,11 +23,11 @@ void myBarLexerAction() { /* do something*/ };
// Lexer API functions.
}
channels { COMMENTS_CHANNEL, DIRECTIVE }
// channels { COMMENTS_CHANNEL, DIRECTIVE }
ID: [a-z]+ ;
LessThan: '<' -> pushMode(Mode1);
GreaterThan: '>' -> popMode;
LessThan: '<';// -> pushMode(Mode1);
GreaterThan: '>';// -> popMode;
Foo: {canTestFoo()}? 'foo' {isItFoo()}? { myFooLexerAction(); };
Bar: 'bar' {isItBar()}? { myBarLexerAction(); };
Any: Foo Dot Bar? DotDot Baz;

2
tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Cpp.stg
View File

@ -41,6 +41,7 @@ cppTypeInitMap ::= [
]
// Lexer class templates.
DefaultLexerSuperClassHeader(s) ::= "Lexer"
DefaultLexerSuperClass(s) ::= "Lexer"
LexerHeader(lexer, atn, actionFuncs, sempredFuncs, superClass) ::= <<
@ -228,6 +229,7 @@ bool <r.factory.grammar.name>::<r.name>Sempred(<r.ctxType> *context, int predica
//--------------------------------------------------------------------------------------------------
DefaultParserSuperClassHeader(s) ::= "Parser"
DefaultParserSuperClass(s) ::= "Parser" // Is that used at all?
ParserHeader(parser, funcs, atn, sempredFuncs, superClass = {Parser}) ::= <<

3
tool/resources/org/antlr/v4/tool/templates/codegen/Cpp/Files.stg
View File

@ -113,7 +113,7 @@ ParserFile(file, parser, namedActions) ::= <<
#include "NoViableAltException.h"
#include "ParseTreeListener.h"
#include "<file.grammarName>Listener.h"
#include "<file.factory.grammar.name>Listener.h"
#include "<file.factory.grammar.name>.h"
@ -121,6 +121,7 @@ using namespace antlrcpp;
<if (file.genPackage)>using namespace <file.genPackage>;<endif>
<if (namedActions.header)><namedActions.header><endif>
<parser>
>>
BaseListenerFileHeader(file, header) ::= <<

41
tool/src/org/antlr/v4/codegen/CodeGenPipeline.java
View File

@ -49,9 +49,8 @@ public class CodeGenPipeline {
}
public void process() {
if ( !CodeGenerator.targetExists(g.getOptionString("language")) ) return;
CodeGenerator gen = new CodeGenerator(g);
IntervalSet idTypes = new IntervalSet();
idTypes.add(ANTLRParser.ID);
idTypes.add(ANTLRParser.RULE_REF);
@ -106,18 +105,16 @@ public class CodeGenPipeline {
gen.writeListener(listener, false);
}
if (gen.getTarget().wantsBaseListener()) {
if (gen.getTarget().needsHeader()) {
ST baseListener = gen.generateBaseListener(true);
if (g.tool.errMgr.getNumErrors() == errorCount) {
gen.writeBaseListener(baseListener, true);
}
}
ST baseListener = gen.generateBaseListener(false);
if (g.tool.errMgr.getNumErrors() == errorCount) {
gen.writeBaseListener(baseListener, false);
}
}
if (gen.getTarget().needsHeader()) {
ST baseListener = gen.generateBaseListener(true);
if (g.tool.errMgr.getNumErrors() == errorCount) {
gen.writeBaseListener(baseListener, true);
}
}
ST baseListener = gen.generateBaseListener(false);
if (g.tool.errMgr.getNumErrors() == errorCount) {
gen.writeBaseListener(baseListener, false);
}
}
if ( g.tool.gen_visitor ) {
if (gen.getTarget().needsHeader()) {
@ -131,18 +128,16 @@ public class CodeGenPipeline {
gen.writeVisitor(visitor, false);
}
if (gen.getTarget().wantsBaseVisitor()) {
if (gen.getTarget().needsHeader()) {
ST baseVisitor = gen.generateBaseVisitor(true);
if (g.tool.errMgr.getNumErrors() == errorCount) {
gen.writeBaseVisitor(baseVisitor, true);
}
}
ST baseVisitor = gen.generateBaseVisitor(false);
if (gen.getTarget().needsHeader()) {
ST baseVisitor = gen.generateBaseVisitor(true);
if (g.tool.errMgr.getNumErrors() == errorCount) {
gen.writeBaseVisitor(baseVisitor, false);
gen.writeBaseVisitor(baseVisitor, true);
}
}
ST baseVisitor = gen.generateBaseVisitor(false);
if (g.tool.errMgr.getNumErrors() == errorCount) {
gen.writeBaseVisitor(baseVisitor, false);
}
}
}
gen.writeVocabFile();

74
tool/src/org/antlr/v4/codegen/CodeGenerator.java
View File

@ -33,6 +33,7 @@ package org.antlr.v4.codegen;
import org.antlr.v4.Tool;
import org.antlr.v4.codegen.model.OutputModelObject;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.misc.NotNull;
import org.antlr.v4.tool.ErrorType;
import org.antlr.v4.tool.Grammar;
import org.stringtemplate.v4.AutoIndentWriter;
@ -43,7 +44,8 @@ import org.stringtemplate.v4.STWriter;
import java.io.IOException;
import java.io.Writer;
import java.lang.reflect.Constructor;
import java.util.LinkedHashMap;
import java.lang.reflect.InvocationTargetException;
import java.util.HashMap;
import java.util.Map;
/** General controller for code gen. Can instantiate sub generator(s).
@ -52,75 +54,73 @@ public class CodeGenerator {
public static final String TEMPLATE_ROOT = "org/antlr/v4/tool/templates/codegen";
public static final String VOCAB_FILE_EXTENSION = ".tokens";
public static final String DEFAULT_LANGUAGE = "Java";
public static final String vocabFilePattern =
public final static String vocabFilePattern =
"<tokens.keys:{t | <t>=<tokens.(t)>\n}>" +
"<literals.keys:{t | <t>=<literals.(t)>\n}>";
@NotNull
public final Grammar g;
@NotNull
public final Tool tool;
@NotNull
public final String language;
private Target target;
public int lineWidth = 72;
private CodeGenerator(String language) {
this.g = null;
this.tool = null;
this.language = language;
}
public CodeGenerator(Grammar g) {
public CodeGenerator(@NotNull Grammar g) {
this(g.tool, g, g.getOptionString("language"));
}
public CodeGenerator(Tool tool, Grammar g, String language) {
public CodeGenerator(@NotNull Tool tool, @NotNull Grammar g, String language) {
this.g = g;
this.tool = tool;
this.language = language != null ? language : DEFAULT_LANGUAGE;
}
public static boolean targetExists(String language) {
String targetName = "org.antlr.v4.codegen.target."+language+"Target";
try {
Class<? extends Target> c = Class.forName(targetName).asSubclass(Target.class);
Constructor<? extends Target> ctor = c.getConstructor(CodeGenerator.class);
CodeGenerator gen = new CodeGenerator(language);
Target target = ctor.newInstance(gen);
return target.templatesExist();
}
catch (Exception e) { // ignore errors; we're detecting presence only
}
return false;
}
public Target getTarget() {
if ( target == null && targetExists(language) ) {
if (target == null) {
loadLanguageTarget(language);
}
return target;
}
public STGroup getTemplates() {
Target t = getTarget();
return t==null ? null : t.getTemplates();
return getTarget().getTemplates();
}
protected void loadLanguageTarget(String language) {
String targetName = "org.antlr.v4.codegen.target."+language+"Target";
String targetName = "org.antlr.v4.codegen."+language+"Target";
try {
Class<? extends Target> c = Class.forName(targetName).asSubclass(Target.class);
Constructor<? extends Target> ctor = c.getConstructor(CodeGenerator.class);
target = ctor.newInstance(this);
}
catch (Exception e) {
catch (ClassNotFoundException cnfe) {
tool.errMgr.toolError(ErrorType.CANNOT_CREATE_TARGET_GENERATOR,
e,
cnfe,
targetName);
}
catch (NoSuchMethodException nsme) {
tool.errMgr.toolError(ErrorType.CANNOT_CREATE_TARGET_GENERATOR,
nsme,
targetName);
}
catch (InvocationTargetException ite) {
tool.errMgr.toolError(ErrorType.CANNOT_CREATE_TARGET_GENERATOR,
ite,
targetName);
}
catch (InstantiationException ie) {
tool.errMgr.toolError(ErrorType.CANNOT_CREATE_TARGET_GENERATOR,
ie,
targetName);
}
catch (IllegalAccessException cnfe) {
tool.errMgr.toolError(ErrorType.CANNOT_CREATE_TARGET_GENERATOR,
cnfe,
targetName);
}
}
@ -155,7 +155,7 @@ public class CodeGenerator {
*/
ST getTokenVocabOutput() {
ST vocabFileST = new ST(vocabFilePattern);
Map<String,Integer> tokens = new LinkedHashMap<String,Integer>();
Map<String,Integer> tokens = new HashMap<String,Integer>();
// make constants for the token names
for (String t : g.tokenNameToTypeMap.keySet()) {
int tokenType = g.tokenNameToTypeMap.get(t);
@ -166,7 +166,7 @@ public class CodeGenerator {
vocabFileST.add("tokens", tokens);
// now dump the strings
Map<String,Integer> literals = new LinkedHashMap<String,Integer>();
Map<String,Integer> literals = new HashMap<String,Integer>();
for (String literal : g.stringLiteralToTypeMap.keySet()) {
int tokenType = g.stringLiteralToTypeMap.get(literal);
if ( tokenType>=Token.MIN_USER_TOKEN_TYPE) {

2
tool/src/org/antlr/v4/codegen/target/CppTarget.java → tool/src/org/antlr/v4/codegen/CppTarget.java
View File

@ -29,7 +29,7 @@
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.antlr.v4.codegen.target;
package org.antlr.v4.codegen;
import org.antlr.v4.codegen.CodeGenerator;
import org.antlr.v4.codegen.Target;

21
tool/src/org/antlr/v4/codegen/OutputModelController.java
View File

@ -58,7 +58,6 @@ import org.antlr.v4.misc.Utils;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.parse.GrammarASTAdaptor;
import org.antlr.v4.tool.Alternative;
import org.antlr.v4.tool.ErrorType;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.LeftRecursiveRule;
import org.antlr.v4.tool.Rule;
@ -108,14 +107,15 @@ public class OutputModelController {
* extensions too, not just the factory functions in this factory.
*/
public OutputModelObject buildParserOutputModel(boolean header) {
Grammar g = delegate.getGrammar();
CodeGenerator gen = delegate.getGenerator();
ParserFile file = parserFile(gen.getRecognizerFileName(header));
setRoot(file);
file.parser = parser(file);
Parser parser = parser(file);
file.parser = parser;
Grammar g = delegate.getGrammar();
for (Rule r : g.rules.values()) {
buildRuleFunction(file.parser, r);
buildRuleFunction(parser, r);
}
return file;
@ -266,26 +266,17 @@ public class OutputModelController {
for (int i = 0; i < opAltsCode.size(); i++) {
ST altActionST;
LeftRecursiveRuleAltInfo altInfo = r.recOpAlts.getElement(i);
String templateName;
if ( altInfo.altLabel!=null ) {
templateName = "recRuleLabeledAltStartAction";
altActionST = codegenTemplates.getInstanceOf(templateName);
altActionST = codegenTemplates.getInstanceOf("recRuleLabeledAltStartAction");
altActionST.add("currentAltLabel", altInfo.altLabel);
}
else {
templateName = "recRuleAltStartAction";
altActionST = codegenTemplates.getInstanceOf(templateName);
altActionST = codegenTemplates.getInstanceOf("recRuleAltStartAction");
altActionST.add("ctxName", Utils.capitalize(r.name));
}
altActionST.add("ruleName", r.name);
// add label of any lr ref we deleted
altActionST.add("label", altInfo.leftRecursiveRuleRefLabel);
if (altActionST.impl.formalArguments.containsKey("isListLabel")) {
altActionST.add("isListLabel", altInfo.isListLabel);
}
else if (altInfo.isListLabel) {
delegate.getGenerator().tool.errMgr.toolError(ErrorType.CODE_TEMPLATE_ARG_ISSUE, templateName, "isListLabel");
}
Action altAction =
new Action(delegate, function.altLabelCtxs.get(altInfo.altLabel), altActionST);
CodeBlockForAlt alt = opAltsCode.get(i);

227
tool/src/org/antlr/v4/codegen/Target.java
View File

@ -1,6 +1,5 @@
/*
* [The "BSD license"]
* Copyright (c) 2016 Mike Lischke
* Copyright (c) 2012 Terence Parr
* Copyright (c) 2012 Sam Harwell
* All rights reserved.
@ -31,12 +30,10 @@
package org.antlr.v4.codegen;
import org.antlr.v4.Tool;
import org.antlr.v4.codegen.model.RuleFunction;
import org.antlr.v4.codegen.model.SerializedATN;
import org.antlr.v4.misc.Utils;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.runtime.RuntimeMetaData;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.tool.ErrorType;
import org.antlr.v4.tool.Grammar;
@ -90,26 +87,10 @@ public abstract class Target {
return language;
}
/** ANTLR tool should check output templates / target are compatible with tool code generation.
* For now, a simple string match used on x.y of x.y.z scheme. We use a method to avoid mismatches
* between a template called VERSION. This value is checked against Tool.VERSION during load of templates.
*
* This additional method forces all targets 4.3 and beyond to add this method.
*
* @since 4.3
*/
public abstract String getVersion();
public boolean needsHeader() { return false; }; // Override in targets which need header files.
public boolean needsHeader() { return false; }; // Override in targets which need header files.
public STGroup getTemplates() {
if (templates == null) {
String version = getVersion();
if ( version==null ||
!RuntimeMetaData.getMajorMinorVersion(version).equals(RuntimeMetaData.getMajorMinorVersion(Tool.VERSION)))
{
gen.tool.errMgr.toolError(ErrorType.INCOMPATIBLE_TOOL_AND_TEMPLATES, version, Tool.VERSION, language);
}
templates = loadTemplates();
}
@ -128,12 +109,11 @@ public abstract class Target {
* to a token type in the generated code.
*/
public String getTokenTypeAsTargetLabel(Grammar g, int ttype) {
String name = g.getTokenName(ttype);
// If name is not valid, return the token type instead
if ( Grammar.INVALID_TOKEN_NAME.equals(name) ) {
String name = g.getTokenDisplayName(ttype);
// If name is a literal, return the token type instead
if ( name==null || name.charAt(0)=='\'' ) {
return String.valueOf(ttype);
}
return name;
}
@ -196,106 +176,15 @@ public abstract class Target {
}
/**
* <p>Convert from an ANTLR string literal found in a grammar file to an
* Convert from an ANTLR string literal found in a grammar file to an
* equivalent string literal in the target language.
*</p>
* <p>
* For Java, this is the translation {@code 'a\n"'} &rarr; {@code "a\n\""}.
* Expect single quotes around the incoming literal. Just flip the quotes
* and replace double quotes with {@code \"}.
* </p>
* <p>
* Note that we have decided to allow people to use '\"' without penalty, so
* we must build the target string in a loop as {@link String#replace}
* cannot handle both {@code \"} and {@code "} without a lot of messing
* around.
* </p>
*/
public String getTargetStringLiteralFromANTLRStringLiteral(
public abstract String getTargetStringLiteralFromANTLRStringLiteral(
CodeGenerator generator,
String literal,
boolean addQuotes)
{
StringBuilder sb = new StringBuilder();
String is = literal;
if ( addQuotes ) sb.append('"');
for (int i = 1; i < is.length() -1; i++) {
if (is.charAt(i) == '\\') {
// Anything escaped is what it is! We assume that
// people know how to escape characters correctly. However
// we catch anything that does not need an escape in Java (which
// is what the default implementation is dealing with and remove
// the escape. The C target does this for instance.
//
switch (is.charAt(i+1)) {
// Pass through any escapes that Java also needs
//
case '"':
case 'n':
case 'r':
case 't':
case 'b':
case 'f':
case '\\':
// Pass the escape through
sb.append('\\');
break;
case 'u': // Assume unnnn
// Pass the escape through as double \\
// so that Java leaves as \u0000 string not char
sb.append('\\');
sb.append('\\');
break;
default:
// Remove the escape by virtue of not adding it here
// Thus \' becomes ' and so on
break;
}
// Go past the \ character
i++;
} else {
// Characters that don't need \ in ANTLR 'strings' but do in Java
if (is.charAt(i) == '"') {
// We need to escape " in Java
sb.append('\\');
}
}
// Add in the next character, which may have been escaped
sb.append(is.charAt(i));
}
if ( addQuotes ) sb.append('"');
return sb.toString();
}
String literal, boolean addQuotes);
/** Assume 16-bit char */
public String encodeIntAsCharEscape(int v) {
if (v < Character.MIN_VALUE || v > Character.MAX_VALUE) {
throw new IllegalArgumentException(String.format("Cannot encode the specified value: %d", v));
}
if (v >= 0 && v < targetCharValueEscape.length && targetCharValueEscape[v] != null) {
return targetCharValueEscape[v];
}
if (v >= 0x20 && v < 127 && (!Character.isDigit(v) || v == '8' || v == '9')) {
return String.valueOf((char)v);
}
if ( v>=0 && v<=127 ) {
String oct = Integer.toOctalString(v);
return "\\"+ oct;
}
String hex = Integer.toHexString(v|0x10000).substring(1,5);
return "\\u"+hex;
}
public abstract String encodeIntAsCharEscape(int v);
public String getLoopLabel(GrammarAST ast) {
return "loop"+ ast.token.getTokenIndex();
@ -390,43 +279,35 @@ public abstract class Target {
return Integer.MAX_VALUE;
}
/** How many bits should be used to do inline token type tests? Java assumes
* a 64-bit word for bitsets. Must be a valid wordsize for your target like
* 8, 16, 32, 64, etc...
*
* @since 4.5
*/
public int getInlineTestSetWordSize() { return 64; }
public boolean grammarSymbolCausesIssueInGeneratedCode(GrammarAST idNode) {
switch (idNode.getParent().getType()) {
case ANTLRParser.ASSIGN:
switch (idNode.getParent().getParent().getType()) {
case ANTLRParser.ELEMENT_OPTIONS:
case ANTLRParser.OPTIONS:
return false;
default:
break;
}
break;
case ANTLRParser.AT:
case ANTLRParser.ASSIGN:
switch (idNode.getParent().getParent().getType()) {
case ANTLRParser.ELEMENT_OPTIONS:
case ANTLRParser.OPTIONS:
return false;
case ANTLRParser.LEXER_ACTION_CALL:
if (idNode.getChildIndex() == 0) {
// first child is the command name which is part of the ANTLR language
return false;
}
// arguments to the command should be checked
break;
default:
break;
}
break;
case ANTLRParser.AT:
case ANTLRParser.ELEMENT_OPTIONS:
return false;
case ANTLRParser.LEXER_ACTION_CALL:
if (idNode.getChildIndex() == 0) {
// first child is the command name which is part of the ANTLR language
return false;
}
// arguments to the command should be checked
break;
default:
break;
}
return visibleGrammarSymbolCausesIssueInGeneratedCode(idNode);
@ -434,31 +315,8 @@ public abstract class Target {
protected abstract boolean visibleGrammarSymbolCausesIssueInGeneratedCode(GrammarAST idNode);
public boolean templatesExist() {
String groupFileName = CodeGenerator.TEMPLATE_ROOT + "/" + getLanguage() + "/" + getLanguage() + STGroup.GROUP_FILE_EXTENSION;
STGroup result = null;
try {
result = new STGroupFile(groupFileName);
}
catch (IllegalArgumentException iae) {
result = null;
}
return result!=null;
}
protected STGroup loadTemplates() {
String groupFileName = CodeGenerator.TEMPLATE_ROOT + "/" + getLanguage() + "/" + getLanguage() + STGroup.GROUP_FILE_EXTENSION;
STGroup result = null;
try {
result = new STGroupFile(groupFileName);
}
catch (IllegalArgumentException iae) {
gen.tool.errMgr.toolError(ErrorType.MISSING_CODE_GEN_TEMPLATES,
iae,
language);
}
if ( result==null ) return null;
STGroup result = new STGroupFile(CodeGenerator.TEMPLATE_ROOT+"/"+getLanguage()+"/"+getLanguage()+STGroup.GROUP_FILE_EXTENSION);
result.registerRenderer(Integer.class, new NumberRenderer());
result.registerRenderer(String.class, new StringRenderer());
result.setListener(new STErrorListener() {
@ -489,25 +347,4 @@ public abstract class Target {
return result;
}
/**
* @since 4.3
*/
public boolean wantsBaseListener() {
return true;
}
/**
* @since 4.3
*/
public boolean wantsBaseVisitor() {
return true;
}
/**
* @since 4.3
*/
public boolean supportsOverloadedMethods() {
return true;
}
}