forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

rm T='literal' in tokens { }. Also it's comma-separated not ';' terminated now. tokens { A,B }

This commit is contained in:
Terence Parr 2012-09-06 14:50:44 -07:00
parent 2b5c3723ba
commit b3b02a5449
12 changed files with 26 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tool/playground/T.g
View File

@ -1,3 +1,5 @@
grammar T;
tokens { A,B }
s : A ;

11
tool/src/org/antlr/v4/parse/ANTLRParser.g
View File

@ -283,16 +283,7 @@ delegateGrammar
* {tree} parser.
*/
tokensSpec
: TOKENS_SPEC tokenSpec+ RBRACE -> ^(TOKENS_SPEC tokenSpec+)
;
tokenSpec
: id
( ASSIGN STRING_LITERAL -> ^(ASSIGN id STRING_LITERAL<TerminalAST>)
| -> id
)
SEMI
| RULE_REF // INVALID! (an error alt)
: TOKENS_SPEC id (COMMA id)* RBRACE -> ^(TOKENS_SPEC id+)
;
// A declaration of a language target specifc section,

5
tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g
View File

@ -121,7 +121,7 @@ public void finishGrammar(GrammarRootAST root, GrammarAST ID) { }
public void grammarOption(GrammarAST ID, GrammarAST valueAST) { }
public void ruleOption(GrammarAST ID, GrammarAST valueAST) { }
public void blockOption(GrammarAST ID, GrammarAST valueAST) { }
public void tokenAlias(GrammarAST ID, GrammarAST literal) { }
public void defineToken(GrammarAST ID) { }
public void globalNamedAction(GrammarAST scope, GrammarAST ID, ActionAST action) { }
public void importGrammar(GrammarAST label, GrammarAST ID) { }
@ -225,8 +225,7 @@ tokensSpec
;
tokenSpec
: ^(ASSIGN ID STRING_LITERAL) {tokenAlias($ID, $STRING_LITERAL);}
| ID {tokenAlias($ID, null);}
: ID {defineToken($ID);}
;
action

5
tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java
View File

@ -125,11 +125,6 @@ public class BasicSemanticChecks extends GrammarTreeVisitor {
checkNumPrequels(options, imports, tokens);
}
@Override
public void tokenAlias(GrammarAST ID, GrammarAST literal) {
if ( literal!=null ) checkTokenAlias(ID.token);
}
@Override
public void importGrammar(GrammarAST label, GrammarAST ID) {
checkImport(ID.token);

52
tool/src/org/antlr/v4/semantics/SymbolChecks.java
View File

@ -90,7 +90,6 @@ public class SymbolChecks {
// done in sem pipe for now
checkForRuleConflicts(g.rules.values()); // sets nameToRuleMap
checkActionRedefinitions(collector.namedActions);
checkTokenAliasRedefinitions(collector.tokensDefs);
//checkRuleArgs(collector.rulerefs);
checkForTokenConflicts(collector.tokenIDRefs); // sets tokenIDs
checkForLabelConflicts(g.rules.values());
@ -140,57 +139,6 @@ public class SymbolChecks {
}
}
/** Catch:
tokens { A='a'; A; } can't redefine token type if has alias
tokens { A; A='a'; } can't redefine token type if has alias
tokens { A='a'; A='b'; } can't have two aliases for single token type
tokens { A='a'; B='a'; } can't have to token types for same string alias
*/
public void checkTokenAliasRedefinitions(List<GrammarAST> aliases) {
if ( aliases==null ) return;
// map names, strings to root of A or (= A 'a')
Map<String, GrammarAST> aliasTokenNames = new HashMap<String, GrammarAST>();
Map<String, GrammarAST> aliasStringValues = new HashMap<String, GrammarAST>();
for (int i=0; i<aliases.size(); i++) {
GrammarAST a = aliases.get(i);
GrammarAST idNode = a;
if ( a.getChildCount()>0 ) idNode = (GrammarAST)a.getChild(0);
GrammarAST prevToken = aliasTokenNames.get(idNode.getText());
GrammarAST stringNode = null;
if ( a.getChildCount()>0 ) stringNode = (GrammarAST)a.getChild(1);
GrammarAST prevString = null;
if ( stringNode!=null ) prevString = aliasStringValues.get(stringNode.getText());
if ( a.getType() == ANTLRParser.ASSIGN ) { // A='a'
idNode = (GrammarAST)a.getChild(0);
if ( prevString==null ) { // not seen string before
if ( stringNode!=null ) aliasStringValues.put(stringNode.getText(), a);
}
}
if ( prevToken==null ) { // not seen before, define it
aliasTokenNames.put(idNode.getText(), a);
}
// we've defined token names and strings at this point if not seen previously.
// now, look for trouble.
if ( prevToken!=null ) {
if ( !(prevToken.getChildCount()==0 && a.getChildCount()==0) ) {
// one or both have strings; disallow
errMgr.grammarError(ErrorType.TOKEN_NAME_REASSIGNMENT,
a.g.fileName, idNode.token, idNode.getText());
}
}
if ( prevString!=null ) {
// A='a' and A='a' are ok but not B='a' and A='a' are ok
if ( !prevString.getChild(0).getText().equals(idNode.getText()) ) {
errMgr.grammarError(ErrorType.TOKEN_STRING_REASSIGNMENT,
a.g.fileName, idNode.token, idNode.getText()+"="+stringNode.getText(),
prevString.getChild(0).getText());
}
}
}
}
public void checkForTokenConflicts(List<GrammarAST> tokenIDRefs) {
// for (GrammarAST a : tokenIDRefs) {
// Token t = a.token;

10
tool/src/org/antlr/v4/semantics/SymbolCollector.java
View File

@ -81,19 +81,11 @@ public class SymbolCollector extends GrammarTreeVisitor {
}
@Override
public void tokenAlias(GrammarAST ID, GrammarAST literal) {
if ( literal==null ) {
public void defineToken(GrammarAST ID) {
terminals.add(ID);
tokenIDRefs.add(ID);
tokensDefs.add(ID);
}
else {
terminals.add(ID);
tokenIDRefs.add(ID);
tokensDefs.add((GrammarAST)ID.getParent());
strings.add(literal.getText());
}
}
@Override
public void discoverRule(RuleAST rule, GrammarAST ID,

4
tool/src/org/antlr/v4/tool/ErrorType.java
View File

@ -123,8 +123,8 @@ public enum ErrorType {
HETERO_ILLEGAL_IN_REWRITE_ALT(104, "", ErrorSeverity.ERROR),
NO_SUCH_GRAMMAR_SCOPE(105, "reference to undefined grammar in rule reference: <arg>.<arg2>", ErrorSeverity.ERROR),
NO_SUCH_RULE_IN_SCOPE(106, "rule <arg2> is not defined in grammar <arg>", ErrorSeverity.ERROR),
TOKEN_STRING_REASSIGNMENT(107, "cannot alias <arg> in tokens {}; string already assigned to <arg2>", ErrorSeverity.ERROR),
TOKEN_NAME_REASSIGNMENT(108, "cannot redefine <arg>; token name already <if(arg2)>assigned to <arg2><else>defined<endif>", ErrorSeverity.ERROR),
// TOKEN_STRING_REASSIGNMENT(107, "cannot alias <arg> in tokens {}; string already assigned to <arg2>", ErrorSeverity.ERROR),
// TOKEN_NAME_REASSIGNMENT(108, "cannot redefine <arg>; token name already <if(arg2)>assigned to <arg2><else>defined<endif>", ErrorSeverity.ERROR),
//TOKEN_VOCAB_IN_DELEGATE(, "tokenVocab option ignored in imported grammar <arg>", ErrorSeverity.ERROR),
OPTIONS_IN_DELEGATE(109, "options ignored in imported grammar <arg>", ErrorSeverity.WARNING),
// TOKEN_ALIAS_IN_DELEGATE(, "can't assign string to token name <arg> to string in imported grammar <arg2>", ErrorSeverity.ERROR),

4
tool/test/org/antlr/v4/test/TestASTStructure.java
View File

@ -69,7 +69,7 @@ public class TestASTStructure {
@Test public void test_grammarSpec2() throws Exception {
// gunit test on line 18
RuleReturnScope rstruct = (RuleReturnScope)execParser("grammarSpec", "\n parser grammar P;\n tokens { A; B='33'; }\n @header {foo}\n a : A;\n ", 18);
RuleReturnScope rstruct = (RuleReturnScope)execParser("grammarSpec", "\n parser grammar P;\n tokens { A, B }\n @header {foo}\n a : A;\n ", 18);
Object actual = ((Tree)rstruct.getTree()).toStringTree();
Object expecting = "(PARSER_GRAMMAR P (tokens { A (= B '33')) (@ header {foo}) (RULES (RULE a (BLOCK (ALT A)))))";
assertEquals("testing rule grammarSpec", expecting, actual);
@ -77,7 +77,7 @@ public class TestASTStructure {
@Test public void test_grammarSpec3() throws Exception {
// gunit test on line 30
RuleReturnScope rstruct = (RuleReturnScope)execParser("grammarSpec", "\n parser grammar P;\n @header {foo}\n tokens { A; B='33'; }\n a : A;\n ", 30);
RuleReturnScope rstruct = (RuleReturnScope)execParser("grammarSpec", "\n parser grammar P;\n @header {foo}\n tokens { A,B }\n a : A;\n ", 30);
Object actual = ((Tree)rstruct.getTree()).toStringTree();
Object expecting = "(PARSER_GRAMMAR P (@ header {foo}) (tokens { A (= B '33')) (RULES (RULE a (BLOCK (ALT A)))))";
assertEquals("testing rule grammarSpec", expecting, actual);

4
tool/test/org/antlr/v4/test/TestATNDeserialization.java
View File

@ -33,7 +33,7 @@ public class TestATNDeserialization extends BaseTest {
@Test public void testNot() throws Exception {
Grammar g = new Grammar(
"parser grammar T;\n"+
"tokens {A; B; C;}\n" +
"tokens {A, B, C}\n" +
"a : ~A ;");
checkDeserializationIsStable(g);
}
@ -41,7 +41,7 @@ public class TestATNDeserialization extends BaseTest {
@Test public void testWildcard() throws Exception {
Grammar g = new Grammar(
"parser grammar T;\n"+
"tokens {A; B; C;}\n" +
"tokens {A, B, C}\n" +
"a : . ;");
checkDeserializationIsStable(g);
}

4
tool/test/org/antlr/v4/test/TestATNInterpreter.java
View File

@ -40,7 +40,7 @@ public class TestATNInterpreter extends BaseTest {
"C : 'c' ;\n");
Grammar g = new Grammar(
"parser grammar T;\n"+
"tokens {A; B; C;}\n" +
"tokens {A,B,C}\n" +
"a : ~A ;");
checkMatchedAlt(lg, g, "b", 1);
}
@ -254,7 +254,7 @@ public class TestATNInterpreter extends BaseTest {
);
Grammar g = new Grammar(
"parser grammar T;\n"+
"tokens {A;B;C;LP;RP;INT;}\n" +
"tokens {A,B,C,LP,RP,INT}\n" +
"a : e B | e C ;\n" +
"e : LP e RP\n" +
" | INT\n" +

8
tool/test/org/antlr/v4/test/TestATNParserPrediction.java
View File

@ -186,7 +186,7 @@ public class TestATNParserPrediction extends BaseTest {
"C : 'c' ;\n");
Grammar g = new Grammar(
"parser grammar T;\n"+
"tokens {A;B;C;}\n" +
"tokens {A,B,C}\n" +
"a : x B ;\n" +
"b : x C ;\n" +
"x : A | ;\n");
@ -302,7 +302,7 @@ public class TestATNParserPrediction extends BaseTest {
);
Grammar g = new Grammar(
"parser grammar T;\n"+
"tokens {A;B;C;LP;RP;INT;}\n" +
"tokens {A,B,C,LP,RP,INT}\n" +
"a : e B | e C ;\n" +
"e : LP e RP\n" +
" | INT\n" +
@ -361,7 +361,7 @@ public class TestATNParserPrediction extends BaseTest {
);
Grammar g = new Grammar(
"parser grammar T;\n"+
"tokens {A;B;C;LP;RP;INT;}\n" +
"tokens {A,B,C,LP,RP,INT}\n" +
"a : e A | e A B ;\n" +
"e : LP e RP\n" +
" | INT\n" +
@ -425,7 +425,7 @@ public class TestATNParserPrediction extends BaseTest {
);
Grammar g = new Grammar(
"parser grammar T;\n"+
"tokens {ID;SEMI;INT;}\n" +
"tokens {ID,SEMI,INT}\n" +
"a : (ID | ID ID?) SEMI ;");
int decision = 1;
checkPredictedAlt(lg, g, decision, "a;", 1);

48
tool/test/org/antlr/v4/test/TestTokenTypeAssignment.java
View File

@ -27,8 +27,8 @@ public class TestTokenTypeAssignment extends BaseTest {
Grammar g = new Grammar(
"parser grammar t;\n" +
"tokens {\n" +
" C;\n" +
" D;" +
" C,\n" +
" D" +
"}\n"+
"a : A | B;\n" +
"b : C ;");
@ -41,8 +41,8 @@ public class TestTokenTypeAssignment extends BaseTest {
LexerGrammar g = new LexerGrammar(
"lexer grammar t;\n" +
"tokens {\n" +
" C;\n" +
" D;" +
" C,\n" +
" D" +
"}\n"+
"A : 'a';\n" +
"C : 'c' ;");
@ -51,20 +51,6 @@ public class TestTokenTypeAssignment extends BaseTest {
checkSymbols(g, rules, tokenNames);
}
@Test public void testTokensSectionWithAssignmentSection() throws Exception {
Grammar g = new Grammar(
"grammar t;\n" +
"tokens {\n" +
" C='c';\n" +
" D;" +
"}\n"+
"a : A | B;\n" +
"b : C ;");
String rules = "a, b";
String tokenNames = "A, B, C, D, 'c'";
checkSymbols(g, rules, tokenNames);
}
@Test public void testCombinedGrammarLiterals() throws Exception {
Grammar g = new Grammar(
"grammar t;\n"+
@ -145,32 +131,6 @@ public class TestTokenTypeAssignment extends BaseTest {
assertEquals("'\\n'", literals.toArray()[0]);
}
@Test public void testTokenInTokensSectionAndTokenRuleDef() throws Exception {
String grammar =
"grammar P;\n" +
"tokens { B='}'; }\n"+
"a : A B {System.out.println(_input.getText());} ;\n"+
"A : 'a' ;\n" +
"B : '}' ;\n"+
"WS : (' '|'\\n') {skip();} ;";
String found = execParser("P.g4", grammar, "PParser", "PLexer",
"a", "a}", false);
assertEquals("a}\n", found);
}
@Test public void testTokenInTokensSectionAndTokenRuleDef2() throws Exception {
String grammar =
"grammar P;\n" +
"tokens { B='}'; }\n"+
"a : A '}' {System.out.println(_input.getText());} ;\n"+
"A : 'a' ;\n" +
"B : '}' ;\n"+
"WS : (' '|'\\n') {skip();} ;";
String found = execParser("P.g4", grammar, "PParser", "PLexer",
"a", "a}", false);
assertEquals("a}\n", found);
}
protected void checkSymbols(Grammar g,
String rulesStr,
String allValidTokensStr)