got basic pattern compilation going
This commit is contained in:
Terence Parr
parent
56cd7cdf2c
commit
531c31861a
|
|
@ -0,0 +1,64 @@
|
|||
package org.antlr.v4.runtime.tree.pattern;
|
||||
|
||||
import org.antlr.v4.runtime.CharStream;
|
||||
import org.antlr.v4.runtime.CommonToken;
|
||||
import org.antlr.v4.runtime.CommonTokenFactory;
|
||||
import org.antlr.v4.runtime.Token;
|
||||
import org.antlr.v4.runtime.TokenFactory;
|
||||
import org.antlr.v4.runtime.TokenSource;
|
||||
import org.antlr.v4.runtime.misc.NotNull;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/** Convert a list of token objects to a source of tokens for a TokenStream. */
|
||||
public class ListTokenSource implements TokenSource {
|
||||
protected List<? extends Token> tokens;
|
||||
protected int i = 0;
|
||||
protected TokenFactory<?> _factory = CommonTokenFactory.DEFAULT;
|
||||
|
||||
public ListTokenSource(List<? extends Token> tokens) {
|
||||
assert tokens!=null;
|
||||
this.tokens = tokens;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getCharPositionInLine() {
|
||||
return -1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Token nextToken() {
|
||||
if ( i>=tokens.size() ) {
|
||||
Token eof = new CommonToken(Token.EOF); // ignore factory
|
||||
return eof;
|
||||
}
|
||||
Token t = tokens.get(i);
|
||||
i++;
|
||||
return t;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getLine() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public CharStream getInputStream() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getSourceName() {
|
||||
return "<List>";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setTokenFactory(@NotNull TokenFactory<?> factory) {
|
||||
this._factory = factory;
|
||||
}
|
||||
|
||||
@Override
|
||||
public TokenFactory<?> getTokenFactory() {
|
||||
return _factory;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,70 @@
|
|||
package org.antlr.v4.runtime.tree.pattern;
|
||||
|
||||
import org.antlr.v4.runtime.CharStream;
|
||||
import org.antlr.v4.runtime.Token;
|
||||
import org.antlr.v4.runtime.TokenSource;
|
||||
|
||||
public class RuleTagToken implements Token {
|
||||
protected String ruleName;
|
||||
protected int ruleIndex;
|
||||
|
||||
public RuleTagToken(String ruleName, int ruleIndex) {
|
||||
this.ruleName = ruleName;
|
||||
this.ruleIndex = ruleIndex;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getChannel() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getText() {
|
||||
return "<"+ruleName+">";
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getType() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getLine() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getCharPositionInLine() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getTokenIndex() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getStartIndex() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getStopIndex() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public TokenSource getTokenSource() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public CharStream getInputStream() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return ruleName+":"+ruleIndex;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
package org.antlr.v4.runtime.tree.pattern;
|
||||
|
||||
import org.antlr.v4.runtime.CommonToken;
|
||||
|
||||
public class TokenTagToken extends CommonToken {
|
||||
protected String tokenName;
|
||||
|
||||
public TokenTagToken(String tokenName, int type) {
|
||||
super(type);
|
||||
this.tokenName = tokenName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getText() {
|
||||
return "<"+tokenName+">";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return tokenName+":"+type;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
package org.antlr.v4.test;
|
||||
|
||||
import org.antlr.v4.runtime.Lexer;
|
||||
import org.antlr.v4.runtime.Parser;
|
||||
import org.antlr.v4.runtime.Token;
|
||||
import org.antlr.v4.runtime.tree.ParseTree;
|
||||
import org.antlr.v4.runtime.tree.pattern.ParseTreePatternMatcher;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.File;
|
||||
import java.net.URL;
|
||||
import java.net.URLClassLoader;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
public class TestParseTreeMatcher extends BaseTest {
|
||||
@Test
|
||||
public void testChunking() throws Exception {
|
||||
// tests
|
||||
ParseTreePatternMatcher p = new ParseTreePatternMatcher();
|
||||
System.out.println( p.split("<ID> = <expr> ;") );
|
||||
System.out.println( p.split(" <ID> = <expr>") );
|
||||
System.out.println( p.split("<ID> = <expr>") );
|
||||
System.out.println( p.split("<expr>") );
|
||||
System.out.println(p.split("\\<x\\> foo"));
|
||||
System.out.println(p.split("foo \\<x\\> bar <tag>"));
|
||||
// System.out.println( p.split(">expr<") );
|
||||
|
||||
p.setDelimiters("<<", ">>", "$");
|
||||
System.out.println(p.split("<<ID>> = <<expr>> ;$<< ick $>>"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testTokenizingPattern() throws Exception {
|
||||
String grammar =
|
||||
"grammar T;\n" +
|
||||
"s : ID '=' expr ';' ;\n" +
|
||||
"expr : ID | INT ;\n" +
|
||||
"ID : [a-z]+ ;\n" +
|
||||
"INT : [0-9]+ ;\n" +
|
||||
"WS : [ \\r\\n\\t]+ -> skip ;\n";
|
||||
boolean ok =
|
||||
rawGenerateAndBuildRecognizer("T.g4", grammar, "TParser", "TLexer", false);
|
||||
assertTrue(ok);
|
||||
|
||||
ClassLoader loader =
|
||||
new URLClassLoader(new URL[] { new File(tmpdir).toURI().toURL() },
|
||||
ClassLoader.getSystemClassLoader());
|
||||
final Class<? extends Lexer> lexerClass = (Class<? extends Lexer>)loader.loadClass("TLexer");
|
||||
final Class<? extends Parser> parserClass = (Class<? extends Parser>)loader.loadClass("TParser");
|
||||
|
||||
ParseTreePatternMatcher p = new ParseTreePatternMatcher(lexerClass, parserClass);
|
||||
List<? extends Token> tokens = p.tokenizePattern("<ID> = <expr> ;");
|
||||
String results = tokens.toString();
|
||||
String expected = "[ID:3, [@-1,1:1='=',<1>,1:1], expr:1, [@-1,0:0=';',<2>,1:0]]";
|
||||
assertEquals(expected, results);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCompilingPattern() throws Exception {
|
||||
String grammar =
|
||||
"grammar T;\n" +
|
||||
"s : ID '=' expr ';' ;\n" +
|
||||
"expr : ID | INT ;\n" +
|
||||
"ID : [a-z]+ ;\n" +
|
||||
"INT : [0-9]+ ;\n" +
|
||||
"WS : [ \\r\\n\\t]+ -> skip ;\n";
|
||||
boolean ok =
|
||||
rawGenerateAndBuildRecognizer("T.g4", grammar, "TParser", "TLexer", false);
|
||||
assertTrue(ok);
|
||||
|
||||
ClassLoader loader =
|
||||
new URLClassLoader(new URL[] { new File(tmpdir).toURI().toURL() },
|
||||
ClassLoader.getSystemClassLoader());
|
||||
final Class<? extends Lexer> lexerClass = (Class<? extends Lexer>)loader.loadClass("TLexer");
|
||||
final Class<? extends Parser> parserClass = (Class<? extends Parser>)loader.loadClass("TParser");
|
||||
|
||||
ParseTreePatternMatcher p = new ParseTreePatternMatcher(lexerClass, parserClass);
|
||||
ParseTree t = p.compilePattern("s", "<ID> = <expr> ;");
|
||||
String results = t.toStringTree(p.getParser());
|
||||
String expected = "(s <ID> = expr ;)";
|
||||
assertEquals(expected, results);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue