jasder
/
antlr
1
0
Fork 1
Code Issues Pull Requests Releases 1 Wiki Activity

refactored Token.js to use es6 classes 

use const/let for better scoping
use jsdoc
This commit is contained in:
Camilo Roca 2020-03-16 00:16:11 +01:00
parent 5baa2326ad
commit 5b607b34ba
1 changed files with 108 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

218
runtime/JavaScript/src/antlr4/Token.js
View File

@ -2,150 +2,148 @@
* Use of this file is governed by the BSD 3-clause license that * Use of this file is governed by the BSD 3-clause license that
* can be found in the LICENSE.txt file in the project root. * can be found in the LICENSE.txt file in the project root.
*/ */
//
// A token has properties: text, type, line, character position in the line /**
// (so we can ignore tabs), token channel, index, and source from which * A token has properties: text, type, line, character position in the line
// we obtained this token. * (so we can ignore tabs), token channel, index, and source from which
* we obtained this token.
*/
class Token {
constructor() {
this.source = null;
this.type = null; // token type of the token
this.channel = null; // The parser ignores everything not on DEFAULT_CHANNEL
this.start = null; // optional; return -1 if not implemented.
this.stop = null; // optional; return -1 if not implemented.
this.tokenIndex = null; // from 0..n-1 of the token object in the input stream
this.line = null; // line=1..n of the 1st character
this.column = null; // beginning of the line at which it occurs, 0..n-1
this._text = null; // text of the token.
}
function Token() { getTokenSource() {
this.source = null; return this.source[0];
this.type = null; // token type of the token }
this.channel = null; // The parser ignores everything not on DEFAULT_CHANNEL
this.start = null; // optional; return -1 if not implemented. getInputStream() {
this.stop = null; // optional; return -1 if not implemented. return this.source[1];
this.tokenIndex = null; // from 0..n-1 of the token object in the input stream }
this.line = null; // line=1..n of the 1st character
this.column = null; // beginning of the line at which it occurs, 0..n-1 get text(){
this._text = null; // text of the token. return this._text;
return this; }
set text(text) {
this._text = text;
}
} }
Token.INVALID_TYPE = 0; Token.INVALID_TYPE = 0;
// During lookahead operations, this "token" signifies we hit rule end ATN state /**
// and did not follow it despite needing to. * During lookahead operations, this "token" signifies we hit rule end ATN state
* and did not follow it despite needing to.
*/
Token.EPSILON = -2; Token.EPSILON = -2;
Token.MIN_USER_TOKEN_TYPE = 1; Token.MIN_USER_TOKEN_TYPE = 1;
Token.EOF = -1; Token.EOF = -1;
// All tokens go to the parser (unless skip() is called in that rule) /**
// on a particular "channel". The parser tunes to a particular channel * All tokens go to the parser (unless skip() is called in that rule)
// so that whitespace etc... can go to the parser on a "hidden" channel. * on a particular "channel". The parser tunes to a particular channel
* so that whitespace etc... can go to the parser on a "hidden" channel.
*/
Token.DEFAULT_CHANNEL = 0; Token.DEFAULT_CHANNEL = 0;
// Anything on different channel than DEFAULT_CHANNEL is not parsed /**
// by parser. * Anything on different channel than DEFAULT_CHANNEL is not parsed
* by parser.
*/
Token.HIDDEN_CHANNEL = 1; Token.HIDDEN_CHANNEL = 1;
// Explicitly set the text for this token. If {code text} is not
// {@code null}, then {@link //getText} will return this value rather than
// extracting the text from the input.
//
// @param text The explicit text of the token, or {@code null} if the text
// should be obtained from the input along with the start and stop indexes
// of the token.
Object.defineProperty(Token.prototype, "text", { class CommonToken extends Token {
get : function() { constructor(source, type, channel, start, stop) {
return this._text; super();
}, this.source = source !== undefined ? source : CommonToken.EMPTY_SOURCE;
set : function(text) { this.type = type !== undefined ? type : null;
this._text = text; this.channel = channel !== undefined ? channel : Token.DEFAULT_CHANNEL;
this.start = start !== undefined ? start : -1;
this.stop = stop !== undefined ? stop : -1;
this.tokenIndex = -1;
if (this.source[0] !== null) {
this.line = source[0].line;
this.column = source[0].column;
} else {
this.column = -1;
}
} }
});
Token.prototype.getTokenSource = function() { /**
return this.source[0]; * Constructs a new {@link CommonToken} as a copy of another {@link Token}.
}; *
* <p>
Token.prototype.getInputStream = function() { * If {@code oldToken} is also a {@link CommonToken} instance, the newly
return this.source[1]; * constructed token will share a reference to the {@link //text} field and
}; * the {@link Pair} stored in {@link //source}. Otherwise, {@link //text} will
* be assigned the result of calling {@link //getText}, and {@link //source}
function CommonToken(source, type, channel, start, stop) { * will be constructed from the result of {@link Token//getTokenSource} and
Token.call(this); * {@link Token//getInputStream}.</p>
this.source = source !== undefined ? source : CommonToken.EMPTY_SOURCE; *
this.type = type !== undefined ? type : null; * @param oldToken The token to copy.
this.channel = channel !== undefined ? channel : Token.DEFAULT_CHANNEL; */
this.start = start !== undefined ? start : -1; clone() {
this.stop = stop !== undefined ? stop : -1; const t = new CommonToken(this.source, this.type, this.channel, this.start, this.stop);
this.tokenIndex = -1; t.tokenIndex = this.tokenIndex;
if (this.source[0] !== null) { t.line = this.line;
this.line = source[0].line; t.column = this.column;
this.column = source[0].column; t.text = this.text;
} else { return t;
this.column = -1;
} }
return this;
}
CommonToken.prototype = Object.create(Token.prototype); toString() {
CommonToken.prototype.constructor = CommonToken; let txt = this.text;
if (txt !== null) {
txt = txt.replace(/\n/g, "\\n").replace(/\r/g, "\\r").replace(/\t/g, "\\t");
} else {
txt = "<no text>";
}
return "[@" + this.tokenIndex + "," + this.start + ":" + this.stop + "='" +
txt + "',<" + this.type + ">" +
(this.channel > 0 ? ",channel=" + this.channel : "") + "," +
this.line + ":" + this.column + "]";
}
// An empty {@link Pair} which is used as the default value of get text(){
// {@link //source} for tokens that do not have a source.
CommonToken.EMPTY_SOURCE = [ null, null ];
// Constructs a new {@link CommonToken} as a copy of another {@link Token}.
//
// <p>
// If {@code oldToken} is also a {@link CommonToken} instance, the newly
// constructed token will share a reference to the {@link //text} field and
// the {@link Pair} stored in {@link //source}. Otherwise, {@link //text} will
// be assigned the result of calling {@link //getText}, and {@link //source}
// will be constructed from the result of {@link Token//getTokenSource} and
// {@link Token//getInputStream}.</p>
//
// @param oldToken The token to copy.
//
CommonToken.prototype.clone = function() {
var t = new CommonToken(this.source, this.type, this.channel, this.start,
this.stop);
t.tokenIndex = this.tokenIndex;
t.line = this.line;
t.column = this.column;
t.text = this.text;
return t;
};
Object.defineProperty(CommonToken.prototype, "text", {
get : function() {
if (this._text !== null) { if (this._text !== null) {
return this._text; return this._text;
} }
var input = this.getInputStream(); const input = this.getInputStream();
if (input === null) { if (input === null) {
return null; return null;
} }
var n = input.size; const n = input.size;
if (this.start < n && this.stop < n) { if (this.start < n && this.stop < n) {
return input.getText(this.start, this.stop); return input.getText(this.start, this.stop);
} else { } else {
return "<EOF>"; return "<EOF>";
} }
}, }
set : function(text) {
set text(text) {
this._text = text; this._text = text;
} }
}); }
CommonToken.prototype.toString = function() { /**
var txt = this.text; * An empty {@link Pair} which is used as the default value of
if (txt !== null) { * {@link //source} for tokens that do not have a source.
txt = txt.replace(/\n/g, "\\n").replace(/\r/g, "\\r").replace(/\t/g, "\\t"); */
} else { CommonToken.EMPTY_SOURCE = [ null, null ];
txt = "<no text>";
}
return "[@" + this.tokenIndex + "," + this.start + ":" + this.stop + "='" +
txt + "',<" + this.type + ">" +
(this.channel > 0 ? ",channel=" + this.channel : "") + "," +
this.line + ":" + this.column + "]";
};
exports.Token = Token; module.exports = {
exports.CommonToken = CommonToken; Token,
CommonToken
}