forgetest1
/
antlr
forked from jasder/antlr
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

First pass on token stream types

This commit is contained in:
Peter Boyer 2015-12-15 15:08:06 -05:00
parent e7e1138e5b
commit 7d07d7514e
8 changed files with 458 additions and 432 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

230
runtime/Go/src/antlr4/BufferedTokenStream.go
View File

@ -16,7 +16,7 @@ type TokenStream interface {
}
// this is just to keep meaningful parameter types to Parser
// bt is just to keep meaningful parameter types to Parser
type BufferedTokenStream struct {
tokenSource TokenStream
tokens []Token
@ -28,13 +28,13 @@ func NewBufferedTokenStream(tokenSource TokenStream) BufferedTokenStream {
ts := new(BufferedTokenStream)
// The {@link TokenSource} from which tokens for this stream are fetched.
// The {@link TokenSource} from which tokens for bt stream are fetched.
ts.tokenSource = tokenSource
// A collection of all tokens fetched from the token source. The list is
// considered a complete view of the input once {@link //fetchedEOF} is set
// to {@code true}.
ts.tokens = []
ts.tokens = make([]Token, 0)
// The index into {@link //tokens} of the current token (next token to
// {@link //consume}). {@link //tokens}{@code [}{@link //p}{@code ]} should
@ -60,55 +60,55 @@ func NewBufferedTokenStream(tokenSource TokenStream) BufferedTokenStream {
// //LA}.</li>
// <li>{@link //fetch}: The check to prevent adding multiple EOF symbols
// into
// {@link //tokens} is trivial with this field.</li>
// {@link //tokens} is trivial with bt field.</li>
// <ul>
ts.fetchedEOF = false
return ts
}
func (this *BufferedTokenStream) mark() int {
func (bt *BufferedTokenStream) mark() int {
return 0
}
func (this *BufferedTokenStream) release(marker) {
func (bt *BufferedTokenStream) release(marker) {
// no resources to release
}
func (this *BufferedTokenStream) reset() {
this.seek(0)
func (bt *BufferedTokenStream) reset() {
bt.seek(0)
}
func (this *BufferedTokenStream) seek(index int) {
this.lazyInit()
this.index = this.adjustSeekIndex(index)
func (bt *BufferedTokenStream) seek(index int) {
bt.lazyInit()
bt.index = bt.adjustSeekIndex(index)
}
func (this *BufferedTokenStream) get(index int) {
this.lazyInit()
return this.tokens[index]
func (bt *BufferedTokenStream) get(index int) {
bt.lazyInit()
return bt.tokens[index]
}
func (this *BufferedTokenStream) consume() {
func (bt *BufferedTokenStream) consume() {
var skipEofCheck = false
if (this.index >= 0) {
if (this.fetchedEOF) {
if (bt.index >= 0) {
if (bt.fetchedEOF) {
// the last token in tokens is EOF. skip check if p indexes any
// fetched token except the last.
skipEofCheck = this.index < len(this.tokens) - 1
skipEofCheck = bt.index < len(bt.tokens) - 1
} else {
// no EOF token in tokens. skip check if p indexes a fetched token.
skipEofCheck = this.index < this.tokens.length
skipEofCheck = bt.index < bt.tokens.length
}
} else {
// not yet initialized
skipEofCheck = false
}
if (!skipEofCheck && this.LA(1) == Token.EOF) {
throw "cannot consume EOF"
if (!skipEofCheck && bt.LA(1) == Token.EOF) {
panic( "cannot consume EOF" )
}
if (this.sync(this.index + 1)) {
this.index = this.adjustSeekIndex(this.index + 1)
if (bt.sync(bt.index + 1)) {
bt.index = bt.adjustSeekIndex(bt.index + 1)
}
}
@ -118,10 +118,10 @@ func (this *BufferedTokenStream) consume() {
// {@code false}.
// @see //get(int i)
// /
func (this *BufferedTokenStream) sync(i) {
var n = i - this.tokens.length + 1 // how many more elements we need?
func (bt *BufferedTokenStream) sync(i) {
var n = i - bt.tokens.length + 1 // how many more elements we need?
if (n > 0) {
var fetched = this.fetch(n)
var fetched = bt.fetch(n)
return fetched >= n
}
return true
@ -131,16 +131,16 @@ func (this *BufferedTokenStream) sync(i) {
//
// @return The actual number of elements added to the buffer.
// /
func (this *BufferedTokenStream) fetch(n) {
if (this.fetchedEOF) {
func (bt *BufferedTokenStream) fetch(n) {
if (bt.fetchedEOF) {
return 0
}
for (var i = 0 i < n i++) {
var t = this.tokenSource.nextToken()
t.tokenIndex = this.tokens.length
this.tokens.push(t)
for i := 0; i < n; i++ {
var t = bt.tokenSource.nextToken()
t.tokenIndex = bt.tokens.length
bt.tokens.push(t)
if (t.type == Token.EOF) {
this.fetchedEOF = true
bt.fetchedEOF = true
return i + 1
}
}
@ -148,91 +148,91 @@ func (this *BufferedTokenStream) fetch(n) {
}
// Get all tokens from start..stop inclusively///
func (this *BufferedTokenStream) getTokens(start, stop, types) {
func (bt *BufferedTokenStream) getTokens(start, stop, types) {
if (types == undefined) {
types = null
types = nil
}
if (start < 0 || stop < 0) {
return null
return nil
}
this.lazyInit()
bt.lazyInit()
var subset = []
if (stop >= this.tokens.length) {
stop = this.tokens.length - 1
if (stop >= bt.tokens.length) {
stop = bt.tokens.length - 1
}
for (var i = start i < stop i++) {
var t = this.tokens[i]
for i := start; i < stop; i++ {
var t = bt.tokens[i]
if (t.type == Token.EOF) {
break
}
if (types == null || types.contains(t.type)) {
if (types == nil || types.contains(t.type)) {
subset.push(t)
}
}
return subset
}
func (this *BufferedTokenStream) LA(i) {
return this.LT(i).type
func (bt *BufferedTokenStream) LA(i) {
return bt.LT(i).type
}
func (this *BufferedTokenStream) LB(k) {
if (this.index - k < 0) {
return null
func (bt *BufferedTokenStream) LB(k) {
if (bt.index - k < 0) {
return nil
}
return this.tokens[this.index - k]
return bt.tokens[bt.index - k]
}
func (this *BufferedTokenStream) LT(k) {
this.lazyInit()
func (bt *BufferedTokenStream) LT(k) {
bt.lazyInit()
if (k == 0) {
return null
return nil
}
if (k < 0) {
return this.LB(-k)
return bt.LB(-k)
}
var i = this.index + k - 1
this.sync(i)
if (i >= this.tokens.length) { // return EOF token
var i = bt.index + k - 1
bt.sync(i)
if (i >= bt.tokens.length) { // return EOF token
// EOF must be last token
return this.tokens[this.tokens.length - 1]
return bt.tokens[bt.tokens.length - 1]
}
return this.tokens[i]
return bt.tokens[i]
}
// Allowed derived classes to modify the behavior of operations which change
// the current stream position by adjusting the target token index of a seek
// operation. The default implementation simply returns {@code i}. If an
// exception is thrown in this method, the current stream index should not be
// exception is panic(n in bt method, the current stream index should not be
// changed.
//
// <p>For example, {@link CommonTokenStream} overrides this method to ensure
// <p>For example, {@link CommonTokenStream} overrides bt method to ensure
// that
// the seek target is always an on-channel token.</p>
//
// @param i The target token index.
// @return The adjusted target token index.
func (this *BufferedTokenStream) adjustSeekIndex(i) {
func (bt *BufferedTokenStream) adjustSeekIndex(i) {
return i
}
func (this *BufferedTokenStream) lazyInit() {
if (this.index == -1) {
this.setup()
func (bt *BufferedTokenStream) lazyInit() {
if (bt.index == -1) {
bt.setup()
}
}
func (this *BufferedTokenStream) setup() {
this.sync(0)
this.index = this.adjustSeekIndex(0)
func (bt *BufferedTokenStream) setup() {
bt.sync(0)
bt.index = bt.adjustSeekIndex(0)
}
// Reset this token stream by setting its token source.///
func (this *BufferedTokenStream) setTokenSource(tokenSource) {
this.tokenSource = tokenSource
this.tokens = []
this.index = -1
// Reset bt token stream by setting its token source.///
func (bt *BufferedTokenStream) setTokenSource(tokenSource) {
bt.tokenSource = tokenSource
bt.tokens = []
bt.index = -1
}
@ -240,19 +240,19 @@ func (this *BufferedTokenStream) setTokenSource(tokenSource) {
// Return i if tokens[i] is on channel. Return -1 if there are no tokens
// on channel between i and EOF.
// /
func (this *BufferedTokenStream) nextTokenOnChannel(i, channel) {
this.sync(i)
if (i >= this.tokens.length) {
func (bt *BufferedTokenStream) nextTokenOnChannel(i, channel) {
bt.sync(i)
if (i >= bt.tokens.length) {
return -1
}
var token = this.tokens[i]
while (token.channel != this.channel) {
var token = bt.tokens[i]
while (token.channel != bt.channel) {
if (token.type == Token.EOF) {
return -1
}
i += 1
this.sync(i)
token = this.tokens[i]
bt.sync(i)
token = bt.tokens[i]
}
return i
}
@ -260,8 +260,8 @@ func (this *BufferedTokenStream) nextTokenOnChannel(i, channel) {
// Given a starting index, return the index of the previous token on channel.
// Return i if tokens[i] is on channel. Return -1 if there are no tokens
// on channel between i and 0.
func (this *BufferedTokenStream) previousTokenOnChannel(i, channel) {
while (i >= 0 && this.tokens[i].channel !== channel) {
func (bt *BufferedTokenStream) previousTokenOnChannel(i, channel) {
while (i >= 0 && bt.tokens[i].channel != channel) {
i -= 1
}
return i
@ -270,52 +270,52 @@ func (this *BufferedTokenStream) previousTokenOnChannel(i, channel) {
// Collect all tokens on specified channel to the right of
// the current token up until we see a token on DEFAULT_TOKEN_CHANNEL or
// EOF. If channel is -1, find any non default channel token.
func (this *BufferedTokenStream) getHiddenTokensToRight(tokenIndex,
func (bt *BufferedTokenStream) getHiddenTokensToRight(tokenIndex,
channel) {
if (channel == undefined) {
channel = -1
}
this.lazyInit()
if (this.tokenIndex < 0 || tokenIndex >= this.tokens.length) {
throw "" + tokenIndex + " not in 0.." + this.tokens.length - 1
bt.lazyInit()
if (bt.tokenIndex < 0 || tokenIndex >= bt.tokens.length) {
panic( "" + tokenIndex + " not in 0.." + bt.tokens.length - 1
}
var nextOnChannel = this.nextTokenOnChannel(tokenIndex + 1,
var nextOnChannel = bt.nextTokenOnChannel(tokenIndex + 1,
Lexer.DEFAULT_TOKEN_CHANNEL)
var from_ = tokenIndex + 1
// if none onchannel to right, nextOnChannel=-1 so set to = last token
var to = nextOnChannel == -1 ? this.tokens.length - 1 : nextOnChannel
return this.filterForChannel(from_, to, channel)
var to = nextOnChannel == -1 ? bt.tokens.length - 1 : nextOnChannel
return bt.filterForChannel(from_, to, channel)
}
// Collect all tokens on specified channel to the left of
// the current token up until we see a token on DEFAULT_TOKEN_CHANNEL.
// If channel is -1, find any non default channel token.
func (this *BufferedTokenStream) getHiddenTokensToLeft(tokenIndex,
func (bt *BufferedTokenStream) getHiddenTokensToLeft(tokenIndex,
channel) {
if (channel == undefined) {
channel = -1
}
this.lazyInit()
if (tokenIndex < 0 || tokenIndex >= this.tokens.length) {
throw "" + tokenIndex + " not in 0.." + this.tokens.length - 1
bt.lazyInit()
if (tokenIndex < 0 || tokenIndex >= bt.tokens.length) {
panic( "" + tokenIndex + " not in 0.." + bt.tokens.length - 1
}
var prevOnChannel = this.previousTokenOnChannel(tokenIndex - 1,
var prevOnChannel = bt.previousTokenOnChannel(tokenIndex - 1,
Lexer.DEFAULT_TOKEN_CHANNEL)
if (prevOnChannel == tokenIndex - 1) {
return null
return nil
}
// if none on channel to left, prevOnChannel=-1 then from=0
var from_ = prevOnChannel + 1
var to = tokenIndex - 1
return this.filterForChannel(from_, to, channel)
return bt.filterForChannel(from_, to, channel)
}
func (this *BufferedTokenStream) filterForChannel(left, right, channel) {
func (bt *BufferedTokenStream) filterForChannel(left, right, channel) {
var hidden = []
for (var i = left i < right + 1 i++) {
var t = this.tokens[i]
for var i = left; i < right + 1; i++ {
var t = bt.tokens[i]
if (channel == -1) {
if (t.channel !== Lexer.DEFAULT_TOKEN_CHANNEL) {
if (t.channel != Lexer.DEFAULT_TOKEN_CHANNEL) {
hidden.push(t)
}
} else if (t.channel == channel) {
@ -323,21 +323,21 @@ func (this *BufferedTokenStream) filterForChannel(left, right, channel) {
}
}
if (hidden.length == 0) {
return null
return nil
}
return hidden
}
func (this *BufferedTokenStream) getSourceName() {
return this.tokenSource.getSourceName()
func (bt *BufferedTokenStream) getSourceName() {
return bt.tokenSource.getSourceName()
}
// Get the text of all tokens in this buffer.///
func (this *BufferedTokenStream) getText(interval) string {
this.lazyInit()
this.fill()
if (interval == undefined || interval == null) {
interval = new Interval(0, this.tokens.length - 1)
// Get the text of all tokens in bt buffer.///
func (bt *BufferedTokenStream) getText(interval) string {
bt.lazyInit()
bt.fill()
if (interval == undefined || interval == nil) {
interval = new Interval(0, bt.tokens.length - 1)
}
var start = interval.start
if (start instanceof Token) {
@ -347,15 +347,15 @@ func (this *BufferedTokenStream) getText(interval) string {
if (stop instanceof Token) {
stop = stop.tokenIndex
}
if (start == null || stop == null || start < 0 || stop < 0) {
if (start == nil || stop == nil || start < 0 || stop < 0) {
return ""
}
if (stop >= this.tokens.length) {
stop = this.tokens.length - 1
if (stop >= bt.tokens.length) {
stop = bt.tokens.length - 1
}
var s = ""
for (var i = start i < stop + 1 i++) {
var t = this.tokens[i]
for i := start; i < stop + 1; i++ {
var t = bt.tokens[i]
if (t.type == Token.EOF) {
break
}
@ -365,9 +365,9 @@ func (this *BufferedTokenStream) getText(interval) string {
}
// Get all tokens from lexer until EOF///
func (this *BufferedTokenStream) fill() {
this.lazyInit()
while (this.fetch(1000) == 1000) {
func (bt *BufferedTokenStream) fill() {
bt.lazyInit()
for (bt.fetch(1000) == 1000) {
continue
}
}

22
runtime/Go/src/antlr4/CommonTokenFactory.go
View File

@ -5,11 +5,17 @@
package antlr
type TokenFactory struct {
type TokenFactory interface {
}
func NewCommonTokenFactory(copyText) {
TokenFactory.call(this)
type CommonTokenFactory struct {
copyText bool
}
func NewCommonTokenFactory(copyText bool) CommonTokenFactory {
tf := new(CommonTokenFactory)
// Indicates whether {@link CommonToken//setText} should be called after
// constructing tokens to explicitly set the text. This is useful for cases
// where the input stream might not be able to provide arbitrary substrings
@ -24,12 +30,10 @@ func NewCommonTokenFactory(copyText) {
// The default value is {@code false} to avoid the performance and memory
// overhead of copying text for every token unless explicitly requested.</p>
//
this.copyText = copyText==undefined ? false : copyText
return this
}
tf.copyText = copyText
CommonTokenFactory.prototype = Object.create(TokenFactory.prototype)
CommonTokenFactory.prototype.constructor = CommonTokenFactory
return tf
}
//
// The default {@link CommonTokenFactory} instance.
@ -38,7 +42,7 @@ CommonTokenFactory.prototype.constructor = CommonTokenFactory
// This token factory does not explicitly copy token text when constructing
// tokens.</p>
//
CommonTokenFactory.DEFAULT = new CommonTokenFactory()
var CommonTokenFactoryDEFAULT = NewCommonTokenFactory(false)
func (this *CommonTokenFactory) create(source, type, text, channel, start, stop, line, column) {
var t = new CommonToken(source, type, channel, start, stop)

69
runtime/Go/src/antlr4/CommonTokenStream.go
View File

@ -25,71 +25,72 @@
package antlr
var Token = require('./Token').Token
var BufferedTokenStream = require('./BufferedTokenStream').BufferedTokenStream
func CommonTokenStream(lexer, channel) {
BufferedTokenStream.call(this, lexer)
this.channel = channel==undefined ? Token.DEFAULT_CHANNEL : channel
return this
type CommonTokenStream struct {
BufferedTokenStream
}
CommonTokenStream.prototype = Object.create(BufferedTokenStream.prototype)
CommonTokenStream.prototype.constructor = CommonTokenStream
func NewCommonTokenStream(lexer Lexer, channel) {
func (this *CommonTokenStream) adjustSeekIndex(i) {
return this.nextTokenOnChannel(i, this.channel)
ts := new(BufferedTokenStream)
BufferedTokenStream.call(ts, lexer)
ts.channel = channel
return ts
}
func (this *CommonTokenStream) LB(k) {
if (k==0 || this.index-k<0) {
return null
func (ts *CommonTokenStream) adjustSeekIndex(i int) {
return ts.nextTokenOnChannel(i, ts.channel)
}
func (ts *CommonTokenStream) LB(k int) {
if (k==0 || ts.index-k<0) {
return nil
}
var i = this.index
var i = ts.index
var n = 1
// find k good tokens looking backwards
while (n <= k) {
for (n <= k) {
// skip off-channel tokens
i = this.previousTokenOnChannel(i - 1, this.channel)
i = ts.previousTokenOnChannel(i - 1, ts.channel)
n += 1
}
if (i < 0) {
return null
return nil
}
return this.tokens[i]
return ts.tokens[i]
}
func (this *CommonTokenStream) LT(k) {
this.lazyInit()
func (ts *CommonTokenStream) LT(k int) {
ts.lazyInit()
if (k == 0) {
return null
return nil
}
if (k < 0) {
return this.LB(-k)
return ts.LB(-k)
}
var i = this.index
var i = ts.index
var n = 1 // we know tokens[pos] is a good one
// find k good tokens
while (n < k) {
for n < k {
// skip off-channel tokens, but make sure to not look past EOF
if (this.sync(i + 1)) {
i = this.nextTokenOnChannel(i + 1, this.channel)
if (ts.sync(i + 1)) {
i = ts.nextTokenOnChannel(i + 1, ts.channel)
}
n += 1
}
return this.tokens[i]
return ts.tokens[i]
}
// Count EOF just once.///
func (this *CommonTokenStream) getNumberOfOnChannelTokens() {
func (ts *CommonTokenStream) getNumberOfOnChannelTokens() {
var n = 0
this.fill()
for (var i =0 i< this.tokens.lengthi++) {
var t = this.tokens[i]
if( t.channel==this.channel) {
ts.fill()
for i := 0; i < ts.tokens.length; i++ {
var t = ts.tokens[i]
if t.channel==ts.channel {
n += 1
}
if( t.type==Token.EOF) {
if t.type==Token.EOF {
break
}
}

19
runtime/Go/src/antlr4/FileStream.go
View File

@ -3,19 +3,20 @@ package antlr
//
// This is an InputStream that is loaded from a file all at once
// when you construct the object.
//
var InputStream = require('./InputStream').InputStream
var isNodeJs = typeof window == 'undefined' && typeof importScripts == 'undefined'
var fs = isNodeJs ? require("fs") : null
//
type FileStream struct {
filename string
}
func FileStream(fileName) {
var data = fs.readFileSync(fileName, "utf8")
InputStream.call(this, data)
this.fileName = fileName
return this
fs.fileName = fileName
return fs
}
FileStream.prototype = Object.create(InputStream.prototype)
FileStream.prototype.constructor = FileStream

99
runtime/Go/src/antlr4/InputStream.go
View File

@ -1,105 +1,104 @@
package antlr
var Token = require('./Token').Token
// Vacuum all input from a string and then treat it like a buffer.
type InputStream struct {
name string
strdata string
index int
size int
}
func NewInputStream(data string) *InputStream {
is := new(InputStream)
is.name = "<empty>"
is.strdata = data
_loadString(is)
return is
}
func _loadString(stream) {
stream._index = 0
stream.index = 0
stream.data = []
for (var i = 0 i < stream.strdata.length i++) {
stream.data.push(stream.strdata.charCodeAt(i))
}
stream._size = stream.data.length
stream.size = stream.data.length
}
func InputStream(data) {
this.name = "<empty>"
this.strdata = data
_loadString(this)
return this
}
Object.defineProperty(InputStream.prototype, "index", {
get : function() {
return this._index
}
})
Object.defineProperty(InputStream.prototype, "size", {
get : function() {
return this._size
}
})
// Reset the stream so that it's in the same state it was
// when the object was created *except* the data array is not
// touched.
//
func (this *InputStream) reset() {
this._index = 0
func (is *InputStream) reset() {
is.index = 0
}
func (this *InputStream) consume() {
if (this._index >= this._size) {
// assert this.LA(1) == Token.EOF
throw ("cannot consume EOF")
func (is *InputStream) consume() {
if (is.index >= is.size) {
// assert is.LA(1) == Token.EOF
panic ("cannot consume EOF")
}
this._index += 1
is.index += 1
}
func (this *InputStream) LA(offset) {
func (is *InputStream) LA(offset int) {
if (offset == 0) {
return 0 // undefined
}
if (offset < 0) {
offset += 1 // e.g., translate LA(-1) to use offset=0
}
var pos = this._index + offset - 1
if (pos < 0 || pos >= this._size) { // invalid
var pos = is.index + offset - 1
if (pos < 0 || pos >= is.size) { // invalid
return Token.EOF
}
return this.data[pos]
return is.data[pos]
}
func (this *InputStream) LT(offset) {
return this.LA(offset)
func (is *InputStream) LT(offset int) {
return is.LA(offset)
}
// mark/release do nothing we have entire buffer
func (this *InputStream) mark() {
func (is *InputStream) mark() {
return -1
}
func (this *InputStream) release(marker) {
func (is *InputStream) release(marker int) {
}
// consume() ahead until p==_index can't just set p=_index as we must
// consume() ahead until p==index can't just set p=index as we must
// update line and column. If we seek backwards, just set p
//
func (this *InputStream) seek(_index) {
if (_index <= this._index) {
this._index = _index // just jump don't update stream state (line,
// ...)
func (is *InputStream) seek(index int) {
if (index <= is.index) {
is.index = index // just jump don't update stream state (line,...)
return
}
// seek forward
this._index = Math.min(_index, this._size)
is.index = Math.min(index, is.size)
}
func (this *InputStream) getText(start, stop) {
if (stop >= this._size) {
stop = this._size - 1
func (is *InputStream) getText(start int, stop int) string {
if (stop >= is.size) {
stop = is.size - 1
}
if (start >= this._size) {
if (start >= is.size) {
return ""
} else {
return this.strdata.slice(start, stop + 1)
return is.strdata.slice(start, stop + 1)
}
}
func (this *InputStream) toString() {
return this.strdata
func (is *InputStream) toString() string {
return is.strdata
}

180
runtime/Go/src/antlr4/IntervalSet.go
View File

@ -1,125 +1,129 @@
package antlr
var Token = require('./Token').Token
type Interval struct {
start int
stop int
}
/* stop is not included! */
func Interval(start, stop) {
this.start = start
this.stop = stop
return this
func NewInterval(start int, stop int) Interval{
i := new(Interval)
i.start = start
i.stop = stop
return i
}
func (this *Interval) contains(item) {
return item >= this.start && item < this.stop
func (i *Interval) contains(item int) {
return item >= i.start && item < i.stop
}
func (this *Interval) toString() {
if(this.start==this.stop-1) {
return this.start.toString()
func (i *Interval) toString() {
if(i.start==i.stop-1) {
return i.start.toString()
} else {
return this.start.toString() + ".." + (this.stop-1).toString()
return i.start.toString() + ".." + (i.stop-1).toString()
}
}
Object.defineProperty(Interval.prototype, "length", {
get : function() {
return this.stop - this.start
return i.stop - i.start
}
})
type IntervalSet struct {
this.intervals = null
this.readOnly = false
i.intervals = null
i.readOnly = false
}
func (this *IntervalSet) first(v) {
if (this.intervals == null || this.intervals.length==0) {
func (i *IntervalSet) first(v) {
if (i.intervals == null || i.intervals.length==0) {
return Token.INVALID_TYPE
} else {
return this.intervals[0].start
return i.intervals[0].start
}
}
func (this *IntervalSet) addOne(v) {
this.addInterval(new Interval(v, v + 1))
func (i *IntervalSet) addOne(v) {
i.addInterval(new Interval(v, v + 1))
}
func (this *IntervalSet) addRange(l, h) {
this.addInterval(new Interval(l, h + 1))
func (i *IntervalSet) addRange(l, h) {
i.addInterval(new Interval(l, h + 1))
}
func (this *IntervalSet) addInterval(v) {
if (this.intervals == null) {
this.intervals = []
this.intervals.push(v)
func (i *IntervalSet) addInterval(v) {
if (i.intervals == null) {
i.intervals = []
i.intervals.push(v)
} else {
// find insert pos
for (var k = 0 k < this.intervals.length k++) {
var i = this.intervals[k]
for (var k = 0 k < i.intervals.length k++) {
var i = i.intervals[k]
// distinct range -> insert
if (v.stop < i.start) {
this.intervals.splice(k, 0, v)
i.intervals.splice(k, 0, v)
return
}
// contiguous range -> adjust
else if (v.stop == i.start) {
this.intervals[k].start = v.start
i.intervals[k].start = v.start
return
}
// overlapping range -> adjust and reduce
else if (v.start <= i.stop) {
this.intervals[k] = new Interval(Math.min(i.start, v.start), Math.max(i.stop, v.stop))
this.reduce(k)
i.intervals[k] = new Interval(Math.min(i.start, v.start), Math.max(i.stop, v.stop))
i.reduce(k)
return
}
}
// greater than any existing
this.intervals.push(v)
i.intervals.push(v)
}
}
func (this *IntervalSet) addSet(other) {
if (other.intervals !== null) {
func (i *IntervalSet) addSet(other) {
if (other.intervals != null) {
for (var k = 0 k < other.intervals.length k++) {
var i = other.intervals[k]
this.addInterval(new Interval(i.start, i.stop))
i.addInterval(new Interval(i.start, i.stop))
}
}
return this
return i
}
func (this *IntervalSet) reduce(k) {
func (i *IntervalSet) reduce(k) {
// only need to reduce if k is not the last
if (k < this.intervalslength - 1) {
var l = this.intervals[k]
var r = this.intervals[k + 1]
if (k < i.intervalslength - 1) {
var l = i.intervals[k]
var r = i.intervals[k + 1]
// if r contained in l
if (l.stop >= r.stop) {
this.intervals.pop(k + 1)
this.reduce(k)
i.intervals.pop(k + 1)
i.reduce(k)
} else if (l.stop >= r.start) {
this.intervals[k] = new Interval(l.start, r.stop)
this.intervals.pop(k + 1)
i.intervals[k] = new Interval(l.start, r.stop)
i.intervals.pop(k + 1)
}
}
}
func (this *IntervalSet) complement(start, stop) {
func (i *IntervalSet) complement(start, stop) {
var result = new IntervalSet()
result.addInterval(new Interval(start,stop+1))
for(var i=0 i<this.intervals.length i++) {
result.removeRange(this.intervals[i])
for(var i=0 i<i.intervals.length i++) {
result.removeRange(i.intervals[i])
}
return result
}
func (this *IntervalSet) contains(item) {
if (this.intervals == null) {
func (i *IntervalSet) contains(item) {
if (i.intervals == null) {
return false
} else {
for (var k = 0 k < this.intervals.length k++) {
if(this.intervals[k].contains(item)) {
for (var k = 0 k < i.intervals.length k++) {
if(i.intervals[k].contains(item)) {
return true
}
}
@ -130,100 +134,100 @@ func (this *IntervalSet) contains(item) {
Object.defineProperty(IntervalSet.prototype, "length", {
get : function() {
var len = 0
this.intervals.map(function(i) {len += i.length})
i.intervals.map(function(i) {len += i.length})
return len
}
})
func (this *IntervalSet) removeRange(v) {
func (i *IntervalSet) removeRange(v) {
if(v.start==v.stop-1) {
this.removeOne(v.start)
} else if (this.intervals!==null) {
i.removeOne(v.start)
} else if (i.intervals!=nil) {
var k = 0
for(var n=0 n<this.intervals.length n++) {
var i = this.intervals[k]
for n :=0 n<i.intervals.length n++) {
var i = i.intervals[k]
// intervals are ordered
if (v.stop<=i.start) {
return
}
// check for including range, split it
else if(v.start>i.start && v.stop<i.stop) {
this.intervals[k] = new Interval(i.start, v.start)
i.intervals[k] = new Interval(i.start, v.start)
var x = new Interval(v.stop, i.stop)
this.intervals.splice(k, 0, x)
i.intervals.splice(k, 0, x)
return
}
// check for included range, remove it
else if(v.start<=i.start && v.stop>=i.stop) {
this.intervals.splice(k, 1)
i.intervals.splice(k, 1)
k = k - 1 // need another pass
}
// check for lower boundary
else if(v.start<i.stop) {
this.intervals[k] = new Interval(i.start, v.start)
i.intervals[k] = new Interval(i.start, v.start)
}
// check for upper boundary
else if(v.stop<i.stop) {
this.intervals[k] = new Interval(v.stop, i.stop)
i.intervals[k] = new Interval(v.stop, i.stop)
}
k += 1
}
}
}
func (this *IntervalSet) removeOne(v) {
if (this.intervals !== null) {
for (var k = 0 k < this.intervals.length k++) {
var i = this.intervals[k]
func (i *IntervalSet) removeOne(v) {
if (i.intervals != null) {
for (var k = 0 k < i.intervals.length k++) {
var i = i.intervals[k]
// intervals is ordered
if (v < i.start) {
return
}
// check for single value range
else if (v == i.start && v == i.stop - 1) {
this.intervals.splice(k, 1)
i.intervals.splice(k, 1)
return
}
// check for lower boundary
else if (v == i.start) {
this.intervals[k] = new Interval(i.start + 1, i.stop)
i.intervals[k] = new Interval(i.start + 1, i.stop)
return
}
// check for upper boundary
else if (v == i.stop - 1) {
this.intervals[k] = new Interval(i.start, i.stop - 1)
i.intervals[k] = new Interval(i.start, i.stop - 1)
return
}
// split existing range
else if (v < i.stop - 1) {
var x = new Interval(i.start, v)
i.start = v + 1
this.intervals.splice(k, 0, x)
i.intervals.splice(k, 0, x)
return
}
}
}
}
func (this *IntervalSet) toString(literalNames, symbolicNames, elemsAreChar) {
func (i *IntervalSet) toString(literalNames, symbolicNames, elemsAreChar) {
literalNames = literalNames || null
symbolicNames = symbolicNames || null
elemsAreChar = elemsAreChar || false
if (this.intervals == null) {
if (i.intervals == null) {
return "{}"
} else if(literalNames!==null || symbolicNames!==null) {
return this.toTokenString(literalNames, symbolicNames)
} else if(literalNames!=null || symbolicNames!=null) {
return i.toTokenString(literalNames, symbolicNames)
} else if(elemsAreChar) {
return this.toCharString()
return i.toCharString()
} else {
return this.toIndexString()
return i.toIndexString()
}
}
func (this *IntervalSet) toCharString() {
func (i *IntervalSet) toCharString() {
var names = []
for (var i = 0 i < this.intervals.length i++) {
var v = this.intervals[i]
for (var i = 0 i < i.intervals.length i++) {
var v = i.intervals[i]
if(v.stop==v.start+1) {
if ( v.start==Token.EOF ) {
names.push("<EOF>")
@ -242,10 +246,10 @@ func (this *IntervalSet) toCharString() {
}
func (this *IntervalSet) toIndexString() {
func (i *IntervalSet) toIndexString() {
var names = []
for (var i = 0 i < this.intervals.length i++) {
var v = this.intervals[i]
for (var i = 0 i < i.intervals.length i++) {
var v = i.intervals[i]
if(v.stop==v.start+1) {
if ( v.start==Token.EOF ) {
names.push("<EOF>")
@ -264,12 +268,12 @@ func (this *IntervalSet) toIndexString() {
}
func (this *IntervalSet) toTokenString(literalNames, symbolicNames) {
func (i *IntervalSet) toTokenString(literalNames, symbolicNames) {
var names = []
for (var i = 0 i < this.intervals.length i++) {
var v = this.intervals[i]
for (var i = 0 i < i.intervals.length i++) {
var v = i.intervals[i]
for (var j = v.start j < v.stop j++) {
names.push(this.elementName(literalNames, symbolicNames, j))
names.push(i.elementName(literalNames, symbolicNames, j))
}
}
if (names.length > 1) {
@ -279,7 +283,7 @@ func (this *IntervalSet) toTokenString(literalNames, symbolicNames) {
}
}
func (this *IntervalSet) elementName(literalNames, symbolicNames, a) {
func (i *IntervalSet) elementName(literalNames, symbolicNames, a) {
if (a == Token.EOF) {
return "<EOF>"
} else if (a == Token.EPSILON) {

270
runtime/Go/src/antlr4/Lexer.go
View File

@ -16,6 +16,21 @@ type TokenSource interface {
type Lexer struct {
Recognizer
_input
_factory
_tokenFactorySourcePair
_interp
_token int
_tokenStartCharIndex int
_tokenStartLine int
_tokenStartColumn int
_hitEOF int
_channel int
_type int
lexer._modeStack
lexer._mode int
lexer._text string
}
func NewLexer(input InputStream) {
@ -24,15 +39,15 @@ func NewLexer(input InputStream) {
lexer._input = input
lexer._factory = CommonTokenFactory.DEFAULT
lexer._tokenFactorySourcePair = [ this, input ]
lexer._tokenFactorySourcePair = [ l, input ]
lexer._interp = null // child classes must populate this
lexer._interp = null // child classes must populate l
// The goal of all lexer rules/methods is to create a token object.
// this is an instance variable as multiple rules may collaborate to
// create a single token. nextToken will return this object after
// l is an instance variable as multiple rules may collaborate to
// create a single token. nextToken will return l object after
// matching lexer rule(s). If you subclass to allow multiple token
// emissions, then set this to the last token to be matched or
// emissions, then set l to the last token to be matched or
// something nonnull so that the auto token emit mechanism will not
// emit another token.
lexer._token = null
@ -62,15 +77,16 @@ func NewLexer(input InputStream) {
lexer._mode = Lexer.DEFAULT_MODE
// You can set the text for the current token to override what is in
// the input char buffer. Use setText() or can set this instance var.
// the input char buffer. Use setText() or can set l instance var.
// /
lexer._text = null
return this
return l
}
func InitLexer(input){
func InitLexer(lexer Lexer){
}
@ -80,88 +96,90 @@ const (
LexerSKIP = -3
)
Lexer.DEFAULT_TOKEN_CHANNEL = Token.DEFAULT_CHANNEL
Lexer.HIDDEN = Token.HIDDEN_CHANNEL
Lexer.MIN_CHAR_VALUE = '\u0000'
Lexer.MAX_CHAR_VALUE = '\uFFFE'
const (
LexerDEFAULT_TOKEN_CHANNEL = Token.DEFAULT_CHANNEL
LexerHIDDEN = Token.HIDDEN_CHANNEL
LexerMIN_CHAR_VALUE = '\u0000'
LexerMAX_CHAR_VALUE = '\uFFFE'
)
func (this *Lexer) reset() {
func (l *Lexer) reset() {
// wack Lexer state variables
if (this._input !== null) {
this._input.seek(0) // rewind the input
if (l._input !== null) {
l._input.seek(0) // rewind the input
}
this._token = null
this._type = Token.INVALID_TYPE
this._channel = Token.DEFAULT_CHANNEL
this._tokenStartCharIndex = -1
this._tokenStartColumn = -1
this._tokenStartLine = -1
this._text = null
l._token = null
l._type = Token.INVALID_TYPE
l._channel = Token.DEFAULT_CHANNEL
l._tokenStartCharIndex = -1
l._tokenStartColumn = -1
l._tokenStartLine = -1
l._text = null
this._hitEOF = false
this._mode = Lexer.DEFAULT_MODE
this._modeStack = []
l._hitEOF = false
l._mode = Lexer.DEFAULT_MODE
l._modeStack = []
this._interp.reset()
l._interp.reset()
}
// Return a token from this source i.e., match a token on the char stream.
func (this *Lexer) nextToken() {
if (this._input == null) {
// Return a token from l source i.e., match a token on the char stream.
func (l *Lexer) nextToken() {
if (l._input == null) {
panic("nextToken requires a non-null input stream.")
}
// Mark start location in char stream so unbuffered streams are
// guaranteed at least have text of current token
var tokenStartMarker = this._input.mark()
var tokenStartMarker = l._input.mark()
try {
for (true) {
if (this._hitEOF) {
this.emitEOF()
return this._token
if (l._hitEOF) {
l.emitEOF()
return l._token
}
this._token = null
this._channel = Token.DEFAULT_CHANNEL
this._tokenStartCharIndex = this._input.index
this._tokenStartColumn = this._interp.column
this._tokenStartLine = this._interp.line
this._text = null
l._token = null
l._channel = Token.DEFAULT_CHANNEL
l._tokenStartCharIndex = l._input.index
l._tokenStartColumn = l._interp.column
l._tokenStartLine = l._interp.line
l._text = null
var continueOuter = false
for (true) {
this._type = Token.INVALID_TYPE
l._type = Token.INVALID_TYPE
var ttype = Lexer.SKIP
try {
ttype = this._interp.match(this._input, this._mode)
ttype = l._interp.match(l._input, l._mode)
} catch (e) {
this.notifyListeners(e) // report error
this.recover(e)
l.notifyListeners(e) // report error
l.recover(e)
}
if (this._input.LA(1) == Token.EOF) {
this._hitEOF = true
if (l._input.LA(1) == Token.EOF) {
l._hitEOF = true
}
if (this._type == Token.INVALID_TYPE) {
this._type = ttype
if (l._type == Token.INVALID_TYPE) {
l._type = ttype
}
if (this._type == Lexer.SKIP) {
if (l._type == Lexer.SKIP) {
continueOuter = true
break
}
if (this._type !== Lexer.MORE) {
if (l._type !== Lexer.MORE) {
break
}
}
if (continueOuter) {
continue
}
if (this._token == null) {
this.emit()
if (l._token == null) {
l.emit()
}
return this._token
return l._token
}
} finally {
// make sure we release marker after match or
// unbuffered char stream will keep buffering
this._input.release(tokenStartMarker)
l._input.release(tokenStartMarker)
}
}
@ -171,162 +189,162 @@ func (this *Lexer) nextToken() {
// if token==null at end of any token rule, it creates one for you
// and emits it.
// /
func (this *Lexer) skip() {
this._type = Lexer.SKIP
func (l *Lexer) skip() {
l._type = Lexer.SKIP
}
func (this *Lexer) more() {
this._type = Lexer.MORE
func (l *Lexer) more() {
l._type = Lexer.MORE
}
func (this *Lexer) mode(m) {
this._mode = m
func (l *Lexer) mode(m) {
l._mode = m
}
func (this *Lexer) pushMode(m) {
if (this._interp.debug) {
func (l *Lexer) pushMode(m) {
if (l._interp.debug) {
console.log("pushMode " + m)
}
this._modeStack.push(this._mode)
this.mode(m)
l._modeStack.push(l._mode)
l.mode(m)
}
func (this *Lexer) popMode() {
if (this._modeStack.length == 0) {
func (l *Lexer) popMode() {
if (l._modeStack.length == 0) {
throw "Empty Stack"
}
if (this._interp.debug) {
console.log("popMode back to " + this._modeStack.slice(0, -1))
if (l._interp.debug) {
console.log("popMode back to " + l._modeStack.slice(0, -1))
}
this.mode(this._modeStack.pop())
return this._mode
l.mode(l._modeStack.pop())
return l._mode
}
// Set the char stream and reset the lexer
Object.defineProperty(Lexer.prototype, "inputStream", {
get : function() {
return this._input
return l._input
},
set : function(input) {
this._input = null
this._tokenFactorySourcePair = [ this, this._input ]
this.reset()
this._input = input
this._tokenFactorySourcePair = [ this, this._input ]
l._input = null
l._tokenFactorySourcePair = [ l, l._input ]
l.reset()
l._input = input
l._tokenFactorySourcePair = [ l, l._input ]
}
})
Object.defineProperty(Lexer.prototype, "sourceName", {
get : type sourceName struct {
return this._input.sourceName
return l._input.sourceName
}
})
// By default does not support multiple emits per nextToken invocation
// for efficiency reasons. Subclass and override this method, nextToken,
// for efficiency reasons. Subclass and override l method, nextToken,
// and getToken (to push tokens into a list and pull from that list
// rather than a single variable as this implementation does).
// rather than a single variable as l implementation does).
// /
func (this *Lexer) emitToken(token) {
this._token = token
func (l *Lexer) emitToken(token) {
l._token = token
}
// The standard method called to automatically emit a token at the
// outermost lexical rule. The token object should point into the
// char buffer start..stop. If there is a text override in 'text',
// use that to set the token's text. Override this method to emit
// use that to set the token's text. Override l method to emit
// custom Token objects or provide a new factory.
// /
func (this *Lexer) emit() {
var t = this._factory.create(this._tokenFactorySourcePair, this._type,
this._text, this._channel, this._tokenStartCharIndex, this
.getCharIndex() - 1, this._tokenStartLine,
this._tokenStartColumn)
this.emitToken(t)
func (l *Lexer) emit() {
var t = l._factory.create(l._tokenFactorySourcePair, l._type,
l._text, l._channel, l._tokenStartCharIndex, l
.getCharIndex() - 1, l._tokenStartLine,
l._tokenStartColumn)
l.emitToken(t)
return t
}
func (this *Lexer) emitEOF() {
var cpos = this.column
var lpos = this.line
var eof = this._factory.create(this._tokenFactorySourcePair, Token.EOF,
null, Token.DEFAULT_CHANNEL, this._input.index,
this._input.index - 1, lpos, cpos)
this.emitToken(eof)
func (l *Lexer) emitEOF() {
var cpos = l.column
var lpos = l.line
var eof = l._factory.create(l._tokenFactorySourcePair, Token.EOF,
null, Token.DEFAULT_CHANNEL, l._input.index,
l._input.index - 1, lpos, cpos)
l.emitToken(eof)
return eof
}
Object.defineProperty(Lexer.prototype, "type", {
get : function() {
return this.type
return l.type
},
set : function(type) {
this._type = type
l._type = type
}
})
Object.defineProperty(Lexer.prototype, "line", {
get : function() {
return this._interp.line
return l._interp.line
},
set : function(line) {
this._interp.line = line
l._interp.line = line
}
})
Object.defineProperty(Lexer.prototype, "column", {
get : function() {
return this._interp.column
return l._interp.column
},
set : function(column) {
this._interp.column = column
l._interp.column = column
}
})
// What is the index of the current character of lookahead?///
func (this *Lexer) getCharIndex() {
return this._input.index
func (l *Lexer) getCharIndex() {
return l._input.index
}
// Return the text matched so far for the current token or any text override.
//Set the complete text of this token it wipes any previous changes to the text.
//Set the complete text of l token it wipes any previous changes to the text.
Object.defineProperty(Lexer.prototype, "text", {
get : function() {
if (this._text !== null) {
return this._text
if (l._text !== null) {
return l._text
} else {
return this._interp.getText(this._input)
return l._interp.getText(l._input)
}
},
set : function(text) {
this._text = text
l._text = text
}
})
// Return a list of all Token objects in input char stream.
// Forces load of all tokens. Does not include EOF token.
// /
func (this *Lexer) getAllTokens() {
func (l *Lexer) getAllTokens() {
var tokens = []
var t = this.nextToken()
var t = l.nextToken()
while (t.type !== Token.EOF) {
tokens.push(t)
t = this.nextToken()
t = l.nextToken()
}
return tokens
}
func (this *Lexer) notifyListeners(e) {
var start = this._tokenStartCharIndex
var stop = this._input.index
var text = this._input.getText(start, stop)
var msg = "token recognition error at: '" + this.getErrorDisplay(text) + "'"
var listener = this.getErrorListenerDispatch()
listener.syntaxError(this, null, this._tokenStartLine,
this._tokenStartColumn, msg, e)
func (l *Lexer) notifyListeners(e) {
var start = l._tokenStartCharIndex
var stop = l._input.index
var text = l._input.getText(start, stop)
var msg = "token recognition error at: '" + l.getErrorDisplay(text) + "'"
var listener = l.getErrorListenerDispatch()
listener.syntaxError(l, null, l._tokenStartLine,
l._tokenStartColumn, msg, e)
}
func (this *Lexer) getErrorDisplay(s) {
func (l *Lexer) getErrorDisplay(s) {
var d = make([]string,s.length)
for i := 0; i < s.length; i++ {
d[i] = s[i]
@ -334,7 +352,7 @@ func (this *Lexer) getErrorDisplay(s) {
return strings.Join(d, "")
}
func (this *Lexer) getErrorDisplayForChar(c) {
func (l *Lexer) getErrorDisplayForChar(c) {
if (c.charCodeAt(0) == Token.EOF) {
return "<EOF>"
} else if (c == '\n') {
@ -348,8 +366,8 @@ func (this *Lexer) getErrorDisplayForChar(c) {
}
}
func (this *Lexer) getCharErrorDisplay(c) {
return "'" + this.getErrorDisplayForChar(c) + "'"
func (l *Lexer) getCharErrorDisplay(c) {
return "'" + l.getErrorDisplayForChar(c) + "'"
}
// Lexers can normally match any char in it's vocabulary after matching
@ -357,14 +375,14 @@ func (this *Lexer) getCharErrorDisplay(c) {
// it all works out. You can instead use the rule invocation stack
// to do sophisticated error recovery if you are in a fragment rule.
// /
func (this *Lexer) recover(re) {
if (this._input.LA(1) !== Token.EOF) {
func (l *Lexer) recover(re) {
if (l._input.LA(1) !== Token.EOF) {
if (re instanceof LexerNoViableAltException) {
// skip a char and try again
this._interp.consume(this._input)
l._interp.consume(l._input)
} else {
// TODO: Do we lose character or line position information?
this._input.consume()
l._input.consume()
}
}
}

1
runtime/Go/src/antlr4/tree/Trees.go
View File

@ -7,7 +7,6 @@ var ErrorNode = require('./Tree').ErrorNode
var TerminalNode = require('./Tree').TerminalNode
var ParserRuleContext = require('./../ParserRuleContext').ParserRuleContext
/** A set of utility routines useful for all kinds of ANTLR trees. */
type Trees struct {
}