go-pkg/expr
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

scanner.go: replaced self receiver

Browse Source
This commit is contained in:
Celestino Amoroso 2024-07-08 07:30:58 +02:00
parent a711333a2e
commit 867806155e
1 changed files with 185 additions and 185 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

370
scanner.go
View File

@ -49,257 +49,257 @@ func DefaultTranslations() map[Symbol]Symbol {
// return self.current // return self.current
// } // }
func (self *scanner) readChar() (ch byte, err error) { func (scanner *scanner) readChar() (ch byte, err error) {
if ch, err = self.stream.ReadByte(); err == nil { if ch, err = scanner.stream.ReadByte(); err == nil {
if ch == '\n' { if ch == '\n' {
self.row++ scanner.row++
self.column = 0 scanner.column = 0
} else { } else {
self.column++ scanner.column++
} }
} }
return return
} }
func (self *scanner) unreadChar() (err error) { func (scanner *scanner) unreadChar() (err error) {
if err = self.stream.UnreadByte(); err == nil { if err = scanner.stream.UnreadByte(); err == nil {
if self.column--; self.column == 0 { if scanner.column--; scanner.column == 0 {
if self.row--; self.row == 0 { if scanner.row--; scanner.row == 0 {
err = errors.New("unread beyond the stream boundary") err = errors.New("unread beyond the stream boundary")
} else { } else {
self.column = 1 scanner.column = 1
} }
} }
} }
return return
} }
func (self *scanner) lastPos() (r, c int) { func (scanner *scanner) lastPos() (r, c int) {
if self.prev != nil { if scanner.prev != nil {
r = self.prev.row r = scanner.prev.row
c = self.prev.col c = scanner.prev.col
} }
return return
} }
func (self *scanner) Previous() *Token { func (scanner *scanner) Previous() *Token {
return self.prev return scanner.prev
} }
func (self *scanner) Next() (tk *Token) { func (scanner *scanner) Next() (tk *Token) {
self.prev = self.current scanner.prev = scanner.current
tk = self.current tk = scanner.current
self.current = self.fetchNextToken() scanner.current = scanner.fetchNextToken()
return tk return tk
} }
func (self *scanner) fetchNextToken() (tk *Token) { func (scanner *scanner) fetchNextToken() (tk *Token) {
var ch byte var ch byte
if err := self.skipBlanks(); err != nil { if err := scanner.skipBlanks(); err != nil {
return self.makeErrorToken(err) return scanner.makeErrorToken(err)
} }
escape := false escape := false
for { for {
ch, _ = self.readChar() ch, _ = scanner.readChar()
switch ch { switch ch {
case '+': case '+':
if next, _ := self.peek(); next == '+' { if next, _ := scanner.peek(); next == '+' {
tk = self.moveOn(SymDoublePlus, ch, next) tk = scanner.moveOn(SymDoublePlus, ch, next)
} else if next == '=' { } else if next == '=' {
tk = self.moveOn(SymPlusEqual, ch, next) tk = scanner.moveOn(SymPlusEqual, ch, next)
} else { } else {
tk = self.makeToken(SymPlus, ch) tk = scanner.makeToken(SymPlus, ch)
} }
case '-': case '-':
if next, _ := self.peek(); next == '-' { if next, _ := scanner.peek(); next == '-' {
tk = self.moveOn(SymDoubleMinus, ch, next) tk = scanner.moveOn(SymDoubleMinus, ch, next)
} else if next == '=' { } else if next == '=' {
tk = self.moveOn(SymMinusEqual, ch, next) tk = scanner.moveOn(SymMinusEqual, ch, next)
} else { } else {
tk = self.makeToken(SymMinus, ch) tk = scanner.makeToken(SymMinus, ch)
} }
case '*': case '*':
if next, _ := self.peek(); next == '*' { if next, _ := scanner.peek(); next == '*' {
tk = self.moveOn(SymDoubleStar, ch, next) tk = scanner.moveOn(SymDoubleStar, ch, next)
// } else if next == '/' { // } else if next == '/' {
// tk = self.moveOn(SymClosedComment, ch, next) // tk = self.moveOn(SymClosedComment, ch, next)
} else { } else {
tk = self.makeToken(SymStar, ch) tk = scanner.makeToken(SymStar, ch)
} }
case '/': case '/':
if next, _ := self.peek(); next == '*' { if next, _ := scanner.peek(); next == '*' {
self.readChar() scanner.readChar()
tk = self.fetchBlockComment() tk = scanner.fetchBlockComment()
} else if next == '/' { } else if next == '/' {
self.readChar() scanner.readChar()
tk = self.fetchOnLineComment() tk = scanner.fetchOnLineComment()
} else { } else {
tk = self.makeToken(SymSlash, ch) tk = scanner.makeToken(SymSlash, ch)
} }
case '\\': case '\\':
if escape { if escape {
tk = self.makeToken(SymBackSlash, ch) tk = scanner.makeToken(SymBackSlash, ch)
escape = false escape = false
} else { } else {
escape = true escape = true
} }
case '|': case '|':
if next, _ := self.peek(); next == '|' { if next, _ := scanner.peek(); next == '|' {
tk = self.moveOn(SymDoubleVertBar, ch, next) tk = scanner.moveOn(SymDoubleVertBar, ch, next)
} else { } else {
tk = self.makeToken(SymVertBar, ch) tk = scanner.makeToken(SymVertBar, ch)
} }
case ',': case ',':
tk = self.makeToken(SymComma, ch) tk = scanner.makeToken(SymComma, ch)
case '^': case '^':
tk = self.makeToken(SymCaret, ch) tk = scanner.makeToken(SymCaret, ch)
case ':': case ':':
if next, _ := self.peek(); next == ':' { if next, _ := scanner.peek(); next == ':' {
tk = self.moveOn(SymDoubleColon, ch, next) tk = scanner.moveOn(SymDoubleColon, ch, next)
} else { } else {
tk = self.makeToken(SymColon, ch) tk = scanner.makeToken(SymColon, ch)
} }
case ';': case ';':
tk = self.makeToken(SymSemiColon, ch) tk = scanner.makeToken(SymSemiColon, ch)
case '.': case '.':
//if next, _ := self.peek(); next >= '0' && next <= '9' { //if next, _ := self.peek(); next >= '0' && next <= '9' {
// tk = self.parseNumber(ch) // tk = self.parseNumber(ch)
//} else if next == '/' { //} else if next == '/' {
if next, _ := self.peek(); next == '/' { if next, _ := scanner.peek(); next == '/' {
tk = self.moveOn(SymDotSlash, ch, next) tk = scanner.moveOn(SymDotSlash, ch, next)
} else if next == '.' { } else if next == '.' {
if next1, _ := self.peek(); next1 == '.' { if next1, _ := scanner.peek(); next1 == '.' {
tk = self.moveOn(SymTripleDot, ch, next, next1) tk = scanner.moveOn(SymTripleDot, ch, next, next1)
} else { } else {
tk = self.moveOn(SymDoubleDot, ch, next) tk = scanner.moveOn(SymDoubleDot, ch, next)
} }
} else { } else {
tk = self.makeToken(SymDot, ch) tk = scanner.makeToken(SymDot, ch)
} }
case '\'': case '\'':
if escape { if escape {
tk = self.makeToken(SymQuote, ch) tk = scanner.makeToken(SymQuote, ch)
escape = false escape = false
} else { } else {
tk = self.fetchString(ch) tk = scanner.fetchString(ch)
} }
case '"': case '"':
if escape { if escape {
tk = self.makeToken(SymDoubleQuote, ch) tk = scanner.makeToken(SymDoubleQuote, ch)
escape = false escape = false
} else { } else {
tk = self.fetchString(ch) tk = scanner.fetchString(ch)
} }
case '`': case '`':
tk = self.makeToken(SymBackTick, ch) tk = scanner.makeToken(SymBackTick, ch)
case '!': case '!':
if next, _ := self.peek(); next == '=' { if next, _ := scanner.peek(); next == '=' {
tk = self.moveOn(SymNotEqual, ch, next) tk = scanner.moveOn(SymNotEqual, ch, next)
} else { } else {
tk = self.makeToken(SymExclamation, ch) tk = scanner.makeToken(SymExclamation, ch)
} }
case '?': case '?':
if next, _ := self.peek(); next == '?' { if next, _ := scanner.peek(); next == '?' {
tk = self.moveOn(SymDoubleQuestion, ch, next) tk = scanner.moveOn(SymDoubleQuestion, ch, next)
} else if next == '=' { } else if next == '=' {
tk = self.moveOn(SymQuestionEqual, ch, next) tk = scanner.moveOn(SymQuestionEqual, ch, next)
} else if next == '!' { } else if next == '!' {
tk = self.moveOn(SymQuestionExclam, ch, next) tk = scanner.moveOn(SymQuestionExclam, ch, next)
} else { } else {
tk = self.makeToken(SymQuestion, ch) tk = scanner.makeToken(SymQuestion, ch)
} }
case '&': case '&':
if next, _ := self.peek(); next == '&' { if next, _ := scanner.peek(); next == '&' {
tk = self.moveOn(SymDoubleAmpersand, ch, next) tk = scanner.moveOn(SymDoubleAmpersand, ch, next)
} else { } else {
tk = self.makeToken(SymAmpersand, ch) tk = scanner.makeToken(SymAmpersand, ch)
} }
case '%': case '%':
tk = self.makeToken(SymPercent, ch) tk = scanner.makeToken(SymPercent, ch)
case '#': case '#':
tk = self.makeToken(SymHash, ch) tk = scanner.makeToken(SymHash, ch)
case '@': case '@':
if next, _ := self.peek(); (next >= 'a' && next <= 'z') || (next >= 'A' && next <= 'Z') { if next, _ := scanner.peek(); (next >= 'a' && next <= 'z') || (next >= 'A' && next <= 'Z') {
self.readChar() scanner.readChar()
if tk = self.fetchIdentifier(next); tk.Sym == SymIdentifier { if tk = scanner.fetchIdentifier(next); tk.Sym == SymIdentifier {
//tk.Sym = SymIdRef //tk.Sym = SymIdRef
tk.source = "@" + tk.source tk.source = "@" + tk.source
} else { } else {
tk = self.makeErrorToken(fmt.Errorf("invalid variable reference %q", tk.source)) tk = scanner.makeErrorToken(fmt.Errorf("invalid variable reference %q", tk.source))
} }
} else if next == '@' { } else if next == '@' {
tk = self.moveOn(SymDoubleAt, ch, next) tk = scanner.moveOn(SymDoubleAt, ch, next)
} else { } else {
tk = self.makeToken(SymAt, ch) tk = scanner.makeToken(SymAt, ch)
} }
case '_': case '_':
tk = self.makeToken(SymUndescore, ch) tk = scanner.makeToken(SymUndescore, ch)
case '=': case '=':
if next, _ := self.peek(); next == '=' { if next, _ := scanner.peek(); next == '=' {
tk = self.moveOn(SymDoubleEqual, ch, next) tk = scanner.moveOn(SymDoubleEqual, ch, next)
} else { } else {
tk = self.makeToken(SymEqual, ch) tk = scanner.makeToken(SymEqual, ch)
} }
case '<': case '<':
if next, _ := self.peek(); next == '=' { if next, _ := scanner.peek(); next == '=' {
tk = self.moveOn(SymLessOrEqual, ch, next) tk = scanner.moveOn(SymLessOrEqual, ch, next)
} else if next == '<' { } else if next == '<' {
tk = self.moveOn(SymAppend, ch, next) tk = scanner.moveOn(SymAppend, ch, next)
} else if next == '>' { } else if next == '>' {
tk = self.moveOn(SymLessGreater, ch, next) tk = scanner.moveOn(SymLessGreater, ch, next)
} else { } else {
tk = self.makeToken(SymLess, ch) tk = scanner.makeToken(SymLess, ch)
} }
case '>': case '>':
if next, _ := self.peek(); next == '=' { if next, _ := scanner.peek(); next == '=' {
tk = self.moveOn(SymGreaterOrEqual, ch, next) tk = scanner.moveOn(SymGreaterOrEqual, ch, next)
} else if next == '>' { } else if next == '>' {
tk = self.moveOn(SymInsert, ch, next) tk = scanner.moveOn(SymInsert, ch, next)
} else { } else {
tk = self.makeToken(SymGreater, ch) tk = scanner.makeToken(SymGreater, ch)
} }
case '$': case '$':
if next, _ := self.peek(); next == '(' { if next, _ := scanner.peek(); next == '(' {
tk = self.moveOn(SymDollarRound, ch, next) tk = scanner.moveOn(SymDollarRound, ch, next)
tk.source += ")" tk.source += ")"
} else if next == '$' { } else if next == '$' {
tk = self.moveOn(SymDoubleDollar, ch, next) tk = scanner.moveOn(SymDoubleDollar, ch, next)
} else { } else {
tk = self.makeToken(SymDollar, ch) tk = scanner.makeToken(SymDollar, ch)
} }
case '(': case '(':
if next, _ := self.peek(); next == ')' { if next, _ := scanner.peek(); next == ')' {
tk = self.moveOn(SymOpenClosedRound, ch, next) tk = scanner.moveOn(SymOpenClosedRound, ch, next)
} else { } else {
tk = self.makeToken(SymOpenRound, ch) tk = scanner.makeToken(SymOpenRound, ch)
} }
case ')': case ')':
tk = self.makeToken(SymClosedRound, ch) tk = scanner.makeToken(SymClosedRound, ch)
case '[': case '[':
tk = self.makeToken(SymOpenSquare, ch) tk = scanner.makeToken(SymOpenSquare, ch)
case ']': case ']':
tk = self.makeToken(SymClosedSquare, ch) tk = scanner.makeToken(SymClosedSquare, ch)
case '{': case '{':
tk = self.makeToken(SymOpenBrace, ch) tk = scanner.makeToken(SymOpenBrace, ch)
case '}': case '}':
tk = self.makeToken(SymClosedBrace, ch) tk = scanner.makeToken(SymClosedBrace, ch)
case '~': case '~':
tk = self.makeToken(SymTilde, ch) tk = scanner.makeToken(SymTilde, ch)
case 0: case 0:
if escape { if escape {
tk = self.makeErrorToken(errors.New("incomplete escape sequence")) tk = scanner.makeErrorToken(errors.New("incomplete escape sequence"))
} }
escape = false escape = false
default: default:
if /*ch == '_' ||*/ (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') { if /*ch == '_' ||*/ (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') {
if tk = self.fetchIdentifier(ch); tk.Sym == SymKwFunc { if tk = scanner.fetchIdentifier(ch); tk.Sym == SymKwFunc {
if next, _ := self.peek(); next == '(' { if next, _ := scanner.peek(); next == '(' {
tk = self.moveOn(SymFuncDef, ch, next) tk = scanner.moveOn(SymFuncDef, ch, next)
} }
} }
} else if ch >= '0' && ch <= '9' { } else if ch >= '0' && ch <= '9' {
tk = self.parseNumber(ch) tk = scanner.parseNumber(ch)
} }
} }
if !escape { if !escape {
@ -307,14 +307,14 @@ func (self *scanner) fetchNextToken() (tk *Token) {
} }
} }
if tk == nil { if tk == nil {
tk = NewErrorToken(self.row, self.column, fmt.Errorf("unknown symbol '%c'", ch)) tk = NewErrorToken(scanner.row, scanner.column, fmt.Errorf("unknown symbol '%c'", ch))
} }
return return
} }
func (self *scanner) sync(err error) error { func (scanner *scanner) sync(err error) error {
if err == nil { if err == nil {
err = self.unreadChar() err = scanner.unreadChar()
} }
return err return err
} }
@ -335,32 +335,32 @@ func isHexDigit(ch byte) bool {
return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F') return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F')
} }
func (self *scanner) initBase(sb *strings.Builder, currentFirstCh byte) (firstCh byte, numBase int, digitFunc func(byte) bool, err error) { func (scanner *scanner) initBase(currentFirstCh byte) (firstCh byte, numBase int, digitFunc func(byte) bool, err error) {
var ch byte var ch byte
var digitType string var digitType string
firstCh = currentFirstCh firstCh = currentFirstCh
digitFunc = isDecimalDigit digitFunc = isDecimalDigit
numBase = 10 numBase = 10
if ch, err = self.peek(); err == nil { if ch, err = scanner.peek(); err == nil {
if ch == 'b' || ch == 'B' { if ch == 'b' || ch == 'B' {
numBase = 2 numBase = 2
digitType = "binary" digitType = "binary"
self.readChar() scanner.readChar()
digitFunc = isBinaryDigit digitFunc = isBinaryDigit
firstCh, err = self.readChar() firstCh, err = scanner.readChar()
} else if ch == 'o' || ch == 'O' { } else if ch == 'o' || ch == 'O' {
numBase = 8 numBase = 8
digitType = "octal" digitType = "octal"
self.readChar() scanner.readChar()
digitFunc = isOctalDigit digitFunc = isOctalDigit
firstCh, err = self.readChar() firstCh, err = scanner.readChar()
} else if ch == 'x' || ch == 'X' { } else if ch == 'x' || ch == 'X' {
numBase = 16 numBase = 16
digitType = "hex" digitType = "hex"
self.readChar() scanner.readChar()
digitFunc = isHexDigit digitFunc = isHexDigit
firstCh, err = self.readChar() firstCh, err = scanner.readChar()
} }
if err == nil && !digitFunc(firstCh) { if err == nil && !digitFunc(firstCh) {
if len(digitType) == 0 { if len(digitType) == 0 {
@ -374,7 +374,7 @@ func (self *scanner) initBase(sb *strings.Builder, currentFirstCh byte) (firstCh
return return
} }
func (self *scanner) parseNumber(firstCh byte) (tk *Token) { func (scanner *scanner) parseNumber(firstCh byte) (tk *Token) {
var err error var err error
var ch byte var ch byte
var sym Symbol = SymInteger var sym Symbol = SymInteger
@ -383,9 +383,9 @@ func (self *scanner) parseNumber(firstCh byte) (tk *Token) {
var numBase = 10 var numBase = 10
if firstCh == '0' { if firstCh == '0' {
firstCh, numBase, isDigit, err = self.initBase(&sb, firstCh) firstCh, numBase, isDigit, err = scanner.initBase(firstCh)
} }
for ch = firstCh; err == nil && isDigit(ch); ch, err = self.readChar() { for ch = firstCh; err == nil && isDigit(ch); ch, err = scanner.readChar() {
sb.WriteByte(ch) sb.WriteByte(ch)
} }
@ -393,9 +393,9 @@ func (self *scanner) parseNumber(firstCh byte) (tk *Token) {
if err == nil && ch == '.' { if err == nil && ch == '.' {
sym = SymFloat sym = SymFloat
sb.WriteByte(ch) sb.WriteByte(ch)
ch, err = self.readChar() ch, err = scanner.readChar()
if ch >= '0' && ch <= '9' { if ch >= '0' && ch <= '9' {
for ; err == nil && (ch >= '0' && ch <= '9'); ch, err = self.readChar() { for ; err == nil && (ch >= '0' && ch <= '9'); ch, err = scanner.readChar() {
sb.WriteByte(ch) sb.WriteByte(ch)
} }
} }
@ -404,32 +404,32 @@ func (self *scanner) parseNumber(firstCh byte) (tk *Token) {
if ch == 'e' || ch == 'E' { if ch == 'e' || ch == 'E' {
sym = SymFloat sym = SymFloat
sb.WriteByte(ch) sb.WriteByte(ch)
if ch, err = self.readChar(); err == nil { if ch, err = scanner.readChar(); err == nil {
if ch == '+' || ch == '-' { if ch == '+' || ch == '-' {
sb.WriteByte(ch) sb.WriteByte(ch)
ch, err = self.readChar() ch, err = scanner.readChar()
} }
if ch >= '0' && ch <= '9' { if ch >= '0' && ch <= '9' {
for ; err == nil && (ch >= '0' && ch <= '9'); ch, err = self.readChar() { for ; err == nil && (ch >= '0' && ch <= '9'); ch, err = scanner.readChar() {
sb.WriteByte(ch) sb.WriteByte(ch)
} }
} else { } else {
err = fmt.Errorf("[%d:%d] expected integer exponent, got %c", self.row, self.column, ch) err = fmt.Errorf("[%d:%d] expected integer exponent, got %c", scanner.row, scanner.column, ch)
} }
} }
} else if ch == '(' { } else if ch == '(' {
sym = SymFraction sym = SymFraction
sb.WriteByte(ch) sb.WriteByte(ch)
ch, err = self.readChar() ch, err = scanner.readChar()
for ; err == nil && (ch >= '0' && ch <= '9'); ch, err = self.readChar() { for ; err == nil && (ch >= '0' && ch <= '9'); ch, err = scanner.readChar() {
sb.WriteByte(ch) sb.WriteByte(ch)
} }
if err == nil { if err == nil {
if ch != ')' { if ch != ')' {
err = fmt.Errorf("[%d:%d] expected ')', got '%c'", self.row, self.column, ch) err = fmt.Errorf("[%d:%d] expected ')', got '%c'", scanner.row, scanner.column, ch)
} else { } else {
sb.WriteByte(ch) sb.WriteByte(ch)
_, err = self.readChar() _, err = scanner.readChar()
} }
} }
} }
@ -437,10 +437,10 @@ func (self *scanner) parseNumber(firstCh byte) (tk *Token) {
} }
if err != nil && err != io.EOF { if err != nil && err != io.EOF {
tk = self.makeErrorToken(err) tk = scanner.makeErrorToken(err)
} else { } else {
var value any var value any
err = self.sync(err) err = scanner.sync(err) // TODO: Check this function
txt := sb.String() txt := sb.String()
if sym == SymFloat { if sym == SymFloat {
value, err = strconv.ParseFloat(txt, 64) value, err = strconv.ParseFloat(txt, 64)
@ -450,39 +450,39 @@ func (self *scanner) parseNumber(firstCh byte) (tk *Token) {
value, err = strconv.ParseInt(txt, numBase, 64) value, err = strconv.ParseInt(txt, numBase, 64)
} }
if err == nil { if err == nil {
tk = self.makeValueToken(sym, txt, value) tk = scanner.makeValueToken(sym, txt, value)
} else { } else {
tk = self.makeErrorToken(err) tk = scanner.makeErrorToken(err)
} }
} }
return return
} }
func (self *scanner) fetchIdentifier(firstCh byte) (tk *Token) { func (scanner *scanner) fetchIdentifier(firstCh byte) (tk *Token) {
var err error var err error
var sb strings.Builder var sb strings.Builder
for ch := firstCh; err == nil && (ch == '_' || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9')); ch, err = self.readChar() { for ch := firstCh; err == nil && (ch == '_' || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9')); ch, err = scanner.readChar() {
sb.WriteByte(ch) sb.WriteByte(ch)
} }
if err != nil && err != io.EOF { if err != nil && err != io.EOF {
tk = self.makeErrorToken(err) tk = scanner.makeErrorToken(err)
} else if err = self.sync(err); err != nil && err != io.EOF { } else if err = scanner.sync(err); err != nil && err != io.EOF {
tk = self.makeErrorToken(err) tk = scanner.makeErrorToken(err)
} else { } else {
txt := sb.String() txt := sb.String()
uptxt := strings.ToUpper(txt) uptxt := strings.ToUpper(txt)
if sym, ok := keywords[uptxt]; ok { if sym, ok := keywords[uptxt]; ok {
tk = self.makeKeywordToken(sym, uptxt) tk = scanner.makeKeywordToken(sym, uptxt)
} else if uptxt == `TRUE` { } else if uptxt == `TRUE` {
tk = self.makeValueToken(SymBool, txt, true) tk = scanner.makeValueToken(SymBool, txt, true)
} else if uptxt == `FALSE` { } else if uptxt == `FALSE` {
tk = self.makeValueToken(SymBool, txt, false) tk = scanner.makeValueToken(SymBool, txt, false)
} else if ch, _ := self.peek(); ch == '(' { } else if ch, _ := scanner.peek(); ch == '(' {
self.readChar() scanner.readChar()
tk = self.makeValueToken(SymFuncCall, txt+"(", txt) tk = scanner.makeValueToken(SymFuncCall, txt+"(", txt)
} else { } else {
tk = self.makeValueToken(SymIdentifier, txt, txt) tk = scanner.makeValueToken(SymIdentifier, txt, txt)
} }
} }
@ -502,29 +502,29 @@ func (self *scanner) fetchIdentifier(firstCh byte) (tk *Token) {
return return
} }
func (self *scanner) fetchBlockComment() *Token { func (scanner *scanner) fetchBlockComment() *Token {
return self.fetchUntil(SymComment, false, '*', '/') return scanner.fetchUntil(SymComment, false, '*', '/')
} }
func (self *scanner) fetchOnLineComment() *Token { func (scanner *scanner) fetchOnLineComment() *Token {
return self.fetchUntil(SymComment, true, '\n') return scanner.fetchUntil(SymComment, true, '\n')
} }
func (self *scanner) fetchUntil(sym Symbol, allowEos bool, endings ...byte) (tk *Token) { func (scanner *scanner) fetchUntil(sym Symbol, allowEos bool, endings ...byte) (tk *Token) {
var err error var err error
var ch byte var ch byte
var sb strings.Builder var sb strings.Builder
var value string var value string
ring := NewByteSlider(len(endings)) ring := NewByteSlider(len(endings))
endReached := false endReached := false
for ch, err = self.readChar(); err == nil && !endReached; { for ch, err = scanner.readChar(); err == nil && !endReached; {
sb.WriteByte(ch) sb.WriteByte(ch)
ring.PushEnd(ch) ring.PushEnd(ch)
if ring.Equal(endings) { if ring.Equal(endings) {
value = sb.String()[0 : sb.Len()-len(endings)] value = sb.String()[0 : sb.Len()-len(endings)]
endReached = true endReached = true
} else { } else {
ch, err = self.readChar() ch, err = scanner.readChar()
} }
} }
if !endReached && allowEos { if !endReached && allowEos {
@ -533,18 +533,18 @@ func (self *scanner) fetchUntil(sym Symbol, allowEos bool, endings ...byte) (tk
} }
if endReached { if endReached {
tk = self.makeValueToken(sym, "", value) tk = scanner.makeValueToken(sym, "", value)
} else { } else {
tk = self.makeErrorToken(err) tk = scanner.makeErrorToken(err)
} }
return return
} }
func (self *scanner) fetchString(termCh byte) (tk *Token) { func (scanner *scanner) fetchString(termCh byte) (tk *Token) {
var err error var err error
var ch, prev byte var ch, prev byte
var sb strings.Builder var sb strings.Builder
for ch, err = self.readChar(); err == nil; ch, err = self.readChar() { for ch, err = scanner.readChar(); err == nil; ch, err = scanner.readChar() {
if prev == '\\' { if prev == '\\' {
switch ch { switch ch {
case '"': case '"':
@ -572,65 +572,65 @@ func (self *scanner) fetchString(termCh byte) (tk *Token) {
} }
if err != nil { if err != nil {
if err == io.EOF { if err == io.EOF {
tk = self.makeErrorToken(errors.New("missing string termination \"")) tk = scanner.makeErrorToken(errors.New("missing string termination \""))
} else { } else {
tk = self.makeErrorToken(err) tk = scanner.makeErrorToken(err)
} }
} else { } else {
txt := sb.String() txt := sb.String()
tk = self.makeValueToken(SymString, `"`+txt+`"`, txt) tk = scanner.makeValueToken(SymString, `"`+txt+`"`, txt)
} }
return return
} }
func (self *scanner) peek() (next byte, err error) { func (scanner *scanner) peek() (next byte, err error) {
var one []byte var one []byte
if one, err = self.stream.Peek(1); err == nil { if one, err = scanner.stream.Peek(1); err == nil {
next = one[0] next = one[0]
} }
return return
} }
func (self *scanner) skipBlanks() (err error) { func (scanner *scanner) skipBlanks() (err error) {
var one []byte var one []byte
for one, err = self.stream.Peek(1); err == nil && one[0] <= 32; one, err = self.stream.Peek(1) { for one, err = scanner.stream.Peek(1); err == nil && one[0] <= 32; one, err = scanner.stream.Peek(1) {
self.readChar() scanner.readChar()
} }
return return
} }
func (self *scanner) translate(sym Symbol) Symbol { func (scanner *scanner) translate(sym Symbol) Symbol {
if self.translations != nil { if scanner.translations != nil {
if translatedSym, ok := self.translations[sym]; ok { if translatedSym, ok := scanner.translations[sym]; ok {
return translatedSym return translatedSym
} }
} }
return sym return sym
} }
func (self *scanner) moveOn(sym Symbol, chars ...byte) (tk *Token) { func (scanner *scanner) moveOn(sym Symbol, chars ...byte) (tk *Token) {
tk = NewToken(self.row, self.column, self.translate(sym), string(chars)) tk = NewToken(scanner.row, scanner.column, scanner.translate(sym), string(chars))
for i := 1; i < len(chars); i++ { for i := 1; i < len(chars); i++ {
self.readChar() scanner.readChar()
} }
return return
} }
func (self *scanner) makeToken(sym Symbol, chars ...byte) (tk *Token) { func (scanner *scanner) makeToken(sym Symbol, chars ...byte) (tk *Token) {
tk = NewToken(self.row, self.column, self.translate(sym), string(chars)) tk = NewToken(scanner.row, scanner.column, scanner.translate(sym), string(chars))
return return
} }
func (self *scanner) makeKeywordToken(sym Symbol, upperCaseKeyword string) (tk *Token) { func (scanner *scanner) makeKeywordToken(sym Symbol, upperCaseKeyword string) (tk *Token) {
tk = NewToken(self.row, self.column, self.translate(sym), upperCaseKeyword) tk = NewToken(scanner.row, scanner.column, scanner.translate(sym), upperCaseKeyword)
return return
} }
func (self *scanner) makeValueToken(sym Symbol, source string, value any) (tk *Token) { func (scanner *scanner) makeValueToken(sym Symbol, source string, value any) (tk *Token) {
tk = NewValueToken(self.row, self.column, self.translate(sym), source, value) tk = NewValueToken(scanner.row, scanner.column, scanner.translate(sym), source, value)
return return
} }
func (self *scanner) makeErrorToken(err error) *Token { func (scanner *scanner) makeErrorToken(err error) *Token {
return NewErrorToken(self.row, self.column, err) return NewErrorToken(scanner.row, scanner.column, err)
} }