go-pkg/expr
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

scanner.go: New function UnreadToken().

Browse Source 
Currently it only supports one staging level.
This commit is contained in:
Celestino Amoroso 2024-07-23 15:32:25 +02:00
parent 5f8ca47ef0
commit 463e3634ba
1 changed files with 25 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
scanner.go
View File

@ -16,6 +16,7 @@ import (
type scanner struct { type scanner struct {
current *Token current *Token
prev *Token prev *Token
stage *Token
stream *bufio.Reader stream *bufio.Reader
row int row int
column int column int
@ -74,6 +75,16 @@ func (scanner *scanner) unreadChar() (err error) {
return return
} }
func (scanner *scanner) UnreadToken() (err error) {
if scanner.stage == nil {
scanner.stage = scanner.current
scanner.current = scanner.prev
} else {
err = fmt.Errorf("staging already present, currently one level only of staging is allowed")
}
return
}
func (scanner *scanner) lastPos() (r, c int) { func (scanner *scanner) lastPos() (r, c int) {
if scanner.prev != nil { if scanner.prev != nil {
r = scanner.prev.row r = scanner.prev.row
@ -89,7 +100,12 @@ func (scanner *scanner) Previous() *Token {
func (scanner *scanner) Next() (tk *Token) { func (scanner *scanner) Next() (tk *Token) {
scanner.prev = scanner.current scanner.prev = scanner.current
tk = scanner.current tk = scanner.current
scanner.current = scanner.fetchNextToken() if scanner.stage != nil {
scanner.current = scanner.stage
scanner.stage = nil
} else {
scanner.current = scanner.fetchNextToken()
}
return tk return tk
} }
@ -124,6 +140,8 @@ func (scanner *scanner) fetchNextToken() (tk *Token) {
tk = scanner.moveOn(SymDoubleStar, ch, next) tk = scanner.moveOn(SymDoubleStar, ch, next)
// } else if next == '/' { // } else if next == '/' {
// tk = self.moveOn(SymClosedComment, ch, next) // tk = self.moveOn(SymClosedComment, ch, next)
} else if next, _ = scanner.peek(); next == '=' {
tk = scanner.moveOn(SymStarEqual, ch, next)
} else { } else {
tk = scanner.makeToken(SymStar, ch) tk = scanner.makeToken(SymStar, ch)
} }
@ -269,11 +287,11 @@ func (scanner *scanner) fetchNextToken() (tk *Token) {
tk = scanner.makeToken(SymDollar, ch) tk = scanner.makeToken(SymDollar, ch)
} }
case '(': case '(':
// if next, _ := scanner.peek(); next == ')' { // if next, _ := scanner.peek(); next == ')' {
// tk = scanner.moveOn(SymOpenClosedRound, ch, next) // tk = scanner.moveOn(SymOpenClosedRound, ch, next)
// } else { // } else {
tk = scanner.makeToken(SymOpenRound, ch) tk = scanner.makeToken(SymOpenRound, ch)
// } // }
case ')': case ')':
tk = scanner.makeToken(SymClosedRound, ch) tk = scanner.makeToken(SymClosedRound, ch)
case '[': case '[':
@ -440,7 +458,7 @@ func (scanner *scanner) parseNumber(firstCh byte) (tk *Token) {
tk = scanner.makeErrorToken(err) tk = scanner.makeErrorToken(err)
} else { } else {
var value any var value any
err = scanner.sync(err) // TODO: Check this function err = scanner.sync(err) // TODO: Check this function
txt := sb.String() txt := sb.String()
if sym == SymFloat { if sym == SymFloat {
value, err = strconv.ParseFloat(txt, 64) value, err = strconv.ParseFloat(txt, 64)