// Copyright (c) 2024-2026 Celestino Amoroso (celestino.amoroso@gmail.com). // All rights reserved. // expr project scanner.go package scan import ( "bufio" "errors" "fmt" "io" "strconv" "strings" "git.portale-stac.it/go-pkg/expr/sym" "git.portale-stac.it/go-pkg/expr/util" ) type Scanner struct { current *Token prev *Token stage *Token stream *bufio.Reader row int column int translations map[sym.Symbol]sym.Symbol unreadByte byte prevRowLength int } func NewScanner(s io.Reader, translations map[sym.Symbol]sym.Symbol) (inst *Scanner) { inst = &Scanner{ stream: bufio.NewReader(s), row: 1, column: 1, translations: translations, } inst.current = inst.fetchNextToken() return inst } func DefaultTranslations() map[sym.Symbol]sym.Symbol { return map[sym.Symbol]sym.Symbol{ sym.SymDoubleAmpersand: sym.SymAnd, sym.SymKwAnd: sym.SymAnd, sym.SymDoubleVertBar: sym.SymOr, sym.SymKwOr: sym.SymOr, sym.SymKwNot: sym.SymNot, sym.SymLessGreater: sym.SymNotEqual, // SymTilde: sym.SymNot, } } // func (self *scanner) Current() *Token { // return self.current // } func (scanner *Scanner) Current() *Token { return scanner.current } func (scanner *Scanner) setUnreadChar(ch byte) { scanner.column-- if scanner.column == 0 { scanner.column = scanner.prevRowLength scanner.row-- } scanner.unreadByte = ch } func (scanner *Scanner) readChar() (ch byte, err error) { if scanner.unreadByte != 0 { ch = scanner.unreadByte scanner.unreadByte = 0 } else { ch, err = scanner.stream.ReadByte() } if err == nil { if ch == '\n' { scanner.prevRowLength = scanner.column scanner.row++ scanner.column = 0 } else { scanner.column++ } } return } func (scanner *Scanner) unreadChar() (err error) { if err = scanner.stream.UnreadByte(); err == nil { if scanner.column--; scanner.column == 0 { if scanner.row--; scanner.row == 0 { err = errors.New("unread beyond the stream boundary") } else { scanner.column = 1 } } } return } func (scanner *Scanner) UnreadToken() (err error) { if scanner.stage == nil { scanner.stage = scanner.current scanner.current = scanner.prev } else { err = fmt.Errorf("staging already present, currently one level only of staging is allowed") } return } func (scanner *Scanner) LastPos() (r, c int) { if scanner.prev != nil { r = scanner.prev.row c = scanner.prev.col } return } func (scanner *Scanner) Previous() *Token { return scanner.prev } func (scanner *Scanner) Next() (tk *Token) { scanner.prev = scanner.current tk = scanner.current if scanner.stage != nil { scanner.current = scanner.stage scanner.stage = nil } else { scanner.current = scanner.fetchNextToken() } return tk } func (scanner *Scanner) fetchNextToken() (tk *Token) { var ch byte if err := scanner.skipBlanks(); err != nil { return scanner.makeErrorToken(err) } escape := false for { ch, _ = scanner.readChar() switch ch { case '+': if next, _ := scanner.peek(); next == '+' { tk = scanner.moveOn(sym.SymDoublePlus, ch, next) } else if next == '=' { tk = scanner.moveOn(sym.SymPlusEqual, ch, next) } else if next == '>' { tk = scanner.moveOn(sym.SymPlusGreater, ch, next) } else { tk = scanner.MakeToken(sym.SymPlus, ch) } case '-': if next, _ := scanner.peek(); next == '-' { tk = scanner.moveOn(sym.SymDoubleMinus, ch, next) } else if next == '=' { tk = scanner.moveOn(sym.SymMinusEqual, ch, next) } else { tk = scanner.MakeToken(sym.SymMinus, ch) } case '*': if next, _ := scanner.peek(); next == '*' { tk = scanner.moveOn(sym.SymDoubleStar, ch, next) // } else if next == '/' { // tk = self.moveOn(SymClosedComment, ch, next) } else if next, _ = scanner.peek(); next == '=' { tk = scanner.moveOn(sym.SymStarEqual, ch, next) } else { tk = scanner.MakeToken(sym.SymStar, ch) } case '/': if next, _ := scanner.peek(); next == '*' { scanner.readChar() tk = scanner.fetchBlockComment() } else if next, _ = scanner.peek(); next == '=' { tk = scanner.moveOn(sym.SymSlashEqual, ch, next) } else if next == '/' { scanner.readChar() tk = scanner.fetchOnLineComment() } else { tk = scanner.MakeToken(sym.SymSlash, ch) } case '\\': if escape { tk = scanner.MakeToken(sym.SymBackSlash, ch) escape = false } else { escape = true } case '|': if next, _ := scanner.peek(); next == '|' { tk = scanner.moveOn(sym.SymDoubleVertBar, ch, next) } else if next, _ = scanner.peek(); next == '=' { tk = scanner.moveOn(sym.SymVertBarEqual, ch, next) } else { tk = scanner.MakeToken(sym.SymVertBar, ch) } case ',': tk = scanner.MakeToken(sym.SymComma, ch) case '^': if next, _ := scanner.peek(); next == '=' { tk = scanner.moveOn(sym.SymCaretEqual, ch, next) } else { tk = scanner.MakeToken(sym.SymCaret, ch) } case ':': if next, _ := scanner.peek(); next == ':' { tk = scanner.moveOn(sym.SymDoubleColon, ch, next) } else if next == '=' { tk = scanner.moveOn(sym.SymColonEqual, ch, next) } else { tk = scanner.MakeToken(sym.SymColon, ch) } case ';': tk = scanner.MakeToken(sym.SymSemiColon, ch) case '.': //if next, _ := self.peek(); next >= '0' && next <= '9' { // tk = self.parseNumber(ch) //} else if next == '/' { if next, _ := scanner.peek(); next == '/' { tk = scanner.moveOn(sym.SymDotSlash, ch, next) } else if next == '.' { tk = scanner.moveOn(sym.SymDoubleDot, ch, next) // TODO: this section needs verification if next1, _ := scanner.peek(); next1 == '.' { tk = scanner.moveOn(sym.SymTripleDot, next, next1) } } else { tk = scanner.MakeToken(sym.SymDot, ch) } case '\'': if escape { tk = scanner.MakeToken(sym.SymQuote, ch) escape = false } else { tk = scanner.fetchString(ch, true) } case '"': if escape { tk = scanner.MakeToken(sym.SymDoubleQuote, ch) escape = false } else { tk = scanner.fetchString(ch, true) } case '`': tk = scanner.MakeToken(sym.SymBackTick, ch) case '!': if next, _ := scanner.peek(); next == '=' { tk = scanner.moveOn(sym.SymNotEqual, ch, next) } else { tk = scanner.MakeToken(sym.SymExclamation, ch) } case '?': if next, _ := scanner.peek(); next == '?' { tk = scanner.moveOn(sym.SymDoubleQuestion, ch, next) } else if next == '=' { tk = scanner.moveOn(sym.SymQuestionEqual, ch, next) } else if next == '!' { tk = scanner.moveOn(sym.SymQuestionExclam, ch, next) } else { tk = scanner.MakeToken(sym.SymQuestion, ch) } case '&': if next, _ := scanner.peek(); next == '&' { tk = scanner.moveOn(sym.SymDoubleAmpersand, ch, next) } else if next, _ = scanner.peek(); next == '=' { tk = scanner.moveOn(sym.SymAmpersandEqual, ch, next) } else { tk = scanner.MakeToken(sym.SymAmpersand, ch) } case '%': if next, _ := scanner.peek(); next == '=' { tk = scanner.moveOn(sym.SymPercEqual, ch, next) } else { tk = scanner.MakeToken(sym.SymPercent, ch) } case '#': tk = scanner.MakeToken(sym.SymHash, ch) case '@': if next, _ := scanner.peek(); (next >= 'a' && next <= 'z') || (next >= 'A' && next <= 'Z') { scanner.readChar() if tk = scanner.fetchIdentifier(next); tk.Sym == sym.SymIdentifier { //tk.Sym = SymIdRef tk.source = "@" + tk.source } else { tk = scanner.makeErrorToken(fmt.Errorf("invalid variable reference %q", tk.source)) } } else if next == '@' { tk = scanner.moveOn(sym.SymDoubleAt, ch, next) } else { tk = scanner.MakeToken(sym.SymAt, ch) } case '_': tk = scanner.MakeToken(sym.SymUndescore, ch) case '=': if next, _ := scanner.peek(); next == '=' { tk = scanner.moveOn(sym.SymDoubleEqual, ch, next) } else { tk = scanner.MakeToken(sym.SymEqual, ch) } case '<': if next, _ := scanner.peek(); next == '=' { tk = scanner.moveOn(sym.SymLessOrEqual, ch, next) } else if next == '<' { scanner.readChar() next2, _ := scanner.readChar() scanner.unreadChar() if next2 == '=' { tk = scanner.moveOn(sym.SymDoubleLessEqual, ch, next, next2) } else { tk = scanner.accept(sym.SymDoubleLess, ch, next) } } else if next == '>' { tk = scanner.moveOn(sym.SymLessGreater, ch, next) } else if next == '+' { tk = scanner.moveOn(sym.SymLessPlus, ch, next) } else { tk = scanner.MakeToken(sym.SymLess, ch) } case '>': if next, _ := scanner.peek(); next == '=' { tk = scanner.moveOn(sym.SymGreaterOrEqual, ch, next) } else if next == '>' { scanner.readChar() next2, _ := scanner.readChar() scanner.unreadChar() if next2 == '=' { tk = scanner.moveOn(sym.SymDoubleGreaterEqual, ch, next, next2) } else { tk = scanner.accept(sym.SymDoubleGreater, ch, next) } } else if next == ']' { tk = scanner.moveOn(sym.SymGreaterClosedSquare, ch, next) } else { tk = scanner.MakeToken(sym.SymGreater, ch) } case '$': if next, _ := scanner.peek(); next == '(' { tk = scanner.moveOn(sym.SymDollarRound, ch, next) tk.source += ")" } else if next == '$' { tk = scanner.moveOn(sym.SymDoubleDollar, ch, next) } else if next == '{' { scanner.readChar() if tk = scanner.fetchString('}', false); tk != nil { tk.Sym = sym.SymIdentifier } } else if next == '_' || (next >= 'a' && next <= 'z') || (next >= 'A' && next <= 'Z') { scanner.readChar() tk = scanner.fetchIdentifier(next) } else { tk = scanner.MakeToken(sym.SymDollar, ch) } case '(': // if next, _ := scanner.peek(); next == ')' { // tk = scanner.moveOn(SymOpenClosedRound, ch, next) // } else { tk = scanner.MakeToken(sym.SymOpenRound, ch) // } case ')': tk = scanner.MakeToken(sym.SymClosedRound, ch) case '[': if next, _ := scanner.peek(); next == '<' { tk = scanner.moveOn(sym.SymOpenSquareLess, ch, next) } else { tk = scanner.MakeToken(sym.SymOpenSquare, ch) } case ']': tk = scanner.MakeToken(sym.SymClosedSquare, ch) case '{': tk = scanner.MakeToken(sym.SymOpenBrace, ch) case '}': tk = scanner.MakeToken(sym.SymClosedBrace, ch) case '~': tk = scanner.MakeToken(sym.SymTilde, ch) case 0: if escape { tk = scanner.makeErrorToken(errors.New("incomplete escape sequence")) } escape = false default: if /*ch == '_' ||*/ (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') { if tk = scanner.fetchIdentifier(ch); tk.Sym == sym.SymKwFunc { if next, _ := scanner.peek(); next == '(' { tk = scanner.moveOn(sym.SymFuncDef, ch, next) } } } else if ch >= '0' && ch <= '9' { tk = scanner.parseNumber(ch) } } if !escape { break } } if tk == nil { tk = NewErrorToken(scanner.row, scanner.column, fmt.Errorf("unknown symbol '%c'", ch)) } return } func (scanner *Scanner) sync(err error) error { if err == nil { err = scanner.unreadChar() } return err } func isBinaryDigit(ch byte) bool { return ch == '0' || ch == '1' } func isOctalDigit(ch byte) bool { return ch >= '0' && ch <= '7' } func isDecimalDigit(ch byte) bool { return ch >= '0' && ch <= '9' } func isHexDigit(ch byte) bool { return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F') } func (scanner *Scanner) initBase(currentFirstCh byte) (firstCh byte, numBase int, digitFunc func(byte) bool, err error) { var ch byte var digitType string firstCh = currentFirstCh digitFunc = isDecimalDigit numBase = 10 if ch, err = scanner.peek(); err == nil { switch ch { case 'b', 'B': numBase = 2 digitType = "binary" scanner.readChar() digitFunc = isBinaryDigit firstCh, err = scanner.readChar() case 'o', 'O': numBase = 8 digitType = "octal" scanner.readChar() digitFunc = isOctalDigit firstCh, err = scanner.readChar() case 'x', 'X': numBase = 16 digitType = "hex" scanner.readChar() digitFunc = isHexDigit firstCh, err = scanner.readChar() } if err == nil && !digitFunc(firstCh) { if len(digitType) == 0 { digitType = "decimal" } err = fmt.Errorf("expected %s digit, got '%c'", digitType, firstCh) } } else if err == io.EOF { err = nil } return } func (scanner *Scanner) parseNumber(firstCh byte) (tk *Token) { var err error var ch byte var symbol sym.Symbol = sym.SymInteger var sb strings.Builder var isDigit func(byte) bool = isDecimalDigit var numBase = 10 if firstCh == '0' { firstCh, numBase, isDigit, err = scanner.initBase(firstCh) } for ch = firstCh; err == nil && isDigit(ch); ch, err = scanner.readChar() { sb.WriteByte(ch) } if numBase == 10 { if err == nil && ch == '.' { if next, _ := scanner.peek(); next == '.' { // do nothing, this is an interval operator scanner.setUnreadChar('.') } else { symbol = sym.SymFloat sb.WriteByte(ch) ch, err = scanner.readChar() if ch >= '0' && ch <= '9' { for ; err == nil && (ch >= '0' && ch <= '9'); ch, err = scanner.readChar() { sb.WriteByte(ch) } } } } if err == nil { switch ch { case 'e', 'E': symbol = sym.SymFloat sb.WriteByte(ch) if ch, err = scanner.readChar(); err == nil { if ch == '+' || ch == '-' { sb.WriteByte(ch) ch, err = scanner.readChar() } if ch >= '0' && ch <= '9' { for ; err == nil && (ch >= '0' && ch <= '9'); ch, err = scanner.readChar() { sb.WriteByte(ch) } } else { err = fmt.Errorf("[%d:%d] expected integer exponent, got %c", scanner.row, scanner.column, ch) } } case '(': symbol = sym.SymFraction sb.WriteByte(ch) ch, err = scanner.readChar() for ; err == nil && (ch >= '0' && ch <= '9'); ch, err = scanner.readChar() { sb.WriteByte(ch) } if err == nil { if ch != ')' { err = fmt.Errorf("[%d:%d] expected ')', got '%c'", scanner.row, scanner.column, ch) } else { sb.WriteByte(ch) _, err = scanner.readChar() } } } } } if err != nil && err != io.EOF { tk = scanner.makeErrorToken(err) } else { var value any _ = scanner.sync(err) // TODO: Check this function txt := sb.String() switch symbol { case sym.SymFloat: value, err = strconv.ParseFloat(txt, 64) case sym.SymFraction: // value, err = kern.MakeGeneratingFraction(txt) value = txt default: value, err = strconv.ParseInt(txt, numBase, 64) } tk = scanner.MakeValueToken(symbol, txt, value) // if err == nil { // tk = scanner.MakeValueToken(sym, txt, value) // } else { // tk = scanner.makeErrorToken(err) // } } return } func (scanner *Scanner) fetchIdentifier(firstCh byte) (tk *Token) { var err error var sb strings.Builder for ch := firstCh; err == nil && (ch == '_' || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9')); ch, err = scanner.readChar() { sb.WriteByte(ch) } if err != nil && err != io.EOF { tk = scanner.makeErrorToken(err) } else if err = scanner.sync(err); err != nil && err != io.EOF { tk = scanner.makeErrorToken(err) } else { txt := sb.String() uptxt := strings.ToUpper(txt) if symbol, ok := sym.Keywords[uptxt]; ok { tk = scanner.makeKeywordToken(symbol, uptxt) } else if uptxt == `TRUE` { tk = scanner.MakeValueToken(sym.SymBool, txt, true) } else if uptxt == `FALSE` { tk = scanner.MakeValueToken(sym.SymBool, txt, false) } else if ch, _ := scanner.peek(); ch == '(' { scanner.readChar() tk = scanner.MakeValueToken(sym.SymFuncCall, txt+"(", txt) } else { tk = scanner.MakeValueToken(sym.SymIdentifier, txt, txt) } } // if err != nil && err != io.EOF { // tk = self.makeErrorToken(err) // } else if err = self.sync(err); err != nil && err != io.EOF { // tk = self.makeErrorToken(err) // } else { // txt := sb.String() // uptxt := strings.ToUpper(txt) // if sym, ok := keywords[uptxt]; ok { // tk = self.makeValueToken(sym, txt, "") // } else { // tk = self.makeValueToken(SymIdentifier, txt, txt) // } // } return } func (scanner *Scanner) fetchBlockComment() *Token { return scanner.fetchUntil(sym.SymComment, false, '*', '/') } func (scanner *Scanner) fetchOnLineComment() *Token { return scanner.fetchUntil(sym.SymComment, true, '\n') } func (scanner *Scanner) fetchUntil(sym sym.Symbol, allowEos bool, endings ...byte) (tk *Token) { var err error var ch byte var sb strings.Builder var value string ring := util.NewByteSlider(len(endings)) endReached := false for ch, err = scanner.readChar(); err == nil && !endReached; { sb.WriteByte(ch) ring.PushEnd(ch) if ring.Equal(endings) { value = sb.String()[0 : sb.Len()-len(endings)] endReached = true } else { ch, err = scanner.readChar() } } if !endReached && allowEos { value = sb.String() endReached = true } if endReached { tk = scanner.MakeValueToken(sym, "", value) } else { tk = scanner.makeErrorToken(err) } return } func (scanner *Scanner) fetchString(termCh byte, addQuote bool) (tk *Token) { var err error var ch, prev byte var sb strings.Builder for ch, err = scanner.readChar(); err == nil; ch, err = scanner.readChar() { if prev == '\\' { switch ch { case '"': sb.WriteByte('"') case 'n': sb.WriteByte('\n') case 'r': sb.WriteByte('\r') case 't': sb.WriteByte('\t') case '\\': sb.WriteByte('\\') default: sb.WriteByte(ch) } prev = 0 } else if ch == termCh { break } else { prev = ch if ch != '\\' { sb.WriteByte(ch) } } } if err != nil { if err == io.EOF { tk = scanner.makeErrorToken(errors.New(string(termCh))) } else { tk = scanner.makeErrorToken(err) } } else { txt := sb.String() if addQuote { tk = scanner.MakeValueToken(sym.SymString, `"`+txt+`"`, txt) } else { tk = scanner.MakeValueToken(sym.SymString, txt, txt) } } return } func (scanner *Scanner) peek() (next byte, err error) { var one []byte if one, err = scanner.stream.Peek(1); err == nil { next = one[0] } return } func (scanner *Scanner) skipBlanks() (err error) { var one []byte for one, err = scanner.stream.Peek(1); err == nil && one[0] <= 32; one, err = scanner.stream.Peek(1) { scanner.readChar() } return } func (scanner *Scanner) translate(sym sym.Symbol) sym.Symbol { if scanner.translations != nil { if translatedSym, ok := scanner.translations[sym]; ok { return translatedSym } } return sym } func (scanner *Scanner) moveOn(sym sym.Symbol, chars ...byte) (tk *Token) { tk = NewToken(scanner.row, scanner.column, scanner.translate(sym), string(chars)) // for i := 1; i < len(chars); i++ { if len(chars) > 1 { scanner.readChar() } // } return } func (scanner *Scanner) accept(sym sym.Symbol, chars ...byte) (tk *Token) { tk = NewToken(scanner.row, scanner.column, scanner.translate(sym), string(chars)) return } func (scanner *Scanner) MakeToken(sym sym.Symbol, chars ...byte) (tk *Token) { tk = NewToken(scanner.row, scanner.column, scanner.translate(sym), string(chars)) return } func (scanner *Scanner) makeKeywordToken(sym sym.Symbol, upperCaseKeyword string) (tk *Token) { tk = NewToken(scanner.row, scanner.column, scanner.translate(sym), upperCaseKeyword) return } func (scanner *Scanner) MakeValueToken(sym sym.Symbol, source string, value any) (tk *Token) { tk = NewValueToken(scanner.row, scanner.column, scanner.translate(sym), source, value) return } func (scanner *Scanner) makeErrorToken(err error) *Token { return NewErrorToken(scanner.row, scanner.column, err) }