expr/scan/scanner.go

// Copyright (c) 2024-2026 Celestino Amoroso (celestino.amoroso@gmail.com).
// All rights reserved.

// expr project scanner.go
package scan

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"strconv"
	"strings"

	"git.portale-stac.it/go-pkg/expr/sym"
	"git.portale-stac.it/go-pkg/expr/util"
)

type Scanner struct {
	current       *Token
	prev          *Token
	stage         *Token
	stream        *bufio.Reader
	row           int
	column        int
	translations  map[sym.Symbol]sym.Symbol
	unreadByte    byte
	prevRowLength int
}

func NewScanner(s io.Reader, translations map[sym.Symbol]sym.Symbol) (inst *Scanner) {
	inst = &Scanner{
		stream:       bufio.NewReader(s),
		row:          1,
		column:       1,
		translations: translations,
	}
	inst.current = inst.fetchNextToken()
	return inst
}

func DefaultTranslations() map[sym.Symbol]sym.Symbol {
	return map[sym.Symbol]sym.Symbol{
		sym.SymDoubleAmpersand: sym.SymAnd,
		sym.SymKwAnd:           sym.SymAnd,
		sym.SymDoubleVertBar:   sym.SymOr,
		sym.SymKwOr:            sym.SymOr,
		sym.SymKwNot:           sym.SymNot,
		sym.SymLessGreater:     sym.SymNotEqual,
		// SymTilde:           sym.SymNot,
	}
}

// func (self *scanner) Current() *Token {
// 	return self.current
// }

func (scanner *Scanner) Current() *Token {
	return scanner.current
}

func (scanner *Scanner) setUnreadChar(ch byte) {
	scanner.column--
	if scanner.column == 0 {
		scanner.column = scanner.prevRowLength
		scanner.row--
	}
	scanner.unreadByte = ch
}

func (scanner *Scanner) readChar() (ch byte, err error) {
	if scanner.unreadByte != 0 {
		ch = scanner.unreadByte
		scanner.unreadByte = 0
	} else {
		ch, err = scanner.stream.ReadByte()
	}
	if err == nil {
		if ch == '\n' {
			scanner.prevRowLength = scanner.column
			scanner.row++
			scanner.column = 0
		} else {
			scanner.column++
		}
	}
	return
}

func (scanner *Scanner) unreadChar() (err error) {
	if err = scanner.stream.UnreadByte(); err == nil {
		if scanner.column--; scanner.column == 0 {
			if scanner.row--; scanner.row == 0 {
				err = errors.New("unread beyond the stream boundary")
			} else {
				scanner.column = 1
			}
		}
	}
	return
}

func (scanner *Scanner) UnreadToken() (err error) {
	if scanner.stage == nil {
		scanner.stage = scanner.current
		scanner.current = scanner.prev
	} else {
		err = fmt.Errorf("staging already present, currently one level only of staging is allowed")
	}
	return
}

func (scanner *Scanner) LastPos() (r, c int) {
	if scanner.prev != nil {
		r = scanner.prev.row
		c = scanner.prev.col
	}
	return
}

func (scanner *Scanner) Previous() *Token {
	return scanner.prev
}

func (scanner *Scanner) Next() (tk *Token) {
	scanner.prev = scanner.current
	tk = scanner.current
	if scanner.stage != nil {
		scanner.current = scanner.stage
		scanner.stage = nil
	} else {
		scanner.current = scanner.fetchNextToken()
	}
	return tk
}

func (scanner *Scanner) fetchNextToken() (tk *Token) {
	var ch byte
	if err := scanner.skipBlanks(); err != nil {
		return scanner.makeErrorToken(err)
	}

	escape := false
	for {
		ch, _ = scanner.readChar()
		switch ch {
		case '+':
			if next, _ := scanner.peek(); next == '+' {
				tk = scanner.moveOn(sym.SymDoublePlus, ch, next)
			} else if next == '=' {
				tk = scanner.moveOn(sym.SymPlusEqual, ch, next)
			} else if next == '>' {
				tk = scanner.moveOn(sym.SymPlusGreater, ch, next)
			} else {
				tk = scanner.MakeToken(sym.SymPlus, ch)
			}
		case '-':
			if next, _ := scanner.peek(); next == '-' {
				tk = scanner.moveOn(sym.SymDoubleMinus, ch, next)
			} else if next == '=' {
				tk = scanner.moveOn(sym.SymMinusEqual, ch, next)
			} else {
				tk = scanner.MakeToken(sym.SymMinus, ch)
			}
		case '*':
			if next, _ := scanner.peek(); next == '*' {
				tk = scanner.moveOn(sym.SymDoubleStar, ch, next)
				// } else if next == '/' {
				// 	tk = self.moveOn(SymClosedComment, ch, next)
			} else if next, _ = scanner.peek(); next == '=' {
				tk = scanner.moveOn(sym.SymStarEqual, ch, next)
			} else {
				tk = scanner.MakeToken(sym.SymStar, ch)
			}
		case '/':
			if next, _ := scanner.peek(); next == '*' {
				scanner.readChar()
				tk = scanner.fetchBlockComment()
			} else if next, _ = scanner.peek(); next == '=' {
				tk = scanner.moveOn(sym.SymSlashEqual, ch, next)
			} else if next == '/' {
				scanner.readChar()
				tk = scanner.fetchOnLineComment()
			} else {
				tk = scanner.MakeToken(sym.SymSlash, ch)
			}
		case '\\':
			if escape {
				tk = scanner.MakeToken(sym.SymBackSlash, ch)
				escape = false
			} else {
				escape = true
			}
		case '|':
			if next, _ := scanner.peek(); next == '|' {
				tk = scanner.moveOn(sym.SymDoubleVertBar, ch, next)
			} else if next, _ = scanner.peek(); next == '=' {
				tk = scanner.moveOn(sym.SymVertBarEqual, ch, next)
			} else {
				tk = scanner.MakeToken(sym.SymVertBar, ch)
			}
		case ',':
			tk = scanner.MakeToken(sym.SymComma, ch)
		case '^':
			if next, _ := scanner.peek(); next == '=' {
				tk = scanner.moveOn(sym.SymCaretEqual, ch, next)
			} else {
				tk = scanner.MakeToken(sym.SymCaret, ch)
			}
		case ':':
			if next, _ := scanner.peek(); next == ':' {
				tk = scanner.moveOn(sym.SymDoubleColon, ch, next)
			} else if next == '=' {
				tk = scanner.moveOn(sym.SymColonEqual, ch, next)
			} else {
				tk = scanner.MakeToken(sym.SymColon, ch)
			}
		case ';':
			tk = scanner.MakeToken(sym.SymSemiColon, ch)
		case '.':
			//if next, _ := self.peek(); next >= '0' && next <= '9' {
			//	tk = self.parseNumber(ch)
			//} else if next == '/' {
			if next, _ := scanner.peek(); next == '/' {
				tk = scanner.moveOn(sym.SymDotSlash, ch, next)
			} else if next == '.' {
				tk = scanner.moveOn(sym.SymDoubleDot, ch, next)
				// TODO: this section needs verification
				if next1, _ := scanner.peek(); next1 == '.' {
					tk = scanner.moveOn(sym.SymTripleDot, next, next1)
				}
			} else {
				tk = scanner.MakeToken(sym.SymDot, ch)
			}
		case '\'':
			if escape {
				tk = scanner.MakeToken(sym.SymQuote, ch)
				escape = false
			} else {
				tk = scanner.fetchString(ch, true)
			}
		case '"':
			if escape {
				tk = scanner.MakeToken(sym.SymDoubleQuote, ch)
				escape = false
			} else {
				tk = scanner.fetchString(ch, true)
			}
		case '`':
			tk = scanner.MakeToken(sym.SymBackTick, ch)
		case '!':
			if next, _ := scanner.peek(); next == '=' {
				tk = scanner.moveOn(sym.SymNotEqual, ch, next)
			} else {
				tk = scanner.MakeToken(sym.SymExclamation, ch)
			}
		case '?':
			if next, _ := scanner.peek(); next == '?' {
				tk = scanner.moveOn(sym.SymDoubleQuestion, ch, next)
			} else if next == '=' {
				tk = scanner.moveOn(sym.SymQuestionEqual, ch, next)
			} else if next == '!' {
				tk = scanner.moveOn(sym.SymQuestionExclam, ch, next)
			} else {
				tk = scanner.MakeToken(sym.SymQuestion, ch)
			}
		case '&':
			if next, _ := scanner.peek(); next == '&' {
				tk = scanner.moveOn(sym.SymDoubleAmpersand, ch, next)
			} else if next, _ = scanner.peek(); next == '=' {
				tk = scanner.moveOn(sym.SymAmpersandEqual, ch, next)
			} else {
				tk = scanner.MakeToken(sym.SymAmpersand, ch)
			}
		case '%':
			if next, _ := scanner.peek(); next == '=' {
				tk = scanner.moveOn(sym.SymPercEqual, ch, next)
			} else {
				tk = scanner.MakeToken(sym.SymPercent, ch)
			}
		case '#':
			tk = scanner.MakeToken(sym.SymHash, ch)
		case '@':
			if next, _ := scanner.peek(); (next >= 'a' && next <= 'z') || (next >= 'A' && next <= 'Z') {
				scanner.readChar()
				if tk = scanner.fetchIdentifier(next); tk.Sym == sym.SymIdentifier {
					//tk.Sym = SymIdRef
					tk.source = "@" + tk.source
				} else {
					tk = scanner.makeErrorToken(fmt.Errorf("invalid variable reference %q", tk.source))
				}
			} else if next == '@' {
				tk = scanner.moveOn(sym.SymDoubleAt, ch, next)
			} else {
				tk = scanner.MakeToken(sym.SymAt, ch)
			}
		case '_':
			tk = scanner.MakeToken(sym.SymUndescore, ch)
		case '=':
			if next, _ := scanner.peek(); next == '=' {
				tk = scanner.moveOn(sym.SymDoubleEqual, ch, next)
			} else {
				tk = scanner.MakeToken(sym.SymEqual, ch)
			}
		case '<':
			if next, _ := scanner.peek(); next == '=' {
				tk = scanner.moveOn(sym.SymLessOrEqual, ch, next)
			} else if next == '<' {
				scanner.readChar()
				next2, _ := scanner.readChar()
				scanner.unreadChar()
				if next2 == '=' {
					tk = scanner.moveOn(sym.SymDoubleLessEqual, ch, next, next2)
				} else {
					tk = scanner.accept(sym.SymDoubleLess, ch, next)
				}
			} else if next == '>' {
				tk = scanner.moveOn(sym.SymLessGreater, ch, next)
			} else if next == '+' {
				tk = scanner.moveOn(sym.SymLessPlus, ch, next)
			} else {
				tk = scanner.MakeToken(sym.SymLess, ch)
			}
		case '>':
			if next, _ := scanner.peek(); next == '=' {
				tk = scanner.moveOn(sym.SymGreaterOrEqual, ch, next)
			} else if next == '>' {
				scanner.readChar()
				next2, _ := scanner.readChar()
				scanner.unreadChar()
				if next2 == '=' {
					tk = scanner.moveOn(sym.SymDoubleGreaterEqual, ch, next, next2)
				} else {
					tk = scanner.accept(sym.SymDoubleGreater, ch, next)
				}
			} else if next == ']' {
				tk = scanner.moveOn(sym.SymGreaterClosedSquare, ch, next)
			} else {
				tk = scanner.MakeToken(sym.SymGreater, ch)
			}
		case '$':
			if next, _ := scanner.peek(); next == '(' {
				tk = scanner.moveOn(sym.SymDollarRound, ch, next)
				tk.source += ")"
			} else if next == '$' {
				tk = scanner.moveOn(sym.SymDoubleDollar, ch, next)
			} else if next == '{' {
				scanner.readChar()
				if tk = scanner.fetchString('}', false); tk != nil {
					tk.Sym = sym.SymIdentifier
				}
			} else if next == '_' || (next >= 'a' && next <= 'z') || (next >= 'A' && next <= 'Z') {
				scanner.readChar()
				tk = scanner.fetchIdentifier(next)
			} else {
				tk = scanner.MakeToken(sym.SymDollar, ch)
			}
		case '(':
			//			if next, _ := scanner.peek(); next == ')' {
			//				tk = scanner.moveOn(SymOpenClosedRound, ch, next)
			//			} else {
			tk = scanner.MakeToken(sym.SymOpenRound, ch)
			//			}
		case ')':
			tk = scanner.MakeToken(sym.SymClosedRound, ch)
		case '[':
			if next, _ := scanner.peek(); next == '<' {
				tk = scanner.moveOn(sym.SymOpenSquareLess, ch, next)
			} else {
				tk = scanner.MakeToken(sym.SymOpenSquare, ch)
			}
		case ']':
			tk = scanner.MakeToken(sym.SymClosedSquare, ch)
		case '{':
			tk = scanner.MakeToken(sym.SymOpenBrace, ch)
		case '}':
			tk = scanner.MakeToken(sym.SymClosedBrace, ch)
		case '~':
			tk = scanner.MakeToken(sym.SymTilde, ch)
		case 0:
			if escape {
				tk = scanner.makeErrorToken(errors.New("incomplete escape sequence"))
			}
			escape = false
		default:
			if /*ch == '_' ||*/ (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') {
				if tk = scanner.fetchIdentifier(ch); tk.Sym == sym.SymKwFunc {
					if next, _ := scanner.peek(); next == '(' {
						tk = scanner.moveOn(sym.SymFuncDef, ch, next)
					}
				}
			} else if ch >= '0' && ch <= '9' {
				tk = scanner.parseNumber(ch)
			}
		}
		if !escape {
			break
		}
	}
	if tk == nil {
		tk = NewErrorToken(scanner.row, scanner.column, fmt.Errorf("unknown symbol '%c'", ch))
	}
	return
}

func (scanner *Scanner) sync(err error) error {
	if err == nil {
		err = scanner.unreadChar()
	}
	return err
}

func isBinaryDigit(ch byte) bool {
	return ch == '0' || ch == '1'
}

func isOctalDigit(ch byte) bool {
	return ch >= '0' && ch <= '7'
}

func isDecimalDigit(ch byte) bool {
	return ch >= '0' && ch <= '9'
}

func isHexDigit(ch byte) bool {
	return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F')
}

func (scanner *Scanner) initBase(currentFirstCh byte) (firstCh byte, numBase int, digitFunc func(byte) bool, err error) {
	var ch byte
	var digitType string
	firstCh = currentFirstCh
	digitFunc = isDecimalDigit
	numBase = 10

	if ch, err = scanner.peek(); err == nil {
		switch ch {
		case 'b', 'B':
			numBase = 2
			digitType = "binary"
			scanner.readChar()
			digitFunc = isBinaryDigit
			firstCh, err = scanner.readChar()
		case 'o', 'O':
			numBase = 8
			digitType = "octal"
			scanner.readChar()
			digitFunc = isOctalDigit
			firstCh, err = scanner.readChar()
		case 'x', 'X':
			numBase = 16
			digitType = "hex"
			scanner.readChar()
			digitFunc = isHexDigit
			firstCh, err = scanner.readChar()
		}
		if err == nil && !digitFunc(firstCh) {
			if len(digitType) == 0 {
				digitType = "decimal"
			}
			err = fmt.Errorf("expected %s digit, got '%c'", digitType, firstCh)
		}
	} else if err == io.EOF {
		err = nil
	}
	return
}

func (scanner *Scanner) parseNumber(firstCh byte) (tk *Token) {
	var err error
	var ch byte
	var symbol sym.Symbol = sym.SymInteger
	var sb strings.Builder
	var isDigit func(byte) bool = isDecimalDigit
	var numBase = 10

	if firstCh == '0' {
		firstCh, numBase, isDigit, err = scanner.initBase(firstCh)
	}
	for ch = firstCh; err == nil && isDigit(ch); ch, err = scanner.readChar() {
		sb.WriteByte(ch)
	}

	if numBase == 10 {
		if err == nil && ch == '.' {
			if next, _ := scanner.peek(); next == '.' {
				// do nothing, this is an interval operator
				scanner.setUnreadChar('.')
			} else {
				symbol = sym.SymFloat
				sb.WriteByte(ch)
				ch, err = scanner.readChar()
				if ch >= '0' && ch <= '9' {
					for ; err == nil && (ch >= '0' && ch <= '9'); ch, err = scanner.readChar() {
						sb.WriteByte(ch)
					}
				}
			}
		}
		if err == nil {
			switch ch {
			case 'e', 'E':
				symbol = sym.SymFloat
				sb.WriteByte(ch)
				if ch, err = scanner.readChar(); err == nil {
					if ch == '+' || ch == '-' {
						sb.WriteByte(ch)
						ch, err = scanner.readChar()
					}
					if ch >= '0' && ch <= '9' {
						for ; err == nil && (ch >= '0' && ch <= '9'); ch, err = scanner.readChar() {
							sb.WriteByte(ch)
						}
					} else {
						err = fmt.Errorf("[%d:%d] expected integer exponent, got %c", scanner.row, scanner.column, ch)
					}
				}
			case '(':
				symbol = sym.SymFraction
				sb.WriteByte(ch)
				ch, err = scanner.readChar()
				for ; err == nil && (ch >= '0' && ch <= '9'); ch, err = scanner.readChar() {
					sb.WriteByte(ch)
				}
				if err == nil {
					if ch != ')' {
						err = fmt.Errorf("[%d:%d] expected ')', got '%c'", scanner.row, scanner.column, ch)
					} else {
						sb.WriteByte(ch)
						_, err = scanner.readChar()
					}
				}
			}
		}
	}

	if err != nil && err != io.EOF {
		tk = scanner.makeErrorToken(err)
	} else {
		var value any
		_ = scanner.sync(err) // TODO: Check this function
		txt := sb.String()
		switch symbol {
		case sym.SymFloat:
			value, err = strconv.ParseFloat(txt, 64)
		case sym.SymFraction:
			// value, err = kern.MakeGeneratingFraction(txt)
			value = txt
		default:
			value, err = strconv.ParseInt(txt, numBase, 64)
		}
		tk = scanner.MakeValueToken(symbol, txt, value)
		// if err == nil {
		// 	tk = scanner.MakeValueToken(sym, txt, value)
		// } else {
		// 	tk = scanner.makeErrorToken(err)
		// }
	}
	return
}

func (scanner *Scanner) fetchIdentifier(firstCh byte) (tk *Token) {
	var err error
	var sb strings.Builder
	for ch := firstCh; err == nil && (ch == '_' || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9')); ch, err = scanner.readChar() {
		sb.WriteByte(ch)
	}

	if err != nil && err != io.EOF {
		tk = scanner.makeErrorToken(err)
	} else if err = scanner.sync(err); err != nil && err != io.EOF {
		tk = scanner.makeErrorToken(err)
	} else {
		txt := sb.String()
		uptxt := strings.ToUpper(txt)
		if symbol, ok := sym.Keywords[uptxt]; ok {
			tk = scanner.makeKeywordToken(symbol, uptxt)
		} else if uptxt == `TRUE` {
			tk = scanner.MakeValueToken(sym.SymBool, txt, true)
		} else if uptxt == `FALSE` {
			tk = scanner.MakeValueToken(sym.SymBool, txt, false)
		} else if ch, _ := scanner.peek(); ch == '(' {
			scanner.readChar()
			tk = scanner.MakeValueToken(sym.SymFuncCall, txt+"(", txt)
		} else {
			tk = scanner.MakeValueToken(sym.SymIdentifier, txt, txt)
		}
	}

	// if err != nil && err != io.EOF {
	// 	tk = self.makeErrorToken(err)
	// } else if err = self.sync(err); err != nil && err != io.EOF {
	// 	tk = self.makeErrorToken(err)
	// } else {
	// 	txt := sb.String()
	// 	uptxt := strings.ToUpper(txt)
	// 	if sym, ok := keywords[uptxt]; ok {
	// 		tk = self.makeValueToken(sym, txt, "")
	// 	} else {
	// 		tk = self.makeValueToken(SymIdentifier, txt, txt)
	// 	}
	// }
	return
}

func (scanner *Scanner) fetchBlockComment() *Token {
	return scanner.fetchUntil(sym.SymComment, false, '*', '/')
}

func (scanner *Scanner) fetchOnLineComment() *Token {
	return scanner.fetchUntil(sym.SymComment, true, '\n')
}

func (scanner *Scanner) fetchUntil(sym sym.Symbol, allowEos bool, endings ...byte) (tk *Token) {
	var err error
	var ch byte
	var sb strings.Builder
	var value string
	ring := util.NewByteSlider(len(endings))
	endReached := false
	for ch, err = scanner.readChar(); err == nil && !endReached; {
		sb.WriteByte(ch)
		ring.PushEnd(ch)
		if ring.Equal(endings) {
			value = sb.String()[0 : sb.Len()-len(endings)]
			endReached = true
		} else {
			ch, err = scanner.readChar()
		}
	}
	if !endReached && allowEos {
		value = sb.String()
		endReached = true
	}

	if endReached {
		tk = scanner.MakeValueToken(sym, "", value)
	} else {
		tk = scanner.makeErrorToken(err)
	}
	return
}

func (scanner *Scanner) fetchString(termCh byte, addQuote bool) (tk *Token) {
	var err error
	var ch, prev byte
	var sb strings.Builder
	for ch, err = scanner.readChar(); err == nil; ch, err = scanner.readChar() {
		if prev == '\\' {
			switch ch {
			case '"':
				sb.WriteByte('"')
			case 'n':
				sb.WriteByte('\n')
			case 'r':
				sb.WriteByte('\r')
			case 't':
				sb.WriteByte('\t')
			case '\\':
				sb.WriteByte('\\')
			default:
				sb.WriteByte(ch)
			}
			prev = 0
		} else if ch == termCh {
			break
		} else {
			prev = ch
			if ch != '\\' {
				sb.WriteByte(ch)
			}
		}
	}
	if err != nil {
		if err == io.EOF {
			tk = scanner.makeErrorToken(errors.New(string(termCh)))
		} else {
			tk = scanner.makeErrorToken(err)
		}
	} else {
		txt := sb.String()
		if addQuote {
			tk = scanner.MakeValueToken(sym.SymString, `"`+txt+`"`, txt)
		} else {
			tk = scanner.MakeValueToken(sym.SymString, txt, txt)
		}
	}
	return
}

func (scanner *Scanner) peek() (next byte, err error) {
	var one []byte
	if one, err = scanner.stream.Peek(1); err == nil {
		next = one[0]
	}
	return
}

func (scanner *Scanner) skipBlanks() (err error) {
	var one []byte
	for one, err = scanner.stream.Peek(1); err == nil && one[0] <= 32; one, err = scanner.stream.Peek(1) {
		scanner.readChar()
	}
	return
}

func (scanner *Scanner) translate(sym sym.Symbol) sym.Symbol {
	if scanner.translations != nil {
		if translatedSym, ok := scanner.translations[sym]; ok {
			return translatedSym
		}
	}
	return sym
}

func (scanner *Scanner) moveOn(sym sym.Symbol, chars ...byte) (tk *Token) {
	tk = NewToken(scanner.row, scanner.column, scanner.translate(sym), string(chars))
	// for i := 1; i < len(chars); i++ {
	if len(chars) > 1 {
		scanner.readChar()
	}
	// }
	return
}

func (scanner *Scanner) accept(sym sym.Symbol, chars ...byte) (tk *Token) {
	tk = NewToken(scanner.row, scanner.column, scanner.translate(sym), string(chars))
	return
}

func (scanner *Scanner) MakeToken(sym sym.Symbol, chars ...byte) (tk *Token) {
	tk = NewToken(scanner.row, scanner.column, scanner.translate(sym), string(chars))
	return
}

func (scanner *Scanner) makeKeywordToken(sym sym.Symbol, upperCaseKeyword string) (tk *Token) {
	tk = NewToken(scanner.row, scanner.column, scanner.translate(sym), upperCaseKeyword)
	return
}

func (scanner *Scanner) MakeValueToken(sym sym.Symbol, source string, value any) (tk *Token) {
	tk = NewValueToken(scanner.row, scanner.column, scanner.translate(sym), source, value)
	return
}

func (scanner *Scanner) makeErrorToken(err error) *Token {
	return NewErrorToken(scanner.row, scanner.column, err)
}