moved scanner sources to package 'scan'

scan/ast.go

@@ -0,0 +1,147 @@
+// Copyright (c) 2024 Celestino Amoroso (celestino.amoroso@gmail.com).
+// All rights reserved.
+
+// ast.go
+package scan
+
+import (
+	"errors"
+	"strings"
+
+	"git.portale-stac.it/go-pkg/expr/kern"
+)
+
+//-------- ast
+
+type Ast struct {
+	forest []*Term
+	root   *Term
+}
+
+func NewAst() *Ast {
+	return &Ast{}
+}
+
+func (ast *Ast) TypeName() string {
+	return "Expression"
+}
+
+func (ast *Ast) Root() *Term {
+	return ast.root
+}
+
+func (ast *Ast) ToForest() {
+	if ast.root != nil {
+		if ast.forest == nil {
+			ast.forest = make([]*Term, 0)
+		}
+		ast.forest = append(ast.forest, ast.root)
+		ast.root = nil
+	}
+}
+
+func (ast *Ast) String() string {
+	var sb strings.Builder
+	if ast.root == nil {
+		sb.WriteString("(nil)")
+	} else {
+		ast.root.ToString(&sb)
+	}
+	return sb.String()
+}
+
+func (ast *Ast) AddTokens(tokens ...*Token) (err error) {
+	for _, tk := range tokens {
+		if _, err = ast.AddToken(tk); err != nil {
+			break
+		}
+	}
+	return
+}
+
+// func (expr *ast) addToken(tk *Token) (err error) {
+// 	_, err = expr.addToken2(tk)
+// 	return
+// }
+
+func (ast *Ast) AddToken(tk *Token) (t *Term, err error) {
+	if t = NewTerm(tk); t != nil {
+		err = ast.AddTerm(t)
+	} else {
+		err = tk.Errorf("unexpected token %q", tk.String())
+	}
+	return
+}
+
+func (ast *Ast) AddTerm(node *Term) (err error) {
+	if ast.root == nil {
+		ast.root = node
+	} else {
+		ast.root, err = ast.insert(ast.root, node)
+	}
+	return
+}
+
+func (ast *Ast) insert(tree, node *Term) (root *Term, err error) {
+	if tree.getPriority() < node.getPriority() {
+		root = tree
+		if tree.isComplete() {
+			var subRoot *Term
+			last := tree.removeLastChild()
+			if subRoot, err = ast.insert(last, node); err == nil {
+				subRoot.SetParent(tree)
+			}
+		} else {
+			node.SetParent(tree)
+		}
+	} else if !node.isLeaf() {
+		root = node
+		tree.SetParent(node)
+	} else {
+		err = node.Errorf("two adjacent operators: %q and %q", tree, node)
+	}
+	return
+}
+
+func (ast *Ast) Finish() {
+	if ast.root == nil && ast.forest != nil && len(ast.forest) >= 1 {
+		ast.root = ast.forest[len(ast.forest)-1]
+		ast.forest = ast.forest[0 : len(ast.forest)-1]
+	}
+}
+
+func (ast *Ast) Eval(ctx kern.ExprContext) (result any, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			if errVal, ok := r.(error); ok {
+				err = errVal
+			} else {
+				err = errors.New("unexpected error while evaluating the expression")
+			}
+		}
+	}()
+
+	ast.Finish()
+
+	if ast.root != nil {
+		// initDefaultVars(ctx)
+		if ast.forest != nil {
+			for _, tree := range ast.forest {
+				if result, err = tree.Compute(ctx); err == nil {
+					ctx.UnsafeSetVar(kern.ControlLastResult, result)
+				} else {
+					//err = fmt.Errorf("error in expression nr %d: %v", i+1, err)
+					break
+				}
+			}
+		}
+		if err == nil {
+			if result, err = ast.root.Compute(ctx); err == nil {
+				ctx.UnsafeSetVar(kern.ControlLastResult, result)
+			}
+		}
+		// } else {
+		// 	err = errors.New("empty expression")
+	}
+	return
+}

scan/scanner.go

@@ -0,0 +1,712 @@
+// Copyright (c) 2024 Celestino Amoroso (celestino.amoroso@gmail.com).
+// All rights reserved.
+
+// expr project scanner.go
+package scan
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+
+	"git.portale-stac.it/go-pkg/expr/kern"
+	"git.portale-stac.it/go-pkg/expr/util"
+)
+
+type Scanner struct {
+	current      *Token
+	prev         *Token
+	stage        *Token
+	stream       *bufio.Reader
+	row          int
+	column       int
+	translations map[Symbol]Symbol
+}
+
+func NewScanner(s io.Reader, translations map[Symbol]Symbol) (inst *Scanner) {
+	inst = &Scanner{
+		stream:       bufio.NewReader(s),
+		row:          1,
+		column:       1,
+		translations: translations,
+	}
+	inst.current = inst.fetchNextToken()
+	return inst
+}
+
+func DefaultTranslations() map[Symbol]Symbol {
+	return map[Symbol]Symbol{
+		SymDoubleAmpersand: SymAnd,
+		SymKwAnd:           SymAnd,
+		SymDoubleVertBar:   SymOr,
+		SymKwOr:            SymOr,
+		// SymTilde:           SymNot,
+		SymKwNot:       SymNot,
+		SymLessGreater: SymNotEqual,
+	}
+}
+
+// func (self *scanner) Current() *Token {
+// 	return self.current
+// }
+
+func (scanner *Scanner) Current() *Token {
+	return scanner.current
+}
+
+func (scanner *Scanner) readChar() (ch byte, err error) {
+	if ch, err = scanner.stream.ReadByte(); err == nil {
+		if ch == '\n' {
+			scanner.row++
+			scanner.column = 0
+		} else {
+			scanner.column++
+		}
+	}
+	return
+}
+
+func (scanner *Scanner) unreadChar() (err error) {
+	if err = scanner.stream.UnreadByte(); err == nil {
+		if scanner.column--; scanner.column == 0 {
+			if scanner.row--; scanner.row == 0 {
+				err = errors.New("unread beyond the stream boundary")
+			} else {
+				scanner.column = 1
+			}
+		}
+	}
+	return
+}
+
+func (scanner *Scanner) UnreadToken() (err error) {
+	if scanner.stage == nil {
+		scanner.stage = scanner.current
+		scanner.current = scanner.prev
+	} else {
+		err = fmt.Errorf("staging already present, currently one level only of staging is allowed")
+	}
+	return
+}
+
+func (scanner *Scanner) LastPos() (r, c int) {
+	if scanner.prev != nil {
+		r = scanner.prev.row
+		c = scanner.prev.col
+	}
+	return
+}
+
+func (scanner *Scanner) Previous() *Token {
+	return scanner.prev
+}
+
+func (scanner *Scanner) Next() (tk *Token) {
+	scanner.prev = scanner.current
+	tk = scanner.current
+	if scanner.stage != nil {
+		scanner.current = scanner.stage
+		scanner.stage = nil
+	} else {
+		scanner.current = scanner.fetchNextToken()
+	}
+	return tk
+}
+
+func (scanner *Scanner) fetchNextToken() (tk *Token) {
+	var ch byte
+	if err := scanner.skipBlanks(); err != nil {
+		return scanner.makeErrorToken(err)
+	}
+
+	escape := false
+	for {
+		ch, _ = scanner.readChar()
+		switch ch {
+		case '+':
+			if next, _ := scanner.peek(); next == '+' {
+				tk = scanner.moveOn(SymDoublePlus, ch, next)
+			} else if next == '=' {
+				tk = scanner.moveOn(SymPlusEqual, ch, next)
+			} else if next == '>' {
+				tk = scanner.moveOn(SymPlusGreater, ch, next)
+			} else {
+				tk = scanner.MakeToken(SymPlus, ch)
+			}
+		case '-':
+			if next, _ := scanner.peek(); next == '-' {
+				tk = scanner.moveOn(SymDoubleMinus, ch, next)
+			} else if next == '=' {
+				tk = scanner.moveOn(SymMinusEqual, ch, next)
+			} else {
+				tk = scanner.MakeToken(SymMinus, ch)
+			}
+		case '*':
+			if next, _ := scanner.peek(); next == '*' {
+				tk = scanner.moveOn(SymDoubleStar, ch, next)
+				// } else if next == '/' {
+				// 	tk = self.moveOn(SymClosedComment, ch, next)
+			} else if next, _ = scanner.peek(); next == '=' {
+				tk = scanner.moveOn(SymStarEqual, ch, next)
+			} else {
+				tk = scanner.MakeToken(SymStar, ch)
+			}
+		case '/':
+			if next, _ := scanner.peek(); next == '*' {
+				scanner.readChar()
+				tk = scanner.fetchBlockComment()
+			} else if next, _ = scanner.peek(); next == '=' {
+				tk = scanner.moveOn(SymSlashEqual, ch, next)
+			} else if next == '/' {
+				scanner.readChar()
+				tk = scanner.fetchOnLineComment()
+			} else {
+				tk = scanner.MakeToken(SymSlash, ch)
+			}
+		case '\\':
+			if escape {
+				tk = scanner.MakeToken(SymBackSlash, ch)
+				escape = false
+			} else {
+				escape = true
+			}
+		case '|':
+			if next, _ := scanner.peek(); next == '|' {
+				tk = scanner.moveOn(SymDoubleVertBar, ch, next)
+			} else if next, _ = scanner.peek(); next == '=' {
+				tk = scanner.moveOn(SymVertBarEqual, ch, next)
+			} else {
+				tk = scanner.MakeToken(SymVertBar, ch)
+			}
+		case ',':
+			tk = scanner.MakeToken(SymComma, ch)
+		case '^':
+			if next, _ := scanner.peek(); next == '=' {
+				tk = scanner.moveOn(SymCaretEqual, ch, next)
+			} else {
+				tk = scanner.MakeToken(SymCaret, ch)
+			}
+		case ':':
+			if next, _ := scanner.peek(); next == ':' {
+				tk = scanner.moveOn(SymDoubleColon, ch, next)
+			} else {
+				tk = scanner.MakeToken(SymColon, ch)
+			}
+		case ';':
+			tk = scanner.MakeToken(SymSemiColon, ch)
+		case '.':
+			//if next, _ := self.peek(); next >= '0' && next <= '9' {
+			//	tk = self.parseNumber(ch)
+			//} else if next == '/' {
+			if next, _ := scanner.peek(); next == '/' {
+				tk = scanner.moveOn(SymDotSlash, ch, next)
+			} else if next == '.' {
+				if next1, _ := scanner.peek(); next1 == '.' {
+					tk = scanner.moveOn(SymTripleDot, ch, next, next1)
+				} else {
+					tk = scanner.moveOn(SymDoubleDot, ch, next)
+				}
+			} else {
+				tk = scanner.MakeToken(SymDot, ch)
+			}
+		case '\'':
+			if escape {
+				tk = scanner.MakeToken(SymQuote, ch)
+				escape = false
+			} else {
+				tk = scanner.fetchString(ch, true)
+			}
+		case '"':
+			if escape {
+				tk = scanner.MakeToken(SymDoubleQuote, ch)
+				escape = false
+			} else {
+				tk = scanner.fetchString(ch, true)
+			}
+		case '`':
+			tk = scanner.MakeToken(SymBackTick, ch)
+		case '!':
+			if next, _ := scanner.peek(); next == '=' {
+				tk = scanner.moveOn(SymNotEqual, ch, next)
+			} else {
+				tk = scanner.MakeToken(SymExclamation, ch)
+			}
+		case '?':
+			if next, _ := scanner.peek(); next == '?' {
+				tk = scanner.moveOn(SymDoubleQuestion, ch, next)
+			} else if next == '=' {
+				tk = scanner.moveOn(SymQuestionEqual, ch, next)
+			} else if next == '!' {
+				tk = scanner.moveOn(SymQuestionExclam, ch, next)
+			} else {
+				tk = scanner.MakeToken(SymQuestion, ch)
+			}
+		case '&':
+			if next, _ := scanner.peek(); next == '&' {
+				tk = scanner.moveOn(SymDoubleAmpersand, ch, next)
+			} else if next, _ = scanner.peek(); next == '=' {
+				tk = scanner.moveOn(SymAmpersandEqual, ch, next)
+			} else {
+				tk = scanner.MakeToken(SymAmpersand, ch)
+			}
+		case '%':
+			if next, _ := scanner.peek(); next == '=' {
+				tk = scanner.moveOn(SymPercEqual, ch, next)
+			} else {
+				tk = scanner.MakeToken(SymPercent, ch)
+			}
+		case '#':
+			tk = scanner.MakeToken(SymHash, ch)
+		case '@':
+			if next, _ := scanner.peek(); (next >= 'a' && next <= 'z') || (next >= 'A' && next <= 'Z') {
+				scanner.readChar()
+				if tk = scanner.fetchIdentifier(next); tk.Sym == SymIdentifier {
+					//tk.Sym = SymIdRef
+					tk.source = "@" + tk.source
+				} else {
+					tk = scanner.makeErrorToken(fmt.Errorf("invalid variable reference %q", tk.source))
+				}
+			} else if next == '@' {
+				tk = scanner.moveOn(SymDoubleAt, ch, next)
+			} else {
+				tk = scanner.MakeToken(SymAt, ch)
+			}
+		case '_':
+			tk = scanner.MakeToken(SymUndescore, ch)
+		case '=':
+			if next, _ := scanner.peek(); next == '=' {
+				tk = scanner.moveOn(SymDoubleEqual, ch, next)
+			} else {
+				tk = scanner.MakeToken(SymEqual, ch)
+			}
+		case '<':
+			if next, _ := scanner.peek(); next == '=' {
+				tk = scanner.moveOn(SymLessOrEqual, ch, next)
+			} else if next == '<' {
+				scanner.readChar()
+				next2, _ := scanner.readChar()
+				scanner.unreadChar()
+				if next2 == '=' {
+					tk = scanner.moveOn(SymDoubleLessEqual, ch, next, next2)
+				} else {
+					tk = scanner.accept(SymDoubleLess, ch, next)
+				}
+			} else if next == '>' {
+				tk = scanner.moveOn(SymLessGreater, ch, next)
+			} else if next == '+' {
+				tk = scanner.moveOn(SymLessPlus, ch, next)
+			} else {
+				tk = scanner.MakeToken(SymLess, ch)
+			}
+		case '>':
+			if next, _ := scanner.peek(); next == '=' {
+				tk = scanner.moveOn(SymGreaterOrEqual, ch, next)
+			} else if next == '>' {
+				scanner.readChar()
+				next2, _ := scanner.readChar()
+				scanner.unreadChar()
+				if next2 == '=' {
+					tk = scanner.moveOn(SymDoubleGreaterEqual, ch, next, next2)
+				} else {
+					tk = scanner.accept(SymDoubleGreater, ch, next)
+				}
+			} else {
+				tk = scanner.MakeToken(SymGreater, ch)
+			}
+		case '$':
+			if next, _ := scanner.peek(); next == '(' {
+				tk = scanner.moveOn(SymDollarRound, ch, next)
+				tk.source += ")"
+			} else if next == '$' {
+				tk = scanner.moveOn(SymDoubleDollar, ch, next)
+			} else if next == '{' {
+				scanner.readChar()
+				if tk = scanner.fetchString('}', false); tk != nil {
+					tk.Sym = SymIdentifier
+				}
+			} else if next == '_' || (next >= 'a' && next <= 'z') || (next >= 'A' && next <= 'Z') {
+				scanner.readChar()
+				tk = scanner.fetchIdentifier(next)
+			} else {
+				tk = scanner.MakeToken(SymDollar, ch)
+			}
+		case '(':
+			//			if next, _ := scanner.peek(); next == ')' {
+			//				tk = scanner.moveOn(SymOpenClosedRound, ch, next)
+			//			} else {
+			tk = scanner.MakeToken(SymOpenRound, ch)
+			//			}
+		case ')':
+			tk = scanner.MakeToken(SymClosedRound, ch)
+		case '[':
+			tk = scanner.MakeToken(SymOpenSquare, ch)
+		case ']':
+			tk = scanner.MakeToken(SymClosedSquare, ch)
+		case '{':
+			tk = scanner.MakeToken(SymOpenBrace, ch)
+		case '}':
+			tk = scanner.MakeToken(SymClosedBrace, ch)
+		case '~':
+			tk = scanner.MakeToken(SymTilde, ch)
+		case 0:
+			if escape {
+				tk = scanner.makeErrorToken(errors.New("incomplete escape sequence"))
+			}
+			escape = false
+		default:
+			if /*ch == '_' ||*/ (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') {
+				if tk = scanner.fetchIdentifier(ch); tk.Sym == SymKwFunc {
+					if next, _ := scanner.peek(); next == '(' {
+						tk = scanner.moveOn(SymFuncDef, ch, next)
+					}
+				}
+			} else if ch >= '0' && ch <= '9' {
+				tk = scanner.parseNumber(ch)
+			}
+		}
+		if !escape {
+			break
+		}
+	}
+	if tk == nil {
+		tk = NewErrorToken(scanner.row, scanner.column, fmt.Errorf("unknown symbol '%c'", ch))
+	}
+	return
+}
+
+func (scanner *Scanner) sync(err error) error {
+	if err == nil {
+		err = scanner.unreadChar()
+	}
+	return err
+}
+
+func isBinaryDigit(ch byte) bool {
+	return ch == '0' || ch == '1'
+}
+
+func isOctalDigit(ch byte) bool {
+	return ch >= '0' && ch <= '7'
+}
+
+func isDecimalDigit(ch byte) bool {
+	return ch >= '0' && ch <= '9'
+}
+
+func isHexDigit(ch byte) bool {
+	return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F')
+}
+
+func (scanner *Scanner) initBase(currentFirstCh byte) (firstCh byte, numBase int, digitFunc func(byte) bool, err error) {
+	var ch byte
+	var digitType string
+	firstCh = currentFirstCh
+	digitFunc = isDecimalDigit
+	numBase = 10
+
+	if ch, err = scanner.peek(); err == nil {
+		if ch == 'b' || ch == 'B' {
+			numBase = 2
+			digitType = "binary"
+			scanner.readChar()
+			digitFunc = isBinaryDigit
+			firstCh, err = scanner.readChar()
+		} else if ch == 'o' || ch == 'O' {
+			numBase = 8
+			digitType = "octal"
+			scanner.readChar()
+			digitFunc = isOctalDigit
+			firstCh, err = scanner.readChar()
+		} else if ch == 'x' || ch == 'X' {
+			numBase = 16
+			digitType = "hex"
+			scanner.readChar()
+			digitFunc = isHexDigit
+			firstCh, err = scanner.readChar()
+		}
+		if err == nil && !digitFunc(firstCh) {
+			if len(digitType) == 0 {
+				digitType = "decimal"
+			}
+			err = fmt.Errorf("expected %s digit, got '%c'", digitType, firstCh)
+		}
+	} else if err == io.EOF {
+		err = nil
+	}
+	return
+}
+
+func (scanner *Scanner) parseNumber(firstCh byte) (tk *Token) {
+	var err error
+	var ch byte
+	var sym Symbol = SymInteger
+	var sb strings.Builder
+	var isDigit func(byte) bool = isDecimalDigit
+	var numBase = 10
+
+	if firstCh == '0' {
+		firstCh, numBase, isDigit, err = scanner.initBase(firstCh)
+	}
+	for ch = firstCh; err == nil && isDigit(ch); ch, err = scanner.readChar() {
+		sb.WriteByte(ch)
+	}
+
+	if numBase == 10 {
+		if err == nil && ch == '.' {
+			sym = SymFloat
+			sb.WriteByte(ch)
+			ch, err = scanner.readChar()
+			if ch >= '0' && ch <= '9' {
+				for ; err == nil && (ch >= '0' && ch <= '9'); ch, err = scanner.readChar() {
+					sb.WriteByte(ch)
+				}
+			}
+		}
+		if err == nil {
+			if ch == 'e' || ch == 'E' {
+				sym = SymFloat
+				sb.WriteByte(ch)
+				if ch, err = scanner.readChar(); err == nil {
+					if ch == '+' || ch == '-' {
+						sb.WriteByte(ch)
+						ch, err = scanner.readChar()
+					}
+					if ch >= '0' && ch <= '9' {
+						for ; err == nil && (ch >= '0' && ch <= '9'); ch, err = scanner.readChar() {
+							sb.WriteByte(ch)
+						}
+					} else {
+						err = fmt.Errorf("[%d:%d] expected integer exponent, got %c", scanner.row, scanner.column, ch)
+					}
+				}
+			} else if ch == '(' {
+				sym = SymFraction
+				sb.WriteByte(ch)
+				ch, err = scanner.readChar()
+				for ; err == nil && (ch >= '0' && ch <= '9'); ch, err = scanner.readChar() {
+					sb.WriteByte(ch)
+				}
+				if err == nil {
+					if ch != ')' {
+						err = fmt.Errorf("[%d:%d] expected ')', got '%c'", scanner.row, scanner.column, ch)
+					} else {
+						sb.WriteByte(ch)
+						_, err = scanner.readChar()
+					}
+				}
+			}
+		}
+	}
+
+	if err != nil && err != io.EOF {
+		tk = scanner.makeErrorToken(err)
+	} else {
+		var value any
+		_ = scanner.sync(err) // TODO: Check this function
+		txt := sb.String()
+		if sym == SymFloat {
+			value, err = strconv.ParseFloat(txt, 64)
+		} else if sym == SymFraction {
+			value, err = kern.MakeGeneratingFraction(txt)
+		} else {
+			value, err = strconv.ParseInt(txt, numBase, 64)
+		}
+		if err == nil {
+			tk = scanner.MakeValueToken(sym, txt, value)
+		} else {
+			tk = scanner.makeErrorToken(err)
+		}
+	}
+	return
+}
+
+func (scanner *Scanner) fetchIdentifier(firstCh byte) (tk *Token) {
+	var err error
+	var sb strings.Builder
+	for ch := firstCh; err == nil && (ch == '_' || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9')); ch, err = scanner.readChar() {
+		sb.WriteByte(ch)
+	}
+
+	if err != nil && err != io.EOF {
+		tk = scanner.makeErrorToken(err)
+	} else if err = scanner.sync(err); err != nil && err != io.EOF {
+		tk = scanner.makeErrorToken(err)
+	} else {
+		txt := sb.String()
+		uptxt := strings.ToUpper(txt)
+		if sym, ok := keywords[uptxt]; ok {
+			tk = scanner.makeKeywordToken(sym, uptxt)
+		} else if uptxt == `TRUE` {
+			tk = scanner.MakeValueToken(SymBool, txt, true)
+		} else if uptxt == `FALSE` {
+			tk = scanner.MakeValueToken(SymBool, txt, false)
+		} else if ch, _ := scanner.peek(); ch == '(' {
+			scanner.readChar()
+			tk = scanner.MakeValueToken(SymFuncCall, txt+"(", txt)
+		} else {
+			tk = scanner.MakeValueToken(SymIdentifier, txt, txt)
+		}
+	}
+
+	// if err != nil && err != io.EOF {
+	// 	tk = self.makeErrorToken(err)
+	// } else if err = self.sync(err); err != nil && err != io.EOF {
+	// 	tk = self.makeErrorToken(err)
+	// } else {
+	// 	txt := sb.String()
+	// 	uptxt := strings.ToUpper(txt)
+	// 	if sym, ok := keywords[uptxt]; ok {
+	// 		tk = self.makeValueToken(sym, txt, "")
+	// 	} else {
+	// 		tk = self.makeValueToken(SymIdentifier, txt, txt)
+	// 	}
+	// }
+	return
+}
+
+func (scanner *Scanner) fetchBlockComment() *Token {
+	return scanner.fetchUntil(SymComment, false, '*', '/')
+}
+
+func (scanner *Scanner) fetchOnLineComment() *Token {
+	return scanner.fetchUntil(SymComment, true, '\n')
+}
+
+func (scanner *Scanner) fetchUntil(sym Symbol, allowEos bool, endings ...byte) (tk *Token) {
+	var err error
+	var ch byte
+	var sb strings.Builder
+	var value string
+	ring := util.NewByteSlider(len(endings))
+	endReached := false
+	for ch, err = scanner.readChar(); err == nil && !endReached; {
+		sb.WriteByte(ch)
+		ring.PushEnd(ch)
+		if ring.Equal(endings) {
+			value = sb.String()[0 : sb.Len()-len(endings)]
+			endReached = true
+		} else {
+			ch, err = scanner.readChar()
+		}
+	}
+	if !endReached && allowEos {
+		value = sb.String()
+		endReached = true
+	}
+
+	if endReached {
+		tk = scanner.MakeValueToken(sym, "", value)
+	} else {
+		tk = scanner.makeErrorToken(err)
+	}
+	return
+}
+
+func (scanner *Scanner) fetchString(termCh byte, addQuote bool) (tk *Token) {
+	var err error
+	var ch, prev byte
+	var sb strings.Builder
+	for ch, err = scanner.readChar(); err == nil; ch, err = scanner.readChar() {
+		if prev == '\\' {
+			switch ch {
+			case '"':
+				sb.WriteByte('"')
+			case 'n':
+				sb.WriteByte('\n')
+			case 'r':
+				sb.WriteByte('\r')
+			case 't':
+				sb.WriteByte('\t')
+			case '\\':
+				sb.WriteByte('\\')
+			default:
+				sb.WriteByte(ch)
+			}
+			prev = 0
+		} else if ch == termCh {
+			break
+		} else {
+			prev = ch
+			if ch != '\\' {
+				sb.WriteByte(ch)
+			}
+		}
+	}
+	if err != nil {
+		if err == io.EOF {
+			tk = scanner.makeErrorToken(errors.New(string(termCh)))
+		} else {
+			tk = scanner.makeErrorToken(err)
+		}
+	} else {
+		txt := sb.String()
+		if addQuote {
+			tk = scanner.MakeValueToken(SymString, `"`+txt+`"`, txt)
+		} else {
+			tk = scanner.MakeValueToken(SymString, txt, txt)
+		}
+	}
+	return
+}
+
+func (scanner *Scanner) peek() (next byte, err error) {
+	var one []byte
+	if one, err = scanner.stream.Peek(1); err == nil {
+		next = one[0]
+	}
+	return
+}
+
+func (scanner *Scanner) skipBlanks() (err error) {
+	var one []byte
+	for one, err = scanner.stream.Peek(1); err == nil && one[0] <= 32; one, err = scanner.stream.Peek(1) {
+		scanner.readChar()
+	}
+	return
+}
+
+func (scanner *Scanner) translate(sym Symbol) Symbol {
+	if scanner.translations != nil {
+		if translatedSym, ok := scanner.translations[sym]; ok {
+			return translatedSym
+		}
+	}
+	return sym
+}
+
+func (scanner *Scanner) moveOn(sym Symbol, chars ...byte) (tk *Token) {
+	tk = NewToken(scanner.row, scanner.column, scanner.translate(sym), string(chars))
+	// for i := 1; i < len(chars); i++ {
+	if len(chars) > 1 {
+		scanner.readChar()
+	}
+	// }
+	return
+}
+
+func (scanner *Scanner) accept(sym Symbol, chars ...byte) (tk *Token) {
+	tk = NewToken(scanner.row, scanner.column, scanner.translate(sym), string(chars))
+	return
+}
+
+func (scanner *Scanner) MakeToken(sym Symbol, chars ...byte) (tk *Token) {
+	tk = NewToken(scanner.row, scanner.column, scanner.translate(sym), string(chars))
+	return
+}
+
+func (scanner *Scanner) makeKeywordToken(sym Symbol, upperCaseKeyword string) (tk *Token) {
+	tk = NewToken(scanner.row, scanner.column, scanner.translate(sym), upperCaseKeyword)
+	return
+}
+
+func (scanner *Scanner) MakeValueToken(sym Symbol, source string, value any) (tk *Token) {
+	tk = NewValueToken(scanner.row, scanner.column, scanner.translate(sym), source, value)
+	return
+}
+
+func (scanner *Scanner) makeErrorToken(err error) *Token {
+	return NewErrorToken(scanner.row, scanner.column, err)
+}

scan/symbol-map.go

@@ -0,0 +1,231 @@
+// Copyright (c) 2024 Celestino Amoroso (celestino.amoroso@gmail.com).
+// All rights reserved.
+
+// Symbol.go
+package scan
+
+import (
+	"strings"
+)
+
+var symbolMap map[Symbol]symbolSpec
+
+type SymbolClass int16
+
+const (
+	SymClassOperator SymbolClass = iota
+	SymClassCommand
+	SymClassIdentifier
+	SymClassDelimiter
+	SymClassParenthesis
+	SymClassDeclaration
+	SymClassValue
+	SymClassOther
+)
+
+type symbolSpec struct {
+	repr   string
+	kind   SymbolClass
+	opType TermPosition
+}
+
+func init() {
+
+	symbolMap = map[Symbol]symbolSpec{
+		SymUnknown:            {"<unknown>", SymClassOther, PosLeaf},  //  -1: Unknown symbol
+		SymNone:               {"<null>", SymClassOther, PosLeaf},     //   0: Null value for variable of type symbol
+		SymError:              {"<error>", SymClassOther, PosLeaf},    //   1: Error reading from stream
+		SymEos:                {"<eos>", SymClassOther, PosLeaf},      //   2: End of stream
+		SymMinus:              {"-", SymClassOperator, PosInfix},      //   3: '-'
+		SymMinusEqual:         {"-=", SymClassOperator, PosInfix},     //   4: '-='
+		SymDoubleMinus:        {"--", SymClassOperator, PosPostfix},   //   5: '--'
+		SymPlus:               {"+", SymClassOperator, PosInfix},      //   6: '+'
+		SymPlusEqual:          {"+=", SymClassOperator, PosInfix},     //   7: '+='
+		SymDoublePlus:         {"++", SymClassOperator, PosPostfix},   //   8: '++'
+		SymStar:               {"*", SymClassOperator, PosInfix},      //   9: '*'
+		SymDoubleStar:         {"**", SymClassOperator, PosInfix},     //  10: '**'
+		SymSlash:              {"/", SymClassOperator, PosInfix},      //  11: '/'
+		SymBackSlash:          {"\\", SymClassOperator, PosLeaf},      //  12: '\'
+		SymVertBar:            {"|", SymClassOperator, PosInfix},      //  13: '|'
+		SymDoubleVertBar:      {"||", SymClassOperator, PosInfix},     //  14: '||'
+		SymComma:              {",", SymClassOperator, PosInfix},      //  15: ','
+		SymColon:              {":", SymClassOperator, PosInfix},      //  16: ':'
+		SymSemiColon:          {";", SymClassOperator, PosInfix},      //  17: ';'
+		SymDot:                {".", SymClassOperator, PosInfix},      //  18: '.'
+		SymDotSlash:           {"./", SymClassOperator, PosInfix},     //  19: './'
+		SymQuote:              {"'", SymClassDelimiter, PosLeaf},      //  20: '\''
+		SymDoubleQuote:        {"\"", SymClassDelimiter, PosLeaf},     //  21: '"'
+		SymBackTick:           {"`", SymClassDelimiter, PosLeaf},      //  22: '`'
+		SymExclamation:        {"!", SymClassOperator, PosPostfix},    //  23: '!'
+		SymQuestion:           {"?", SymClassOperator, PosInfix},      //  24: '?'
+		SymAmpersand:          {"&", SymClassOperator, PosInfix},      //  25: '&'
+		SymDoubleAmpersand:    {"&&", SymClassOperator, PosInfix},     //  26: '&&'
+		SymPercent:            {"%", SymClassOperator, PosInfix},      //  27: '%'
+		SymAt:                 {"@", SymClassOperator, PosPrefix},     //  28: '@'
+		SymUndescore:          {"_", SymClassIdentifier, PosLeaf},     //  29: '_'
+		SymEqual:              {"=", SymClassOperator, PosInfix},      //  30: '='
+		SymDoubleEqual:        {"==", SymClassOperator, PosInfix},     //  31: '=='
+		SymLess:               {"<", SymClassOperator, PosInfix},      //  32: '<'
+		SymLessOrEqual:        {"<=", SymClassOperator, PosInfix},     //  33: '<='
+		SymGreater:            {">", SymClassOperator, PosInfix},      //  34: '>'
+		SymGreaterOrEqual:     {">=", SymClassOperator, PosInfix},     //  35: '>='
+		SymLessGreater:        {"<>", SymClassOperator, PosInfix},     //  36: '<>'
+		SymNotEqual:           {"!=", SymClassOperator, PosInfix},     //  37: '!='
+		SymDollar:             {"$", SymClassOperator, PosPrefix},     //  38: '$'
+		SymHash:               {"#", SymClassOperator, PosPrefix},     //  39: '#'
+		SymOpenRound:          {"(", SymClassParenthesis, PosPrefix},  //  40: '('
+		SymClosedRound:        {")", SymClassParenthesis, PosPostfix}, //  41: ')'
+		SymOpenSquare:         {"[", SymClassParenthesis, PosPrefix},  //  42: '['
+		SymClosedSquare:       {"]", SymClassParenthesis, PosPostfix}, //  43: ']'
+		SymOpenBrace:          {"{", SymClassParenthesis, PosPrefix},  //  44: '{'
+		SymClosedBrace:        {"}", SymClassParenthesis, PosPostfix}, //  45: '}'
+		SymTilde:              {"~", SymClassOperator, PosPrefix},     //  46: '~'
+		SymDoubleQuestion:     {"??", SymClassOperator, PosInfix},     //  47: '??'
+		SymQuestionEqual:      {"?=", SymClassOperator, PosInfix},     //  48: '?='
+		SymQuestionExclam:     {"?!", SymClassOperator, PosInfix},     //  49: '?!'
+		SymDoubleAt:           {"@@", SymClassCommand, PosLeaf},       //  50: '@@'
+		SymDoubleColon:        {"::", SymClassOperator, PosInfix},     //  51: '::'
+		SymDoubleGreater:      {">>", SymClassOperator, PosInfix},     //  52: '>>'
+		SymDoubleLess:         {"<<", SymClassOperator, PosInfix},     //  53: '<<'
+		SymCaret:              {"^", SymClassOperator, PosInfix},      //  54: '^'
+		SymDollarRound:        {"$(", SymClassOperator, PosPrefix},    //  55: '$('
+		SymOpenClosedRound:    {"()", SymClassOperator, PosPostfix},   //  56: '()'
+		SymDoubleDollar:       {"$$", SymClassCommand, PosLeaf},       //  57: '$$'
+		SymDoubleDot:          {"..", SymClassOperator, PosInfix},     //  58: '..'
+		SymTripleDot:          {"...", SymClassOperator, PosPostfix},  //  59: '...'
+		SymStarEqual:          {"*=", SymClassOperator, PosInfix},     //  60: '*='
+		SymSlashEqual:         {"/=", SymClassOperator, PosInfix},     //  61: '/='
+		SymPercEqual:          {"%=", SymClassOperator, PosInfix},     //  62: '%='
+		SymDoubleLessEqual:    {"<<=", SymClassOperator, PosInfix},    //  63: '<<='
+		SymDoubleGreaterEqual: {">>=", SymClassOperator, PosInfix},    //  64: '>>='
+		SymAmpersandEqual:     {"&=", SymClassOperator, PosInfix},     //  65: '&='
+		SymVertBarEqual:       {"|=", SymClassOperator, PosInfix},     //  65: '|='
+		SymCaretEqual:         {"^=", SymClassOperator, PosInfix},     //  66: '^='
+		SymPlusGreater:        {"+>", SymClassOperator, PosInfix},     //  67: '+>'
+		SymLessPlus:           {"<+", SymClassOperator, PosInfix},     //  68: '<+'
+		SymPreInc:             {"++", SymClassOperator, PosPrefix},    //    : '++'
+		SymPreDec:             {"--", SymClassOperator, PosPrefix},    //    : '--'
+		// SymChangeSign
+		// SymUnchangeSign
+		// SymIdentifier
+		// SymBool
+		// SymInteger
+		// SymVariable
+		// SymFloat
+		// SymFraction
+		// SymString
+		// SymIterator
+		// SymOr:  "or",
+		// SymAnd: "and",
+		// SymNot: "not",
+		// SymComment
+		// SymFuncCall
+		// SymFuncDef
+		// SymList
+		// SymDict
+		// SymIndex
+		// SymExpression
+		// SymSelector     // <selector> ::= <expr> "?" <selector-case> {":" <selector-case>} ["::" <default-selector-case>]
+		// SymSelectorCase // <selector-case> ::= [<list>] "{" <multi-expr> "}"
+		// // SymOpenComment                       //   0: '/*'
+		// // SymClosedComment                     //   0: '*/'
+		// // SymOneLineComment                    //   0: '//'
+		// keywordBase
+		SymKwAnd:     {"and", SymClassOperator, PosInfix},
+		SymKwNot:     {"not", SymClassOperator, PosInfix},
+		SymKwOr:      {"or", SymClassOperator, PosInfix},
+		SymKwBut:     {"but", SymClassOperator, PosInfix},
+		SymKwMap:     {"map", SymClassOperator, PosInfix},
+		SymKwFilter:  {"filter", SymClassOperator, PosInfix},
+		SymKwDigest:  {"digest", SymClassOperator, PosInfix},
+		SymKwJoin:    {"join", SymClassOperator, PosInfix},
+		SymKwGroupBy: {"groupby", SymClassOperator, PosInfix},
+		SymKwFunc:    {"func(", SymClassDeclaration, PosPrefix},
+		SymKwBuiltin: {"builtin", SymClassOperator, PosPrefix},
+		SymKwPlugin:  {"plugin", SymClassOperator, PosPrefix},
+		SymKwIn:      {"in", SymClassOperator, PosInfix},
+		SymKwInclude: {"include", SymClassOperator, PosPrefix},
+		SymKwNil:     {"nil", SymClassValue, PosLeaf},
+		SymKwUnset:   {"unset", SymClassOperator, PosPrefix},
+	}
+}
+
+func SymToString(sym Symbol) string {
+	if s, ok := symbolMap[sym]; ok {
+		return s.repr
+	}
+	return ""
+}
+
+func SymListToString(symList []Symbol, quote bool) string {
+	var sb strings.Builder
+	if len(symList) == 0 {
+		sb.WriteString("<nothing>")
+	} else {
+		for _, sym := range symList {
+			if sb.Len() > 0 {
+				sb.WriteByte(',')
+				sb.WriteByte(' ')
+			}
+			if quote {
+				sb.WriteByte('`')
+			}
+			sb.WriteString(SymToString(sym))
+			if quote {
+				sb.WriteByte('`')
+			}
+		}
+	}
+	return sb.String()
+}
+
+func StringEndsWithOperator(s string) bool {
+	return endingOperator(s) != SymNone
+}
+
+// func endingOperator(s string) (sym Symbol) {
+// 	var matchLength = 0
+// 	sym = SymNone
+// 	lower := strings.TrimRight(strings.ToLower(s), " \t")
+// 	for symbol, spec := range symbolMap {
+// 		if strings.HasSuffix(lower, spec.repr) {
+// 			if len(spec.repr) > matchLength {
+// 				matchLength = len(spec.repr)
+// 				if spec.kind == symClassOperator && (spec.opType == PosInfix || spec.opType == PosPrefix) {
+// 					sym = symbol
+// 				} else {
+// 					sym = SymNone
+// 				}
+// 			}
+// 		}
+// 	}
+// 	return
+// }
+
+func endingOperator(s string) (sym Symbol) {
+	var matchLength = 0
+	var repr string
+	sym = SymNone
+	lower := strings.TrimRight(strings.ToLower(s), " \t")
+	for symbol, spec := range symbolMap {
+		if len(spec.repr) > matchLength || repr == spec.repr {
+			if strings.HasSuffix(lower, spec.repr) {
+				if isNotEndingSymbol(spec) && repr != spec.repr {
+					repr = spec.repr
+					matchLength = len(spec.repr)
+					sym = symbol
+				} else {
+					sym = SymNone
+					break
+					// matchLength = 0
+				}
+			}
+		}
+	}
+	return
+}
+
+func isNotEndingSymbol(spec symbolSpec) bool {
+	return (spec.kind == SymClassOperator && (spec.opType == PosInfix || spec.opType == PosPrefix)) ||
+		(spec.kind == SymClassParenthesis && spec.opType == PosPrefix)
+}

scan/symbol.go

@@ -0,0 +1,153 @@
+// Copyright (c) 2024 Celestino Amoroso (celestino.amoroso@gmail.com).
+// All rights reserved.
+
+// Symbol.go
+package scan
+
+type Symbol int16
+
+const (
+	SymUnknown            Symbol = iota - 1 //  -1: Unknown symbol
+	SymNone                                 //   0: Null value for variable of type symbol
+	SymError                                //   1: Error reading from stream
+	SymEos                                  //   2: End of stream
+	SymMinus                                //   3: '-'
+	SymMinusEqual                           //   4: '-='
+	SymDoubleMinus                          //   5: '--'
+	SymPlus                                 //   6: '+'
+	SymPlusEqual                            //   7: '+='
+	SymDoublePlus                           //   8: '++'
+	SymStar                                 //   9: '*'
+	SymDoubleStar                           //  10: '**'
+	SymSlash                                //  11: '/'
+	SymBackSlash                            //  12: '\'
+	SymVertBar                              //  13: '|'
+	SymDoubleVertBar                        //  14: '||'
+	SymComma                                //  15: ','
+	SymColon                                //  16: ':'
+	SymSemiColon                            //  17: ';'
+	SymDot                                  //  18: '.'
+	SymDotSlash                             //  19: './'
+	SymQuote                                //  20: '\''
+	SymDoubleQuote                          //  21: '"'
+	SymBackTick                             //  22: '`'
+	SymExclamation                          //  23: '!'
+	SymQuestion                             //  24: '?'
+	SymAmpersand                            //  25: '&'
+	SymDoubleAmpersand                      //  26: '&&'
+	SymPercent                              //  27: '%'
+	SymAt                                   //  28: '@'
+	SymUndescore                            //  29: '_'
+	SymEqual                                //  30: '='
+	SymDoubleEqual                          //  31: '=='
+	SymLess                                 //  32: '<'
+	SymLessOrEqual                          //  33: '<='
+	SymGreater                              //  34: '>'
+	SymGreaterOrEqual                       //  35: '>='
+	SymLessGreater                          //  36: '<>'
+	SymNotEqual                             //  37: '!='
+	SymDollar                               //  38: '$'
+	SymHash                                 //  39: '#'
+	SymOpenRound                            //  40: '('
+	SymClosedRound                          //  41: ')'
+	SymOpenSquare                           //  42: '['
+	SymClosedSquare                         //  43: ']'
+	SymOpenBrace                            //  44: '{'
+	SymClosedBrace                          //  45: '}'
+	SymTilde                                //  46: '~'
+	SymDoubleQuestion                       //  47: '??'
+	SymQuestionEqual                        //  48: '?='
+	SymQuestionExclam                       //  49: '?!'
+	SymDoubleAt                             //  50: '@@'
+	SymDoubleColon                          //  51: '::'
+	SymDoubleGreater                        //  52: '>>'
+	SymDoubleLess                           //  53: '<<'
+	SymCaret                                //  54: '^'
+	SymDollarRound                          //  55: '$('
+	SymOpenClosedRound                      //  56: '()'
+	SymDoubleDollar                         //  57: '$$'
+	SymDoubleDot                            //  58: '..'
+	SymTripleDot                            //  59: '...'
+	SymStarEqual                            //  60: '*='
+	SymSlashEqual                           //  61: '/='
+	SymPercEqual                            //  62: '%='
+	SymDoubleLessEqual                      //  63: '<<='
+	SymDoubleGreaterEqual                   //  64: '>>='
+	SymAmpersandEqual                       //  65: '&='
+	SymVertBarEqual                         //  65: '|='
+	SymCaretEqual                           //  66: '^='
+	SymPlusGreater                          //  67: '+>'
+	SymLessPlus                             //  68: '<+'
+	SymChangeSign
+	SymUnchangeSign
+	SymDereference
+	SymPreInc
+	SymPreDec
+	SymIdentifier
+	SymBool
+	SymInteger
+	SymVariable
+	SymFloat
+	SymFraction
+	SymString
+	SymIterator
+	SymOr
+	SymAnd
+	SymNot
+	SymComment
+	SymFuncCall
+	SymFuncDef
+	SymList
+	SymDict
+	SymIndex
+	SymRange // [index : index]
+	SymExpression
+	SymSelector     // <selector> ::= <expr> "?" <selector-case> {":" <selector-case>} ["::" <default-selector-case>]
+	SymSelectorCase // <selector-case> ::= [<list>] "{" <multi-expr> "}"
+	// SymOpenComment                       //   0: '/*'
+	// SymClosedComment                     //   0: '*/'
+	// SymOneLineComment                    //   0: '//'
+	keywordBase
+)
+const (
+	SymKwAnd = keywordBase + iota
+	SymKwNot
+	SymKwOr
+	SymKwBut
+	SymKwFunc
+	SymKwBuiltin
+	SymKwPlugin
+	SymKwIn
+	SymKwInclude
+	SymKwMap
+	SymKwFilter
+	SymKwDigest
+	SymKwGroupBy
+	SymKwJoin
+	SymKwNil
+	SymKwUnset
+)
+
+var keywords map[string]Symbol
+
+func init() {
+	//keywords = make(map[string]Symbol)
+	keywords = map[string]Symbol{
+		"AND":     SymKwAnd,
+		"BUILTIN": SymKwBuiltin,
+		"PLUGIN":  SymKwPlugin,
+		"BUT":     SymKwBut,
+		"FUNC":    SymKwFunc,
+		"IN":      SymKwIn,
+		"INCLUDE": SymKwInclude,
+		"MAP":     SymKwMap,
+		"FILTER":  SymKwFilter,
+		"NOT":     SymKwNot,
+		"OR":      SymKwOr,
+		"NIL":     SymKwNil,
+		"UNSET":   SymKwUnset,
+		"DIGEST":  SymKwDigest,
+		"JOIN":    SymKwJoin,
+		"GROUPBY": SymKwGroupBy,
+	}
+}

scan/t_scanner_test.go

@@ -0,0 +1,100 @@
+// Copyright (c) 2024 Celestino Amoroso (celestino.amoroso@gmail.com).
+// All rights reserved.
+
+// t_scanner_test.go
+package scan
+
+import (
+	"errors"
+	"reflect"
+	"strings"
+	"testing"
+
+	"git.portale-stac.it/go-pkg/expr/kern"
+)
+
+func TestScanner(t *testing.T) {
+	type inputType struct {
+		source    string
+		wantSym   Symbol
+		wantValue any
+		wantErr   error
+	}
+
+	inputs := []inputType{
+		/*  1 */ {`123`, SymInteger, int64(123), nil},
+		/*  2 */ {"=", SymEqual, nil, nil},
+		/*  3 */ {`--`, SymDoubleMinus, nil, nil},
+		/*  4 */ {`++`, SymDoublePlus, nil, nil},
+		/*  5 */ {`**`, SymDoubleStar, nil, nil},
+		/*  6 */ {`&&`, SymDoubleAmpersand, nil, nil},
+		/*  7 */ {`||`, SymDoubleVertBar, nil, nil},
+		/*  8 */ {`NOT`, SymKwNot, nil, nil},
+		/*  9 */ {`AND`, SymKwAnd, nil, nil},
+		/* 10 */ {`or`, SymKwOr, nil, nil},
+		/* 11 */ {`+=`, SymPlusEqual, nil, nil},
+		/* 12 */ {`-=`, SymMinusEqual, nil, nil},
+		/* 13 */ {`|`, SymVertBar, nil, nil},
+		/* 14 */ {`:`, SymColon, nil, nil},
+		/* 15 */ {`;`, SymSemiColon, nil, nil},
+		/* 16 */ {`.`, SymDot, nil, nil},
+		/* 17 */ {`0.5`, SymFloat, float64(0.5), nil},
+		/* 18 */ {`\\`, SymBackSlash, nil, nil},
+		/* 19 */ {"`", SymBackTick, nil, nil},
+		/* 20 */ {"?", SymQuestion, nil, nil},
+		/* 21 */ {"&", SymAmpersand, nil, nil},
+		/* 22 */ {"@", SymAt, nil, nil},
+		/* 23 */ {`#`, SymHash, nil, nil},
+		/* 24 */ {`%`, SymPercent, nil, nil},
+		/* 25 */ {`\'`, SymQuote, nil, nil},
+		/* 26 */ {`\"`, SymDoubleQuote, nil, nil},
+		/* 27 */ {`_`, SymUndescore, nil, nil},
+		/* 28 */ {`<>`, SymLessGreater, nil, nil},
+		/* 29 */ {`[`, SymOpenSquare, nil, nil},
+		/* 30 */ {`]`, SymClosedSquare, nil, nil},
+		/* 31 */ {`{`, SymOpenBrace, nil, nil},
+		/* 32 */ {`}`, SymClosedBrace, nil, nil},
+		/* 33 */ {`(`, SymOpenRound, nil, nil},
+		/* 34 */ {`)`, SymClosedRound, nil, nil},
+		/* 35 */ {`1E+2`, SymFloat, float64(100), nil},
+		/* 36 */ {`1E+x`, SymError, errors.New("[1:5] expected integer exponent, got x"), nil},
+		/* 37 */ {`$`, SymDollar, nil, nil},
+		/* 38 */ {`\`, SymError, errors.New("incomplete escape sequence"), nil},
+		/* 39 */ {`"string"`, SymString, "string", nil},
+		/* 40 */ {`identifier`, SymIdentifier, "identifier", nil},
+		/* 41 */ {`1.2(3)`, SymFraction, kern.NewFraction(37, 30), nil},
+	}
+
+	for i, input := range inputs {
+		// if i != 40 {
+		// 	continue
+		// }
+		if input.wantErr == nil {
+			t.Logf("[+]Test nr %2d -- %q", i+1, input.source)
+		} else {
+			t.Logf("[-]Test nr %2d -- %q", i+1, input.source)
+		}
+
+		r := strings.NewReader(input.source)
+		scanner := NewScanner(r, nil)
+
+		if tk := scanner.Next(); tk == nil {
+			t.Errorf("%d: %q -> got = (nil), want %v (value %v [%T])", i+1, input.source, input.wantSym, input.wantValue, input.wantValue)
+			// } else if tk.Sym != input.wantSym || tk.Value != input.wantValue {
+		} else if tk.Sym != input.wantSym || !reflect.DeepEqual(tk.Value, input.wantValue) {
+			if tk.Sym == SymError && input.wantSym == tk.Sym {
+				if tkErr, tkOk := tk.Value.(error); tkOk {
+					if inputErr, inputOk := input.wantValue.(error); inputOk {
+						if tkErr.Error() != inputErr.Error() {
+							t.Errorf("%d: %q -> got-error = %v, want-error: %v", i+1,
+								input.source, tk.Value, input.wantValue)
+						}
+					}
+				}
+			} else {
+				t.Errorf("%d: %q -> got = %v (value=%v [%T]), want %v (value=%v [%T])", i+1,
+					input.source, tk.Sym, tk.Value, tk.Value, input.wantSym, input.wantValue, input.wantValue)
+			}
+		}
+	}
+}

scan/term-constructor-registry.go

@@ -0,0 +1,27 @@
+// Copyright (c) 2024 Celestino Amoroso (celestino.amoroso@gmail.com).
+// All rights reserved.
+
+// term-constructor-registry.go
+package scan
+
+const initialRegistryCapacity = 10
+
+type termContructor func(tk *Token) *Term
+
+var constructorRegistry map[Symbol]termContructor = nil
+
+func RegisterTermConstructor(sym Symbol, constructor termContructor) {
+	if constructorRegistry == nil {
+		constructorRegistry = make(map[Symbol]termContructor, initialRegistryCapacity)
+	}
+	constructorRegistry[sym] = constructor
+}
+
+func NewTerm(tk *Token) (inst *Term) {
+	if constructorRegistry != nil {
+		if construct, exists := constructorRegistry[tk.Sym]; exists {
+			inst = construct(tk)
+		}
+	}
+	return
+}

scan/term.go

@@ -0,0 +1,301 @@
+// Copyright (c) 2024 Celestino Amoroso (celestino.amoroso@gmail.com).
+// All rights reserved.
+
+// term.go
+package scan
+
+import (
+	"strings"
+
+	"git.portale-stac.it/go-pkg/expr/kern"
+)
+
+type TermPriority uint32
+
+const (
+	PriNone TermPriority = iota
+	PriRange
+	PriIterOp // map, filter, digest, etc
+	PriBut
+	PriAssign
+	PriInsert
+	PriOr
+	PriAnd
+	PriNot
+	PriRelational
+	PriBitwiseOr
+	PriBitwiseAnd
+	PriBitwiseNot
+	PriSum
+	PriProduct
+	PriFraction
+	PriSelector
+	PriBinShift
+	PriSign
+	PriFact
+	PriIterValue
+	PriDefault
+	PriIncDec
+	PriDot
+	PriDereference
+	PriValue
+	PriBitwiseXor = PriBitwiseOr
+)
+
+type TermPosition uint8
+
+const (
+	PosLeaf TermPosition = iota
+	PosInfix
+	PosPrefix
+	PosPostfix
+	PosMultifix
+)
+
+type EvalFuncType func(ctx kern.ExprContext, self *Term) (v any, err error)
+
+type Term struct {
+	Tk       Token
+	Parent   *Term
+	Children []*Term
+	Position TermPosition // operator position: leaf, infix, prefix, postfix, multifix
+	Priority TermPriority // operator priority: higher value means higher priority
+	EvalFunc EvalFuncType
+}
+
+func (t *Term) Clone() (d *Term) {
+	var children []*Term
+	if t.Children != nil {
+		children = make([]*Term, len(t.Children))
+		for i, c := range t.Children {
+			children[i] = c.Clone()
+		}
+	}
+	d = &Term{
+		Tk:       *t.Tk.Clone(),
+		Parent:   t.Parent,
+		Children: children,
+		Position: t.Position,
+		Priority: t.Priority,
+		EvalFunc: t.EvalFunc,
+	}
+	return
+}
+
+func (t *Term) String() string {
+	var sb strings.Builder
+	t.ToString(&sb)
+	return sb.String()
+}
+
+func (t *Term) ToString(sb *strings.Builder) {
+	if t.Position == PosLeaf {
+		sb.WriteString(t.Tk.String())
+	} else {
+		sb.WriteByte('[')
+		sb.WriteString(t.Tk.String())
+		if t.Children != nil {
+			sb.WriteByte('(')
+			for i, c := range t.Children {
+				if i > 0 {
+					sb.WriteByte(' ')
+				}
+				c.ToString(sb)
+			}
+			sb.WriteByte(')')
+		}
+		sb.WriteByte(']')
+	}
+}
+
+func (t *Term) GetChildCount() (count int) {
+	if t.Position == PosLeaf || t.Children == nil {
+		count = 0
+	} else {
+		count = len(t.Children)
+	}
+	return
+}
+
+func (t *Term) GetRoom() (room int) {
+	switch t.Position {
+	case PosLeaf:
+		room = 0
+	case PosInfix:
+		room = 2
+	case PosPostfix, PosPrefix:
+		room = 1
+	default:
+		panic("Invalid node position")
+	}
+	return
+}
+
+func (t *Term) isComplete() bool {
+	return t.GetChildCount() == t.GetRoom()
+}
+
+func (t *Term) removeLastChild() (child *Term) {
+	if t.Children != nil {
+		child = t.Children[len(t.Children)-1]
+		t.Children = t.Children[0 : len(t.Children)-1]
+	} else {
+		panic("Can't get last child")
+	}
+	return
+}
+
+func (t *Term) isLeaf() bool {
+	return t.Position == PosLeaf
+}
+
+func (t *Term) getPriority() TermPriority {
+	return t.Priority
+}
+
+func (t *Term) SetParent(parent *Term) {
+	t.Parent = parent
+	if parent != nil && len(parent.Children) < cap(parent.Children) {
+		parent.Children = append(parent.Children, t)
+	}
+}
+
+func (t *Term) Symbol() Symbol {
+	return t.Tk.Sym
+}
+
+func (t *Term) Source() string {
+	return t.Tk.source
+}
+
+func (t *Term) Value() any {
+	return t.Tk.Value
+}
+
+func (t *Term) GetChild(index int) (c kern.Term) {
+	if index >= 0 && index < len(t.Children) {
+		c = t.Children[index]
+	}
+	return
+}
+
+func (t *Term) GetChildSource(index int) (s string) {
+	if index >= 0 && index < len(t.Children) {
+		s = t.Children[index].Tk.source
+	}
+	return
+}
+
+func (t *Term) IsAssign() bool {
+	return t.Symbol() == SymEqual
+}
+
+func (t *Term) Compute(ctx kern.ExprContext) (v any, err error) {
+	if t.EvalFunc == nil {
+		err = t.Errorf("undefined eval-func for %q term", t.Source())
+	} else {
+		v, err = t.EvalFunc(ctx, t)
+	}
+	return
+}
+
+// func (term *term) toInt(computedValue any, valueDescription string) (i int, err error) {
+// 	if index64, ok := computedValue.(int64); ok {
+// 		i = int(index64)
+// 	} else {
+// 		err = term.Errorf("%s, got %s (%v)", valueDescription, TypeName(computedValue), computedValue)
+// 	}
+// 	return
+// }
+
+func (t *Term) ErrIncompatibleTypes(leftValue, rightValue any) error {
+	leftType := kern.TypeName(leftValue)
+	leftText := kern.GetFormatted(leftValue, kern.Truncate)
+	rightType := kern.TypeName(rightValue)
+	rightText := kern.GetFormatted(rightValue, kern.Truncate)
+	return t.Tk.Errorf(
+		"left operand '%s' [%s] and right operand '%s' [%s] are not compatible with operator %q",
+		leftText, leftType,
+		rightText, rightType,
+		t.Source())
+}
+
+func (t *Term) ErrIncompatibleType(value any, side string) error {
+	return t.Tk.Errorf(
+		"operator %q does not support operand '%v' [%s] on its %s side",
+		t.Source(), value, kern.TypeName(value), side)
+}
+
+func (t *Term) ErrIncompatiblePrefixPostfixType(value any) error {
+	return t.Tk.Errorf(
+		"prefix/postfix operator %q does not support operand '%v' [%s]",
+		t.Source(), value, kern.TypeName(value))
+}
+
+func (t *Term) ErrDivisionByZero() error {
+	return t.Tk.Errorf("division by zero")
+}
+
+func (t *Term) ErrKeyNotFound(key any) error {
+	return t.Tk.Errorf("key '%v' not found", key)
+}
+
+func (t *Term) Errorf(template string, args ...any) (err error) {
+	err = t.Tk.Errorf(template, args...)
+	return
+}
+
+func (t *Term) CheckOperands() (err error) {
+	switch t.Position {
+	case PosInfix:
+		if t.Children == nil || len(t.Children) != 2 || t.anyChildrenNil() {
+			err = t.Tk.Errorf("infix operator %q requires two non-nil operands, got %d", t.Source(), t.GetChildCount())
+		}
+	case PosPrefix:
+		if t.Children == nil || len(t.Children) != 1 || t.Children[0] == nil {
+			err = t.Tk.Errorf("prefix operator %q requires one non-nil operand", t.Tk.String())
+		}
+	case PosPostfix:
+		if t.Children == nil || len(t.Children) != 1 || t.anyChildrenNil() {
+			err = t.Tk.Errorf("postfix operator %q requires one non-nil operand", t.Tk.String())
+		}
+	case PosMultifix:
+		if len(t.Children) < 3 || t.anyChildrenNil() {
+			err = t.Tk.Errorf("infix operator %q requires at least three not operands, got %d", t.Source(), t.GetChildCount())
+		}
+	}
+	return
+}
+
+func (t *Term) anyChildrenNil() bool {
+	for _, child := range t.Children {
+		if child == nil {
+			return true
+		}
+	}
+	return false
+}
+func (t *Term) EvalInfix(ctx kern.ExprContext) (leftValue, rightValue any, err error) {
+	if err = t.CheckOperands(); err == nil {
+		if leftValue, err = t.Children[0].Compute(ctx); err == nil {
+			rightValue, err = t.Children[1].Compute(ctx)
+		}
+	}
+	return
+}
+
+func (t *Term) EvalPrefix(ctx kern.ExprContext) (childValue any, err error) {
+	if err = t.CheckOperands(); err == nil {
+		childValue, err = t.Children[0].Compute(ctx)
+	}
+	return
+}
+
+// NOTE Temporary solution to support function parameters with default value
+
+func (t *Term) ForceChild(c *Term) {
+	if t.Children == nil {
+		t.Children = make([]*Term, 0, 1)
+	}
+	t.Children = append(t.Children, c)
+}

scan/token.go

@@ -0,0 +1,138 @@
+// Copyright (c) 2024 Celestino Amoroso (celestino.amoroso@gmail.com).
+// All rights reserved.
+
+// token.go
+package scan
+
+import (
+	"fmt"
+	"io"
+	"slices"
+)
+
+type Token struct {
+	row    int
+	col    int
+	Sym    Symbol
+	source string
+	Value  any
+}
+
+func (tk *Token) DevString() string {
+	if tk.Value != nil {
+		return fmt.Sprintf("[%d]%q{%v}", tk.Sym, tk.source, tk.Value)
+	}
+	return fmt.Sprintf("[%d]%q{}", tk.Sym, tk.source)
+}
+
+func (tk *Token) String() string {
+	if tk.Value != nil {
+		if s, ok := tk.Value.(string); ok {
+			return s //fmt.Sprintf("%q", s)
+		} else {
+			return fmt.Sprintf("%v", tk.Value)
+		}
+	}
+	return tk.source
+}
+
+func NewToken(row, col int, sym Symbol, source string) *Token {
+	return &Token{row: row, col: col, Sym: sym, source: source}
+}
+
+func NewValueToken(row, col int, sym Symbol, source string, value any) *Token {
+	return &Token{row: row, col: col, Sym: sym, source: source, Value: value}
+}
+
+func NewErrorToken(row, col int, err error) *Token {
+	if err == io.EOF {
+		return NewToken(row, col, SymEos, "<EOF>")
+	}
+	return NewValueToken(row, col, SymError, fmt.Sprintf("[%d:%d]", row, col), err)
+}
+
+func (tk *Token) Source() string {
+	return tk.source
+}
+
+func (tk *Token) Row() int {
+	return tk.row
+}
+
+func (tk *Token) Col() int {
+	return tk.col
+}
+
+func (tk *Token) Clone() (c *Token) {
+	return NewValueToken(tk.row, tk.col, tk.Sym, tk.source, tk.Value)
+}
+
+func (tk *Token) IsEos() bool {
+	return tk.Sym == SymEos
+}
+
+func (tk *Token) IsError() bool {
+	return tk.Sym == SymError
+}
+
+func (tk *Token) IsTerm(termSymbols []Symbol) bool {
+	return tk.IsEos() || tk.IsError() || tk.IsOneOf(termSymbols)
+}
+
+func (tk *Token) IsOneOf(termSymbols []Symbol) bool {
+	return termSymbols != nil && slices.Index(termSymbols, tk.Sym) >= 0
+}
+
+func (tk *Token) IsOneOfA(termSymbols ...Symbol) bool {
+	return slices.Index(termSymbols, tk.Sym) >= 0
+}
+
+func (tk *Token) IsSymbol(sym Symbol) bool {
+	return tk.Sym == sym
+}
+
+func (tk *Token) SetSymbol(sym Symbol) {
+	tk.Sym = sym
+}
+
+func (tk *Token) Errorf(template string, args ...any) (err error) {
+	err = fmt.Errorf(fmt.Sprintf("[%d:%d] ", tk.row, tk.col)+template, args...)
+	return
+}
+
+func (tk *Token) Error() (err error) {
+	if tk.Sym == SymError {
+		if msg, ok := tk.Value.(error); ok {
+			err = fmt.Errorf("[%d:%d] %v", tk.row, tk.col, msg)
+		}
+	}
+	return
+}
+
+func (tk *Token) ErrorText() (err string) {
+	if tk.Sym == SymError {
+		if msg, ok := tk.Value.(error); ok {
+			err = msg.Error()
+		}
+	}
+	return
+}
+
+func (tk *Token) Errors(msg string) (err error) {
+	err = fmt.Errorf("[%d:%d] %v", tk.row, tk.col, msg)
+	return
+}
+
+func (tk *Token) ErrorExpectedGot(symbol string) (err error) {
+	err = fmt.Errorf("[%d:%d] expected `%s`, got `%s`", tk.row, tk.col, symbol, tk)
+	return
+}
+
+func (tk *Token) ErrorExpectedGotString(symbol, got string) (err error) {
+	return tk.ErrorExpectedGotStringWithPrefix("expected", symbol, got)
+}
+
+func (tk *Token) ErrorExpectedGotStringWithPrefix(prefix, symbol, got string) (err error) {
+	err = fmt.Errorf("[%d:%d] %s %s, got `%s`", tk.row, tk.col, prefix, symbol, got)
+	return
+}