moved scanner sources to package 'scan'

scan/token.go

@@ -0,0 +1,138 @@
+// Copyright (c) 2024 Celestino Amoroso (celestino.amoroso@gmail.com).
+// All rights reserved.
+
+// token.go
+package scan
+
+import (
+	"fmt"
+	"io"
+	"slices"
+)
+
+type Token struct {
+	row    int
+	col    int
+	Sym    Symbol
+	source string
+	Value  any
+}
+
+func (tk *Token) DevString() string {
+	if tk.Value != nil {
+		return fmt.Sprintf("[%d]%q{%v}", tk.Sym, tk.source, tk.Value)
+	}
+	return fmt.Sprintf("[%d]%q{}", tk.Sym, tk.source)
+}
+
+func (tk *Token) String() string {
+	if tk.Value != nil {
+		if s, ok := tk.Value.(string); ok {
+			return s //fmt.Sprintf("%q", s)
+		} else {
+			return fmt.Sprintf("%v", tk.Value)
+		}
+	}
+	return tk.source
+}
+
+func NewToken(row, col int, sym Symbol, source string) *Token {
+	return &Token{row: row, col: col, Sym: sym, source: source}
+}
+
+func NewValueToken(row, col int, sym Symbol, source string, value any) *Token {
+	return &Token{row: row, col: col, Sym: sym, source: source, Value: value}
+}
+
+func NewErrorToken(row, col int, err error) *Token {
+	if err == io.EOF {
+		return NewToken(row, col, SymEos, "<EOF>")
+	}
+	return NewValueToken(row, col, SymError, fmt.Sprintf("[%d:%d]", row, col), err)
+}
+
+func (tk *Token) Source() string {
+	return tk.source
+}
+
+func (tk *Token) Row() int {
+	return tk.row
+}
+
+func (tk *Token) Col() int {
+	return tk.col
+}
+
+func (tk *Token) Clone() (c *Token) {
+	return NewValueToken(tk.row, tk.col, tk.Sym, tk.source, tk.Value)
+}
+
+func (tk *Token) IsEos() bool {
+	return tk.Sym == SymEos
+}
+
+func (tk *Token) IsError() bool {
+	return tk.Sym == SymError
+}
+
+func (tk *Token) IsTerm(termSymbols []Symbol) bool {
+	return tk.IsEos() || tk.IsError() || tk.IsOneOf(termSymbols)
+}
+
+func (tk *Token) IsOneOf(termSymbols []Symbol) bool {
+	return termSymbols != nil && slices.Index(termSymbols, tk.Sym) >= 0
+}
+
+func (tk *Token) IsOneOfA(termSymbols ...Symbol) bool {
+	return slices.Index(termSymbols, tk.Sym) >= 0
+}
+
+func (tk *Token) IsSymbol(sym Symbol) bool {
+	return tk.Sym == sym
+}
+
+func (tk *Token) SetSymbol(sym Symbol) {
+	tk.Sym = sym
+}
+
+func (tk *Token) Errorf(template string, args ...any) (err error) {
+	err = fmt.Errorf(fmt.Sprintf("[%d:%d] ", tk.row, tk.col)+template, args...)
+	return
+}
+
+func (tk *Token) Error() (err error) {
+	if tk.Sym == SymError {
+		if msg, ok := tk.Value.(error); ok {
+			err = fmt.Errorf("[%d:%d] %v", tk.row, tk.col, msg)
+		}
+	}
+	return
+}
+
+func (tk *Token) ErrorText() (err string) {
+	if tk.Sym == SymError {
+		if msg, ok := tk.Value.(error); ok {
+			err = msg.Error()
+		}
+	}
+	return
+}
+
+func (tk *Token) Errors(msg string) (err error) {
+	err = fmt.Errorf("[%d:%d] %v", tk.row, tk.col, msg)
+	return
+}
+
+func (tk *Token) ErrorExpectedGot(symbol string) (err error) {
+	err = fmt.Errorf("[%d:%d] expected `%s`, got `%s`", tk.row, tk.col, symbol, tk)
+	return
+}
+
+func (tk *Token) ErrorExpectedGotString(symbol, got string) (err error) {
+	return tk.ErrorExpectedGotStringWithPrefix("expected", symbol, got)
+}
+
+func (tk *Token) ErrorExpectedGotStringWithPrefix(prefix, symbol, got string) (err error) {
+	err = fmt.Errorf("[%d:%d] %s %s, got `%s`", tk.row, tk.col, prefix, symbol, got)
+	return
+}