Added all source files

token.go

@@ -0,0 +1,55 @@
+// token.go
+package expr
+
+import (
+	"fmt"
+	"io"
+	"slices"
+)
+
+type Token struct {
+	Sym    Symbol
+	source string
+	Value  any
+}
+
+func (tk *Token) DevString() string {
+	if tk.Value != nil {
+		return fmt.Sprintf("[%d]%q{%v}", tk.Sym, tk.source, tk.Value)
+	}
+	return fmt.Sprintf("[%d]%q{}", tk.Sym, tk.source)
+}
+
+func (tk *Token) String() string {
+	if tk.Value != nil {
+		return fmt.Sprintf("%#v", tk.Value)
+	}
+	return fmt.Sprintf("%s", tk.source)
+}
+
+func NewToken(sym Symbol, source string) *Token {
+	return &Token{Sym: sym, source: source}
+}
+
+func NewValueToken(sym Symbol, source string, value any) *Token {
+	return &Token{Sym: sym, source: source, Value: value}
+}
+
+func NewErrorToken(row, col int, err error) *Token {
+	if err == io.EOF {
+		return NewToken(SymEos, "")
+	}
+	return NewValueToken(SymError, fmt.Sprintf("[%d:%d]", row, col), err)
+}
+
+func (tk *Token) IsEos() bool {
+	return tk.Sym == SymEos
+}
+
+func (tk *Token) IsError() bool {
+	return tk.Sym == SymError
+}
+
+func (tk *Token) IsTerm(termSymbols []Symbol) bool {
+	return tk.IsEos() || tk.IsError() || (termSymbols != nil && slices.Index(termSymbols, tk.Sym) >= 0)
+}