From 522b880ca37276c323f997fc882f62d4e5e7245a Mon Sep 17 00:00:00 2001
From: Celestino Amoroso <celestino.amoroso@gmail.com>
Date: Tue, 26 Mar 2024 07:00:53 +0100
Subject: [PATCH] Added all source files
---
ast.go | 84 ++++++
ast_test.go | 50 ++++
byte-ring.go | 31 ++
context.go | 15 +
operand-const.go | 72 +++++
operand-func.go | 33 +++
operand-var.go | 32 +++
operator-bool.go | 106 +++++++
operator-fact.go | 46 +++
operator-prod.go | 95 +++++++
operator-rel.go | 220 ++++++++++++++
operator-sign.go | 61 ++++
operator-sum.go | 84 ++++++
parser.go | 76 +++++
parser_test.go | 215 ++++++++++++++
scanner.go | 552 ++++++++++++++++++++++++++++++++++++
scanner_test.go | 91 ++++++
symbol.go | 82 ++++++
term-constuctor-registry.go | 25 ++
term.go | 229 +++++++++++++++
term_test.go | 41 +++
token.go | 55 ++++
token_test.go | 15 +
utils.go | 51 ++++
24 files changed, 2361 insertions(+)
create mode 100644 ast.go
create mode 100644 ast_test.go
create mode 100644 byte-ring.go
create mode 100644 context.go
create mode 100644 operand-const.go
create mode 100644 operand-func.go
create mode 100644 operand-var.go
create mode 100644 operator-bool.go
create mode 100644 operator-fact.go
create mode 100644 operator-prod.go
create mode 100644 operator-rel.go
create mode 100644 operator-sign.go
create mode 100644 operator-sum.go
create mode 100644 parser.go
create mode 100644 parser_test.go
create mode 100644 scanner.go
create mode 100644 scanner_test.go
create mode 100644 symbol.go
create mode 100644 term-constuctor-registry.go
create mode 100644 term.go
create mode 100644 term_test.go
create mode 100644 token.go
create mode 100644 token_test.go
create mode 100644 utils.go
diff --git a/ast.go b/ast.go
new file mode 100644
index 0000000..22d53f7
--- /dev/null
+++ b/ast.go
@@ -0,0 +1,84 @@
+// ast.go
+package expr
+
+import (
+ "errors"
+ "fmt"
+ "strings"
+)
+
+//-------- ast
+
+type ast struct {
+ root *term
+}
+
+func NewAst() *ast {
+ return &ast{}
+}
+
+func (self *ast) String() string {
+ var sb strings.Builder
+ if self.root == nil {
+ sb.WriteString("(nil)")
+ } else {
+ self.root.toString(&sb)
+ }
+ return sb.String()
+}
+
+func (self *ast) addTokens(tokens ...*Token) (err error) {
+ for _, tk := range tokens {
+ if err = self.addToken(tk); err != nil {
+ break
+ }
+ }
+ return
+}
+
+func (self *ast) addToken(tk *Token) (err error) {
+ if t := newTerm(tk, nil); t != nil {
+ err = self.addTerm(t)
+ } else {
+ err = fmt.Errorf("No term constructor for token %q", tk.String())
+ }
+ return
+}
+
+func (self *ast) addTerm(node *term) (err error) {
+ if self.root == nil {
+ self.root = node
+ } else {
+ self.root, err = self.insert(self.root, node)
+ }
+ return
+}
+
+func (self *ast) insert(tree, node *term) (root *term, err error) {
+ if tree.getPriority() < node.getPriority() {
+ root = tree
+ if tree.isComplete() {
+ var subRoot *term
+ last := tree.removeLastChild()
+ subRoot, err = self.insert(last, node)
+ subRoot.setParent(tree)
+ } else {
+ node.setParent(tree)
+ }
+ } else if !node.isLeaf() {
+ root = node
+ tree.setParent(node)
+ } else {
+ err = fmt.Errorf("two adjacent operators: %q and %q", tree, node)
+ }
+ return
+}
+
+func (self *ast) eval(ctx exprContext) (result any, err error) {
+ if self.root != nil {
+ result, err = self.root.compute(ctx)
+ } else {
+ err = errors.New("empty expression")
+ }
+ return
+}
diff --git a/ast_test.go b/ast_test.go
new file mode 100644
index 0000000..b1a97cd
--- /dev/null
+++ b/ast_test.go
@@ -0,0 +1,50 @@
+// ast_test.go
+package expr
+
+import (
+ "errors"
+ "testing"
+)
+
+func TestAstString(t *testing.T) {
+ tree := NewAst()
+ if gotResult := tree.String(); gotResult != "(nil)" {
+ t.Errorf(`result: got %q, want "(nil)"`, gotResult)
+ }
+}
+
+func TestAddTokensGood(t *testing.T) {
+ tk1 := NewValueToken(SymInteger, "100", 100)
+ tk2 := NewToken(SymPlus, "+")
+ tk3 := NewValueToken(SymInteger, "50", 500)
+
+ tree := NewAst()
+ if gotErr := tree.addTokens(tk1, tk2, tk3); gotErr != nil {
+ t.Errorf("err: got <%v>, want <nil>", gotErr)
+ }
+}
+
+func TestAddTokensBad(t *testing.T) {
+ tk0 := NewValueToken(SymInteger, "200", 200)
+ tk1 := NewValueToken(SymInteger, "100", 100)
+ tk2 := NewToken(SymPlus, "+")
+ tk3 := NewValueToken(SymInteger, "50", 500)
+
+ wantErr := errors.New(`two adjacent operators: "200" and "100"`)
+
+ tree := NewAst()
+ if gotErr := tree.addTokens(tk0, tk1, tk2, tk3); gotErr != nil && gotErr.Error() != wantErr.Error() {
+ t.Errorf("err: got <%v>, want <nil>", gotErr)
+ }
+}
+
+func TestAddUnknownTokens(t *testing.T) {
+ tk0 := NewToken(SymPercent, "%")
+
+ wantErr := errors.New(`No term constructor for token "%"`)
+
+ tree := NewAst()
+ if gotErr := tree.addToken(tk0); gotErr != nil && gotErr.Error() != wantErr.Error() {
+ t.Errorf("err: got <%v>, want <%v>", gotErr, wantErr)
+ }
+}
diff --git a/byte-ring.go b/byte-ring.go
new file mode 100644
index 0000000..7428727
--- /dev/null
+++ b/byte-ring.go
@@ -0,0 +1,31 @@
+// byte-slider.go
+package expr
+
+import "bytes"
+
+type ByteSlider struct {
+ buf []byte
+ length int
+}
+
+func NewByteSlider(size int) *ByteSlider {
+ return &ByteSlider{
+ buf: make([]byte, max(1, size)),
+ length: 0,
+ }
+}
+
+func (self *ByteSlider) PushEnd(b byte) {
+ if self.length == cap(self.buf) {
+ self.length--
+ for i := 0; i < self.length; i++ {
+ self.buf[i] = self.buf[i+1]
+ }
+ }
+ self.buf[self.length] = b
+ self.length++
+}
+
+func (self *ByteSlider) Equal(target []byte) bool {
+ return target != nil && bytes.Equal(self.buf, target)
+}
diff --git a/context.go b/context.go
new file mode 100644
index 0000000..bae6baa
--- /dev/null
+++ b/context.go
@@ -0,0 +1,15 @@
+// context.go
+package expr
+
+type exprFunc interface {
+ Name() string
+ MinArgs() int
+ MaxArgs() int
+}
+
+type exprContext interface {
+ GetValue(varName string) (value any, exists bool)
+ SetValue(varName string, value any)
+ GetFuncInfo(name string) exprFunc
+ Call(name string, args []any) (result any, err error)
+}
diff --git a/operand-const.go b/operand-const.go
new file mode 100644
index 0000000..30ce0e0
--- /dev/null
+++ b/operand-const.go
@@ -0,0 +1,72 @@
+// operand-const.go
+package expr
+
+// -------- bool const term
+func newBoolTerm(tk *Token) *term {
+ return &term{
+ tk: *tk,
+ class: classConst,
+ kind: kindBool,
+ parent: nil,
+ children: nil,
+ position: posLeaf,
+ priority: priValue,
+ evalFunc: evalConst,
+ }
+}
+
+// -------- integer const term
+func newIntegerTerm(tk *Token) *term {
+ return &term{
+ tk: *tk,
+ class: classConst,
+ kind: kindInteger,
+ parent: nil,
+ children: nil,
+ position: posLeaf,
+ priority: priValue,
+ evalFunc: evalConst,
+ }
+}
+
+// -------- float const term
+func newFloatTerm(tk *Token) *term {
+ return &term{
+ tk: *tk,
+ class: classConst,
+ kind: kindFloat,
+ parent: nil,
+ children: nil,
+ position: posLeaf,
+ priority: priValue,
+ evalFunc: evalConst,
+ }
+}
+
+// -------- string const term
+func newStringTerm(tk *Token) *term {
+ return &term{
+ tk: *tk,
+ class: classConst,
+ kind: kindString,
+ parent: nil,
+ children: nil,
+ position: posLeaf,
+ priority: priValue,
+ evalFunc: evalConst,
+ }
+}
+
+// -------- eval func
+func evalConst(ctx exprContext, self *term) (v any, err error) {
+ v = self.tk.Value
+ return
+}
+
+// init
+func init() {
+ registerTermConstructor(SymString, newStringTerm)
+ registerTermConstructor(SymInteger, newIntegerTerm)
+ registerTermConstructor(SymFloat, newFloatTerm)
+ registerTermConstructor(SymBool, newBoolTerm)
+}
diff --git a/operand-func.go b/operand-func.go
new file mode 100644
index 0000000..ef54bee
--- /dev/null
+++ b/operand-func.go
@@ -0,0 +1,33 @@
+// operand-func.go
+package expr
+
+// -------- function term
+func newFuncTerm(tk *Token, args []*term) *term {
+ return &term{
+ tk: *tk,
+ class: classVar,
+ kind: kindUnknown,
+ parent: nil,
+ children: args,
+ position: posLeaf,
+ priority: priValue,
+ evalFunc: evalFunc,
+ }
+}
+
+// -------- eval func
+func evalFunc(ctx exprContext, self *term) (v any, err error) {
+ name, _ := self.tk.Value.(string)
+ params := make([]any, len(self.children))
+ for i, tree := range self.children {
+ var param any
+ if param, err = tree.compute(ctx); err != nil {
+ break
+ }
+ params[i] = param
+ }
+ if err == nil {
+ v, err = ctx.Call(name, params)
+ }
+ return
+}
diff --git a/operand-var.go b/operand-var.go
new file mode 100644
index 0000000..abc484d
--- /dev/null
+++ b/operand-var.go
@@ -0,0 +1,32 @@
+// operand-var.go
+package expr
+
+import "fmt"
+
+// -------- variable term
+func newVarTerm(tk *Token) *term {
+ return &term{
+ tk: *tk,
+ class: classVar,
+ kind: kindUnknown,
+ parent: nil,
+ children: nil,
+ position: posLeaf,
+ priority: priValue,
+ evalFunc: evalVar,
+ }
+}
+
+// -------- eval func
+func evalVar(ctx exprContext, self *term) (v any, err error) {
+ var exists bool
+ if v, exists = ctx.GetValue(self.tk.source); !exists {
+ err = fmt.Errorf("undefined variable %q", self.tk.source)
+ }
+ return
+}
+
+// init
+func init() {
+ registerTermConstructor(SymIdentifier, newVarTerm)
+}
diff --git a/operator-bool.go b/operator-bool.go
new file mode 100644
index 0000000..3ab08b7
--- /dev/null
+++ b/operator-bool.go
@@ -0,0 +1,106 @@
+// operator-bool.go
+package expr
+
+//-------- NOT term
+
+func newNotTerm(tk *Token) (inst *term) {
+ return &term{
+ tk: *tk,
+ class: classOperator,
+ kind: kindBool,
+ children: make([]*term, 0, 1),
+ position: posPrefix,
+ priority: priNot,
+ evalFunc: evalNot,
+ }
+}
+
+func evalNot(ctx exprContext, self *term) (v any, err error) {
+ var rightValue any
+
+ if rightValue, err = self.evalPrefix(ctx); err != nil {
+ return
+ }
+
+ if b, ok := toBool(rightValue); ok {
+ v = !b
+ } else {
+ err = self.errIncompatibleType(rightValue)
+ }
+ return
+}
+
+//-------- AND term
+
+func newAndTerm(tk *Token) (inst *term) {
+ return &term{
+ tk: *tk,
+ class: classOperator,
+ kind: kindBool,
+ children: make([]*term, 0, 2),
+ position: posInfix,
+ priority: priAnd,
+ evalFunc: evalAnd,
+ }
+}
+
+func evalAnd(ctx exprContext, self *term) (v any, err error) {
+ var leftValue, rightValue any
+ var leftBool, rightBool bool
+ var lok, rok bool
+
+ if leftValue, rightValue, err = self.evalInfix(ctx); err != nil {
+ return
+ }
+
+ leftBool, lok = toBool(leftValue)
+ rightBool, rok = toBool(rightValue)
+
+ if lok && rok {
+ v = leftBool && rightBool
+ } else {
+ err = self.errIncompatibleTypes(leftValue, rightValue)
+ }
+ return
+}
+
+//-------- OR term
+
+func newOrTerm(tk *Token) (inst *term) {
+ return &term{
+ tk: *tk,
+ class: classOperator,
+ kind: kindBool,
+ children: make([]*term, 0, 2),
+ position: posInfix,
+ priority: priOr,
+ evalFunc: evalOr,
+ }
+}
+
+func evalOr(ctx exprContext, self *term) (v any, err error) {
+ var leftValue, rightValue any
+ var leftBool, rightBool bool
+ var lok, rok bool
+
+ if leftValue, rightValue, err = self.evalInfix(ctx); err != nil {
+ return
+ }
+
+ leftBool, lok = toBool(leftValue)
+ rightBool, rok = toBool(rightValue)
+
+ if lok && rok {
+ v = leftBool || rightBool
+ } else {
+ err = self.errIncompatibleTypes(leftValue, rightValue)
+ }
+ return
+}
+
+// init
+func init() {
+ registerTermConstructor(SymNot, newNotTerm)
+ registerTermConstructor(SymAnd, newAndTerm)
+ registerTermConstructor(SymOr, newOrTerm)
+}
diff --git a/operator-fact.go b/operator-fact.go
new file mode 100644
index 0000000..4c0050b
--- /dev/null
+++ b/operator-fact.go
@@ -0,0 +1,46 @@
+// operator-fact.go
+package expr
+
+import "fmt"
+
+//-------- fact term
+
+func newFactTerm(tk *Token) (inst *term) {
+ return &term{
+ tk: *tk,
+ class: classOperator,
+ kind: kindInteger,
+ children: make([]*term, 0, 1),
+ position: posPostfix,
+ priority: priFact,
+ evalFunc: evalFact,
+ }
+}
+
+func evalFact(ctx exprContext, self *term) (v any, err error) {
+ var leftValue any
+
+ if leftValue, err = self.evalPrefix(ctx); err != nil {
+ return
+ }
+
+ if isInteger(leftValue) {
+ if i, _ := leftValue.(int64); i >= 0 {
+ f := int64(1)
+ for k := int64(1); k <= i; k++ {
+ f *= k
+ }
+ v = f
+ } else {
+ err = fmt.Errorf("factorial of a negative integer (%d) is not allowed", i)
+ }
+ } else {
+ err = self.errIncompatibleType(leftValue)
+ }
+ return
+}
+
+// init
+func init() {
+ registerTermConstructor(SymExclamation, newFactTerm)
+}
diff --git a/operator-prod.go b/operator-prod.go
new file mode 100644
index 0000000..1502972
--- /dev/null
+++ b/operator-prod.go
@@ -0,0 +1,95 @@
+// operator-prod.go
+package expr
+
+import (
+ "errors"
+ "strings"
+)
+
+//-------- multiply term
+
+func newMultiplyTerm(tk *Token) (inst *term) {
+ return &term{
+ tk: *tk,
+ class: classOperator,
+ kind: kindUnknown,
+ children: make([]*term, 0, 2),
+ position: posInfix,
+ priority: priProduct,
+ evalFunc: evalMultiply,
+ }
+}
+
+func evalMultiply(ctx exprContext, self *term) (v any, err error) {
+ var leftValue, rightValue any
+
+ if leftValue, rightValue, err = self.evalInfix(ctx); err != nil {
+ return
+ }
+
+ if isString(leftValue) && isInteger(rightValue) {
+ s, _ := leftValue.(string)
+ n, _ := rightValue.(int64)
+ v = strings.Repeat(s, int(n))
+ } else if isNumber(leftValue) && isNumber(rightValue) {
+ if isFloat(leftValue) || isFloat(rightValue) {
+ v = numAsFloat(leftValue) * numAsFloat(rightValue)
+ } else {
+ leftInt, _ := leftValue.(int64)
+ rightInt, _ := rightValue.(int64)
+ v = leftInt * rightInt
+ }
+ } else {
+ err = self.errIncompatibleTypes(leftValue, rightValue)
+ }
+ return
+}
+
+//-------- divide term
+
+func newDivideTerm(tk *Token) (inst *term) {
+ return &term{
+ tk: *tk,
+ class: classOperator,
+ kind: kindUnknown,
+ children: make([]*term, 0, 2),
+ position: posInfix,
+ priority: priProduct,
+ evalFunc: evalDivide,
+ }
+}
+
+func evalDivide(ctx exprContext, self *term) (v any, err error) {
+ var leftValue, rightValue any
+
+ if leftValue, rightValue, err = self.evalInfix(ctx); err != nil {
+ return
+ }
+
+ if isNumber(leftValue) && isNumber(rightValue) {
+ if isFloat(leftValue) || isFloat(rightValue) {
+ d := numAsFloat(rightValue)
+ if d == 0.0 {
+ err = errors.New("division by zero")
+ } else {
+ v = numAsFloat(leftValue) / d
+ }
+ } else {
+ leftInt, _ := leftValue.(int64)
+ if rightInt, _ := rightValue.(int64); rightInt == 0 {
+ err = errors.New("division by zero")
+ } else {
+ v = leftInt / rightInt
+ }
+ }
+ } else {
+ err = self.errIncompatibleTypes(leftValue, rightValue)
+ }
+ return
+}
+
+// init
+func init() {
+ registerTermConstructor(SymStar, newMultiplyTerm)
+ registerTermConstructor(SymSlash, newDivideTerm)
+}
diff --git a/operator-rel.go b/operator-rel.go
new file mode 100644
index 0000000..6e3d249
--- /dev/null
+++ b/operator-rel.go
@@ -0,0 +1,220 @@
+// operator-rel.go
+package expr
+
+//-------- equal term
+
+func newEqualTerm(tk *Token) (inst *term) {
+ return &term{
+ tk: *tk,
+ class: classOperator,
+ kind: kindBool,
+ children: make([]*term, 0, 2),
+ position: posInfix,
+ priority: priRelational,
+ evalFunc: evalEqual,
+ }
+}
+
+func evalEqual(ctx exprContext, self *term) (v any, err error) {
+ var leftValue, rightValue any
+
+ if leftValue, rightValue, err = self.evalInfix(ctx); err != nil {
+ return
+ }
+
+ if isNumber(leftValue) && isNumber(rightValue) {
+ if isInteger(leftValue) && isInteger(rightValue) {
+ li, _ := leftValue.(int64)
+ ri, _ := rightValue.(int64)
+ v = li == ri
+ } else {
+ v = numAsFloat(leftValue) == numAsFloat(rightValue)
+ }
+ } else if isString(leftValue) && isString(rightValue) {
+ ls, _ := leftValue.(string)
+ rs, _ := rightValue.(string)
+ v = ls == rs
+ } else {
+ err = self.errIncompatibleTypes(leftValue, rightValue)
+ }
+ return
+}
+
+//-------- not equal term
+
+func newNotEqualTerm(tk *Token) (inst *term) {
+ return &term{
+ tk: *tk,
+ class: classOperator,
+ kind: kindBool,
+ children: make([]*term, 0, 2),
+ position: posInfix,
+ priority: priRelational,
+ evalFunc: evalNotEqual,
+ }
+}
+
+func evalNotEqual(ctx exprContext, self *term) (v any, err error) {
+ if v, err = evalEqual(ctx, self); err == nil {
+ b, _ := toBool(v)
+ v = !b
+ }
+ return
+}
+
+// func evalNotEqual(ctx exprContext, self *term) (v any, err error) {
+// var leftValue, rightValue any
+
+// if leftValue, rightValue, err = self.evalInfix(ctx); err != nil {
+// return
+// }
+
+// if isNumber(leftValue) && isNumber(rightValue) {
+// if isInteger(leftValue) && isInteger(rightValue) {
+// li, _ := leftValue.(int64)
+// ri, _ := rightValue.(int64)
+// v = li != ri
+// } else {
+// v = numAsFloat(leftValue) != numAsFloat(rightValue)
+// }
+// } else if isString(leftValue) && isString(rightValue) {
+// ls, _ := leftValue.(string)
+// rs, _ := rightValue.(string)
+// v = ls != rs
+// } else {
+// err = self.errIncompatibleTypes(leftValue, rightValue)
+// }
+// return
+// }
+
+//-------- less term
+
+func newLessTerm(tk *Token) (inst *term) {
+ return &term{
+ tk: *tk,
+ class: classOperator,
+ kind: kindBool,
+ children: make([]*term, 0, 2),
+ position: posInfix,
+ priority: priRelational,
+ evalFunc: evalLess,
+ }
+}
+
+func evalLess(ctx exprContext, self *term) (v any, err error) {
+ var leftValue, rightValue any
+
+ if leftValue, rightValue, err = self.evalInfix(ctx); err != nil {
+ return
+ }
+
+ if isNumber(leftValue) && isNumber(rightValue) {
+ if isInteger(leftValue) && isInteger(rightValue) {
+ li, _ := leftValue.(int64)
+ ri, _ := rightValue.(int64)
+ v = li < ri
+ } else {
+ v = numAsFloat(leftValue) < numAsFloat(rightValue)
+ }
+ } else if isString(leftValue) && isString(rightValue) {
+ ls, _ := leftValue.(string)
+ rs, _ := rightValue.(string)
+ v = ls < rs
+ } else {
+ err = self.errIncompatibleTypes(leftValue, rightValue)
+ }
+ return
+}
+
+//-------- less or equal term
+
+func newLessEqualTerm(tk *Token) (inst *term) {
+ return &term{
+ tk: *tk,
+ class: classOperator,
+ kind: kindUnknown,
+ children: make([]*term, 0, 2),
+ position: posInfix,
+ priority: priRelational,
+ evalFunc: evalLessEqual,
+ }
+}
+
+func evalLessEqual(ctx exprContext, self *term) (v any, err error) {
+ var leftValue, rightValue any
+
+ if leftValue, rightValue, err = self.evalInfix(ctx); err != nil {
+ return
+ }
+
+ if isNumber(leftValue) && isNumber(rightValue) {
+ if isInteger(leftValue) && isInteger(rightValue) {
+ li, _ := leftValue.(int64)
+ ri, _ := rightValue.(int64)
+ v = li <= ri
+ } else {
+ v = numAsFloat(leftValue) <= numAsFloat(rightValue)
+ }
+ } else if isString(leftValue) && isString(rightValue) {
+ ls, _ := leftValue.(string)
+ rs, _ := rightValue.(string)
+ v = ls <= rs
+ } else {
+ err = self.errIncompatibleTypes(leftValue, rightValue)
+ }
+ return
+}
+
+//-------- greater term
+
+func newGreaterTerm(tk *Token) (inst *term) {
+ return &term{
+ tk: *tk,
+ class: classOperator,
+ kind: kindBool,
+ children: make([]*term, 0, 2),
+ position: posInfix,
+ priority: priRelational,
+ evalFunc: evalGreater,
+ }
+}
+
+func evalGreater(ctx exprContext, self *term) (v any, err error) {
+ if v, err = evalLessEqual(ctx, self); err == nil {
+ b, _ := toBool(v)
+ v = !b
+ }
+ return
+}
+
+//-------- greater or equal term
+
+func newGreaterEqualTerm(tk *Token) (inst *term) {
+ return &term{
+ tk: *tk,
+ class: classOperator,
+ kind: kindBool,
+ children: make([]*term, 0, 2),
+ position: posInfix,
+ priority: priRelational,
+ evalFunc: evalGreaterEqual,
+ }
+}
+
+func evalGreaterEqual(ctx exprContext, self *term) (v any, err error) {
+ if v, err = evalLess(ctx, self); err == nil {
+ b, _ := toBool(v)
+ v = !b
+ }
+ return
+}
+
+// init
+func init() {
+ registerTermConstructor(SymDoubleEqual, newEqualTerm)
+ registerTermConstructor(SymNotEqual, newNotEqualTerm)
+ registerTermConstructor(SymLess, newLessTerm)
+ registerTermConstructor(SymLessOrEqual, newLessEqualTerm)
+ registerTermConstructor(SymGreater, newGreaterTerm)
+ registerTermConstructor(SymGreaterOrEqual, newGreaterEqualTerm)
+}
diff --git a/operator-sign.go b/operator-sign.go
new file mode 100644
index 0000000..bea8f14
--- /dev/null
+++ b/operator-sign.go
@@ -0,0 +1,61 @@
+// operator-sign.go
+package expr
+
+//-------- plus sign term
+
+func newPlusSignTerm(tk *Token) (inst *term) {
+ return &term{
+ tk: *tk,
+ class: classOperator,
+ kind: kindUnknown,
+ children: make([]*term, 0, 1),
+ position: posPrefix,
+ priority: priSign,
+ evalFunc: evalSign,
+ }
+}
+
+func newMinusSignTerm(tk *Token) (inst *term) {
+ return &term{
+ tk: *tk,
+ class: classOperator,
+ kind: kindUnknown,
+ children: make([]*term, 0, 1),
+ position: posPrefix,
+ priority: priSign,
+ evalFunc: evalSign,
+ }
+}
+
+func evalSign(ctx exprContext, self *term) (v any, err error) {
+ var rightValue any
+
+ if rightValue, err = self.evalPrefix(ctx); err != nil {
+ return
+ }
+
+ if isFloat(rightValue) {
+ if self.tk.Sym == SymChangeSign {
+ f, _ := rightValue.(float64)
+ v = -f
+ } else {
+ v = rightValue
+ }
+ } else if isInteger(rightValue) {
+ if self.tk.Sym == SymChangeSign {
+ i, _ := rightValue.(int64)
+ v = -i
+ } else {
+ v = rightValue
+ }
+ } else {
+ err = self.errIncompatibleType(rightValue)
+ }
+ return
+}
+
+// init
+func init() {
+ registerTermConstructor(SymUnchangeSign, newPlusSignTerm)
+ registerTermConstructor(SymChangeSign, newMinusSignTerm)
+}
diff --git a/operator-sum.go b/operator-sum.go
new file mode 100644
index 0000000..a4cb947
--- /dev/null
+++ b/operator-sum.go
@@ -0,0 +1,84 @@
+// operator-sum.go
+package expr
+
+import (
+ "fmt"
+)
+
+//-------- plus term
+
+func newPlusTerm(tk *Token) (inst *term) {
+ return &term{
+ tk: *tk,
+ class: classOperator,
+ kind: kindUnknown,
+ children: make([]*term, 0, 2),
+ position: posInfix,
+ priority: priSum,
+ evalFunc: evalPlus,
+ }
+}
+
+func evalPlus(ctx exprContext, self *term) (v any, err error) {
+ var leftValue, rightValue any
+
+ if leftValue, rightValue, err = self.evalInfix(ctx); err != nil {
+ return
+ }
+
+ if (isString(leftValue) && isNumberString(rightValue)) || (isString(rightValue) && isNumberString(leftValue)) {
+ v = fmt.Sprintf("%v%v", leftValue, rightValue)
+ } else if isNumber(leftValue) && isNumber(rightValue) {
+ if isFloat(leftValue) || isFloat(rightValue) {
+ v = numAsFloat(leftValue) + numAsFloat(rightValue)
+ } else {
+ leftInt, _ := leftValue.(int64)
+ rightInt, _ := rightValue.(int64)
+ v = leftInt + rightInt
+ }
+ } else {
+ err = self.errIncompatibleTypes(leftValue, rightValue)
+ }
+ return
+}
+
+//-------- minus term
+
+func newMinusTerm(tk *Token) (inst *term) {
+ return &term{
+ tk: *tk,
+ class: classOperator,
+ kind: kindUnknown,
+ children: make([]*term, 0, 2),
+ position: posInfix,
+ priority: priSum,
+ evalFunc: evalMinus,
+ }
+}
+
+func evalMinus(ctx exprContext, self *term) (v any, err error) {
+ var leftValue, rightValue any
+
+ if leftValue, rightValue, err = self.evalInfix(ctx); err != nil {
+ return
+ }
+
+ if isNumber(leftValue) && isNumber(rightValue) {
+ if isFloat(leftValue) || isFloat(rightValue) {
+ v = numAsFloat(leftValue) - numAsFloat(rightValue)
+ } else {
+ leftInt, _ := leftValue.(int64)
+ rightInt, _ := rightValue.(int64)
+ v = leftInt - rightInt
+ }
+ } else {
+ err = self.errIncompatibleTypes(leftValue, rightValue)
+ }
+ return
+}
+
+// init
+func init() {
+ registerTermConstructor(SymPlus, newPlusTerm)
+ registerTermConstructor(SymMinus, newMinusTerm)
+}
diff --git a/parser.go b/parser.go
new file mode 100644
index 0000000..6f8d0e0
--- /dev/null
+++ b/parser.go
@@ -0,0 +1,76 @@
+// parser.go
+package expr
+
+import (
+ "fmt"
+)
+
+//-------- parser
+
+type parser struct {
+ ctx exprContext
+}
+
+func NewParser(ctx exprContext) (p *parser) {
+ p = &parser{
+ ctx: ctx,
+ }
+ return p
+}
+
+func (self *parser) parse(scanner *scanner, termSymbols ...Symbol) (tree *ast, err error) {
+ tree = NewAst()
+ firstToken := true
+ for tk := scanner.Next(); err == nil && tk != nil && !tk.IsTerm(termSymbols); tk = scanner.Next() {
+ if tk.Sym == SymComment {
+ continue
+ }
+
+ //fmt.Println("Token:", tk)
+ if firstToken && (tk.Sym == SymMinus || tk.Sym == SymPlus) {
+ if tk.Sym == SymMinus {
+ tk.Sym = SymChangeSign
+ } else {
+ tk.Sym = SymUnchangeSign
+ }
+ }
+ firstToken = false
+ if tk.Sym == SymOpenRound {
+ var subTree *ast
+ if subTree, err = self.parse(scanner, SymClosedRound); err == nil {
+ subTree.root.priority = priValue
+ tree.addTerm(subTree.root)
+ }
+ } else if tk.Sym == SymFunction {
+ name, _ := tk.Value.(string)
+ funcObj := self.ctx.GetFuncInfo(name)
+ if funcObj == nil {
+ err = fmt.Errorf("unknown function %q", name)
+ break
+ }
+ maxArgs := funcObj.MaxArgs()
+ if maxArgs < 0 {
+ maxArgs = funcObj.MinArgs() + 10
+ }
+ args := make([]*term, 0, maxArgs)
+ lastSym := SymUnknown
+ for lastSym != SymClosedRound && lastSym != SymEos {
+ var subTree *ast
+ if subTree, err = self.parse(scanner, SymComma, SymClosedRound); err == nil {
+ args = append(args, subTree.root)
+ } else {
+ break
+ }
+ lastSym = scanner.Previous().Sym
+ }
+ if err == nil {
+ // TODO Check arguments
+ t := newFuncTerm(tk, args)
+ tree.addTerm(t)
+ }
+ } else {
+ err = tree.addToken(tk)
+ }
+ }
+ return
+}
diff --git a/parser_test.go b/parser_test.go
new file mode 100644
index 0000000..8a7a27b
--- /dev/null
+++ b/parser_test.go
@@ -0,0 +1,215 @@
+// parser_test.go
+package expr
+
+import (
+ "errors"
+ "fmt"
+ "strings"
+ "testing"
+)
+
+type testContext struct {
+ store map[string]any
+}
+
+func newTestContext() *testContext {
+ return &testContext{
+ store: make(map[string]any),
+ }
+}
+
+func (ctx *testContext) GetValue(varName string) (v any, exists bool) {
+ v, exists = ctx.store[varName]
+ return
+}
+
+func (ctx *testContext) SetValue(varName string, value any) {
+ ctx.store[varName] = value
+}
+
+func (ctx *testContext) GetFuncInfo(name string) (f exprFunc) {
+ if name == "ADD" {
+ f = &testAddFunc{}
+ }
+ return
+}
+
+func (ctx *testContext) Call(name string, args []any) (result any, err error) {
+ if name == "ADD" {
+ funcObj := &testAddFunc{}
+ result, err = funcObj.Invoke(ctx, args)
+ } else {
+ err = fmt.Errorf("unknown function %q", name)
+ }
+ return
+}
+
+type testAddFunc struct {
+}
+
+func (self *testAddFunc) Name() string {
+ return "ADD"
+}
+
+func (self *testAddFunc) MinArgs() int {
+ return 1
+}
+
+func (self *testAddFunc) MaxArgs() int {
+ return -1
+}
+
+func (self *testAddFunc) Invoke(ctx exprContext, args []any) (result any, err error) {
+ var sumAsFloat = false
+ var floatSum float64 = 0.0
+ var intSum int64 = 0
+
+ for i, v := range args {
+ if !isNumber(v) {
+ err = fmt.Errorf("add(): param nr %d has wrong type %T, number expected", i+1, v)
+ break
+ }
+
+ if !sumAsFloat && isFloat(v) {
+ sumAsFloat = true
+ floatSum = float64(intSum)
+ }
+ if sumAsFloat {
+ floatSum += numAsFloat(v)
+ } else {
+ iv, _ := v.(int64)
+ intSum += iv
+ }
+ }
+ if err == nil {
+ if sumAsFloat {
+ result = floatSum
+ } else {
+ result = intSum
+ }
+ }
+ return
+}
+
+func TestParser(t *testing.T) {
+ type inputType struct {
+ source string
+ wantResult any
+ wantErr error
+ }
+
+ ctx := newTestContext()
+ ctx.SetValue("var1", int64(123))
+ ctx.SetValue("var2", "abc")
+
+ inputs := []inputType{
+ {`123`, int64(123), nil},
+ {`1.`, float64(1.0), nil},
+ {`1.E-2`, float64(0.01), nil},
+ {`1E2`, float64(100), nil},
+ /* 1 */ {`1+/*5*/2`, int64(3), nil},
+ /* 2 */ {`3 == 4`, false, nil},
+ /* 3 */ {`3 != 4`, true, nil},
+ /* 4 */ {`4 == (3-1)*(10/5)`, true, nil},
+ /* 5 */ {`3 <= 4`, true, nil},
+ /* 6 */ {`3 < 4`, true, nil},
+ /* 7 */ {`4 < 3`, false, nil},
+ /* 8 */ {`1+5 < 4`, false, nil},
+ /* 9 */ {`3 > 4`, false, nil},
+ /* 10 */ {`4 >= 4`, true, nil},
+ /* 11 */ {`4 > 3`, true, nil},
+ /* 12 */ {`1+5 > 4`, true, nil},
+ /* 13 */ {`true`, true, nil},
+ /* 14 */ {`not true`, false, nil},
+ /* 15 */ {`true and false`, false, nil},
+ /* 16 */ {`true or false`, true, nil},
+ /* 17 */ {`"uno" + "due"`, `unodue`, nil},
+ /* 18 */ {`"uno" + 2`, `uno2`, nil},
+ /* 19 */ {`"uno" + (2+1)`, `uno3`, nil},
+ /* 20 */ {`"uno" * (2+1)`, `unounouno`, nil},
+ /* 21 */ {"1", int64(1), nil},
+ /* 22 */ {"1.5", float64(1.5), nil},
+ /* 23 */ {"1.5*2", float64(3.0), nil},
+ /* 24 */ {"+1", int64(1), nil},
+ /* 25 */ {"-1", int64(-1), nil},
+ /* 26 */ {"435 + 105 * 2 - 1", int64(644), nil},
+ /* 27 */ {"200 / 2 - 1", int64(99), nil},
+ /* 28 */ {"(1+1)", int64(2), nil},
+ /* 29 */ {"-(1+1)", int64(-2), nil},
+ /* 30 */ {"-(-2+1)", int64(1), nil},
+ /* 31 */ {"(1+1)*5", int64(10), nil},
+ /* 32 */ {"200 / (1+1) - 1", int64(99), nil},
+ /* 33 */ {`add(1,2,3)`, int64(6), nil},
+ /* 34 */ {`mul(1,2,3)`, nil, errors.New(`unknown function "MUL"`)},
+ /* 35 */ {`add(1+4,3+2,5*(3-2))`, int64(15), nil},
+ /* 36 */ {`add(add(1+4),3+2,5*(3-2))`, int64(15), nil},
+ /* 37 */ {`add(add(1+4),/*3+2,*/5*(3-2))`, int64(10), nil},
+ /* 38 */ {`1+(1+(1+(1)+1)+1)+1`, int64(7), nil},
+ /* 39 */ {`(((1)))`, int64(1), nil},
+ /* 40 */ {`"uno_" + var2`, `uno_abc`, nil},
+ /* 41 */ {`0 || 0.0 && "hello"`, false, nil},
+ /* 42 */ {`"s" + true`, nil, errors.New(`left operand 's' [string] is not compatible with right operand 'true' [bool] with respect to operator "+"`)},
+ /* 43 */ {`+false`, nil, errors.New(`prefix/postfix operator "+" do not support operand 'false' [bool]`)},
+ /* 44 */ {`false // very simple expression`, false, nil},
+ /* 45 */ {`1 + // Missing right operator`, nil, errors.New(`infix operator "+" requires two operands, got 1`)},
+ /* 46 */ {"", nil, errors.New(`empty expression`)},
+ /* 47 */ {"4!", int64(24), nil},
+ /* 48 */ {"(-4)!", nil, errors.New(`factorial of a negative integer (-4) is not allowed`)},
+ /* 49 */ {"-4!", int64(-24), nil},
+ /* 50 */ {"1.5 < 7", true, nil},
+ /* 51 */ {"1.5 > 7", false, nil},
+ /* 52 */ {"1.5 <= 7", true, nil},
+ /* 53 */ {"1.5 >= 7", false, nil},
+ /* 54 */ {"1.5 != 7", true, nil},
+ /* 55 */ {"1.5 == 7", false, nil},
+ /* 56 */ {`"1.5" < "7"`, true, nil},
+ /* 57 */ {`"1.5" > "7"`, false, nil},
+ /* 58 */ {`"1.5" == "7"`, false, nil},
+ /* 59 */ {`"1.5" != "7"`, true, nil},
+ /* 60 */ {"1.5 < ", nil, errors.New(`infix operator "<" requires two operands, got 1`)},
+ /* 61 */ {"1.5 > ", nil, errors.New(`infix operator ">" requires two operands, got 1`)},
+ /* 62 */ {"1.5 <= ", nil, errors.New(`infix operator "<=" requires two operands, got 1`)},
+ /* 63 */ {"1.5 >= ", nil, errors.New(`infix operator ">=" requires two operands, got 1`)},
+ /* 64 */ {"1.5 != ", nil, errors.New(`infix operator "!=" requires two operands, got 1`)},
+ /* 65 */ {"1.5 == ", nil, errors.New(`infix operator "==" requires two operands, got 1`)},
+ /* 66 */ {`"1.5" < `, nil, errors.New(`infix operator "<" requires two operands, got 1`)},
+ /* 67 */ {`"1.5" > `, nil, errors.New(`infix operator ">" requires two operands, got 1`)},
+ /* 68 */ {`"1.5" == `, nil, errors.New(`infix operator "==" requires two operands, got 1`)},
+ /* 69 */ {`"1.5" != `, nil, errors.New(`infix operator "!=" requires two operands, got 1`)},
+ /* 70 */ {"+1.5", float64(1.5), nil},
+ /* 71 */ {"+", nil, errors.New(`prefix operator "+" requires one operand`)},
+ /* 72 */ {"4 / 0", nil, errors.New(`division by zero`)},
+ /* 73 */ {"4.0 / 0", nil, errors.New(`division by zero`)},
+ /* 74 */ {"4.0 / \n2", float64(2.0), nil},
+ }
+
+ parser := NewParser(ctx)
+ for i, input := range inputs {
+ var expr *ast
+ var gotResult any
+ var gotErr error
+
+ if input.wantErr == nil {
+ t.Log(fmt.Sprintf("[+]Test nr %2d -- %q", i+1, input.source))
+ } else {
+ t.Log(fmt.Sprintf("[-]Test nr %2d -- %q", i+1, input.source))
+ }
+
+ r := strings.NewReader(input.source)
+ scanner := NewScanner(r, DefaultTranslations())
+
+ if expr, gotErr = parser.parse(scanner); gotErr == nil {
+ gotResult, gotErr = expr.eval(ctx)
+ }
+
+ if gotResult != input.wantResult {
+ t.Errorf("%d: %q -> result = %v [%T], want %v [%T]", i+1, input.source, gotResult, gotResult, input.wantResult, input.wantResult)
+ }
+
+ if gotErr != input.wantErr {
+ if input.wantErr == nil || gotErr == nil || (gotErr.Error() != input.wantErr.Error()) {
+ t.Errorf("%d: %q -> err = <%v>, want <%v>", i+1, input.source, gotErr, input.wantErr)
+ }
+ }
+ }
+}
diff --git a/scanner.go b/scanner.go
new file mode 100644
index 0000000..3ed1c1b
--- /dev/null
+++ b/scanner.go
@@ -0,0 +1,552 @@
+// expr project scanner.go
+package expr
+
+import (
+ "bufio"
+ "errors"
+ "io"
+ "strconv"
+ "strings"
+)
+
+type scanner struct {
+ current *Token
+ prev *Token
+ stream *bufio.Reader
+ row int
+ column int
+ translations map[Symbol]Symbol
+}
+
+func NewScanner(s io.Reader, translations map[Symbol]Symbol) (inst *scanner) {
+ inst = &scanner{
+ stream: bufio.NewReader(s),
+ row: 1,
+ column: 1,
+ translations: translations,
+ }
+ inst.current = inst.fetchNextToken()
+ return inst
+}
+
+func DefaultTranslations() map[Symbol]Symbol {
+ return map[Symbol]Symbol{
+ SymDoubleAmpersand: SymAnd,
+ SymKwAnd: SymAnd,
+ SymDoubleVertBar: SymOr,
+ SymKwOr: SymOr,
+ SymKwNot: SymNot,
+ SymLessGreater: SymNotEqual,
+ }
+}
+
+// func (self *scanner) Current() *Token {
+// return self.current
+// }
+
+func (self *scanner) readChar() (ch byte, err error) {
+ if ch, err = self.stream.ReadByte(); err == nil {
+ if ch == '\n' {
+ self.row++
+ self.column = 0
+ } else {
+ self.column++
+ }
+ }
+ return
+}
+
+func (self *scanner) unreadChar() (err error) {
+ if err = self.stream.UnreadByte(); err == nil {
+ if self.column--; self.column == 0 {
+ if self.row--; self.row == 0 {
+ err = errors.New("unread beyond the stream boundary")
+ } else {
+ self.column = 1
+ }
+ }
+ }
+ return
+}
+
+func (self *scanner) Previous() *Token {
+ return self.prev
+}
+
+func (self *scanner) Next() (tk *Token) {
+ self.prev = self.current
+ tk = self.current
+ self.current = self.fetchNextToken()
+ return tk
+}
+
+func (self *scanner) fetchNextToken() (tk *Token) {
+ if err := self.skipBlanks(); err != nil {
+ return self.makeErrorToken(err)
+ }
+
+ escape := false
+ for {
+ ch, _ := self.readChar()
+ switch ch {
+ case '+':
+ if next, _ := self.peek(); next == '+' {
+ tk = self.moveOn(SymDoublePlus, ch, next)
+ } else if next == '=' {
+ tk = self.moveOn(SymPlusEqual, ch, next)
+ } else {
+ tk = self.makeToken(SymPlus, ch)
+ }
+ case '-':
+ if next, _ := self.peek(); next == '-' {
+ tk = self.moveOn(SymDoubleMinus, ch, next)
+ } else if next == '=' {
+ tk = self.moveOn(SymMinusEqual, ch, next)
+ } else {
+ tk = self.makeToken(SymMinus, ch)
+ }
+ case '*':
+ if next, _ := self.peek(); next == '*' {
+ tk = self.moveOn(SymDoubleStar, ch, next)
+ // } else if next == '/' {
+ // tk = self.moveOn(SymClosedComment, ch, next)
+ } else {
+ tk = self.makeToken(SymStar, ch)
+ }
+ case '/':
+ if next, _ := self.peek(); next == '*' {
+ self.readChar()
+ tk = self.fetchBlockComment()
+ } else if next == '/' {
+ self.readChar()
+ tk = self.fetchOnLineComment()
+ } else {
+ tk = self.makeToken(SymSlash, ch)
+ }
+ case '\\':
+ if escape {
+ tk = self.makeToken(SymBackSlash, ch)
+ escape = false
+ } else {
+ escape = true
+ }
+ case '|':
+ if next, _ := self.peek(); next == '|' {
+ tk = self.moveOn(SymDoubleVertBar, ch, next)
+ } else {
+ tk = self.makeToken(SymVertBar, ch)
+ }
+ case ',':
+ tk = self.makeToken(SymComma, ch)
+ case ':':
+ tk = self.makeToken(SymColon, ch)
+ case ';':
+ tk = self.makeToken(SymSemiColon, ch)
+ case '.':
+ if next, _ := self.peek(); next >= '0' && next <= '9' {
+ tk = self.parseNumber(ch)
+ } else {
+ tk = self.makeToken(SymDot, ch)
+ }
+ case '\'':
+ tk = self.makeToken(SymQuote, ch)
+ case '"':
+ if escape {
+ tk = self.makeToken(SymDoubleQuote, ch)
+ escape = false
+ } else {
+ tk = self.fetchString()
+ }
+ case '`':
+ tk = self.makeToken(SymBackTick, ch)
+ case '!':
+ if next, _ := self.peek(); next == '=' {
+ tk = self.moveOn(SymNotEqual, ch, next)
+ } else {
+ tk = self.makeToken(SymExclamation, ch)
+ }
+ case '?':
+ tk = self.makeToken(SymQuestion, ch)
+ case '&':
+ if next, _ := self.peek(); next == '&' {
+ tk = self.moveOn(SymDoubleAmpersand, ch, next)
+ } else {
+ tk = self.makeToken(SymAmpersand, ch)
+ }
+ case '%':
+ tk = self.makeToken(SymPercent, ch)
+ case '#':
+ tk = self.makeToken(SymHash, ch)
+ case '@':
+ tk = self.makeToken(SymAt, ch)
+ case '_':
+ tk = self.makeToken(SymUndescore, ch)
+ case '=':
+ if next, _ := self.peek(); next == '=' {
+ tk = self.moveOn(SymDoubleEqual, ch, next)
+ } else {
+ tk = self.makeToken(SymEqual, ch)
+ }
+ case '<':
+ if next, _ := self.peek(); next == '=' {
+ tk = self.moveOn(SymLessOrEqual, ch, next)
+ } else if next == '>' {
+ tk = self.moveOn(SymLessGreater, ch, next)
+ } else {
+ tk = self.makeToken(SymLess, ch)
+ }
+ case '>':
+ if next, _ := self.peek(); next == '=' {
+ tk = self.moveOn(SymGreaterOrEqual, ch, next)
+ } else {
+ tk = self.makeToken(SymGreater, ch)
+ }
+ case '$':
+ tk = self.makeToken(SymDollar, ch)
+ case '(':
+ tk = self.makeToken(SymOpenRound, ch)
+ case ')':
+ tk = self.makeToken(SymClosedRound, ch)
+ case '[':
+ tk = self.makeToken(SymOpenSquare, ch)
+ case ']':
+ tk = self.makeToken(SymClosedSquare, ch)
+ case '{':
+ tk = self.makeToken(SymOpenBrace, ch)
+ case '}':
+ tk = self.makeToken(SymClosedBrace, ch)
+ case 0:
+ if escape {
+ tk = self.makeErrorToken(errors.New("incomplete escape sequence"))
+ }
+ escape = false
+ default:
+ if ch == '_' || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') {
+ tk = self.fetchIdentifier(ch)
+ } else if ch >= '0' && ch <= '9' {
+ tk = self.parseNumber(ch)
+ }
+ }
+ if !escape {
+ break
+ }
+ }
+ return
+}
+
+func (self *scanner) sync(err error) error {
+ if err == nil {
+ err = self.unreadChar()
+ }
+ return err
+}
+
+func (self *scanner) parseNumber(firstCh byte) (tk *Token) {
+ var err error
+ var ch byte
+ var sym Symbol = SymInteger
+ var value any
+ var sb strings.Builder
+
+ for ch = firstCh; err == nil && (ch >= '0' && ch <= '9'); ch, err = self.readChar() {
+ sb.WriteByte(ch)
+ }
+ if ch == '.' {
+ sym = SymFloat
+ sb.WriteByte(ch)
+ ch, err = self.readChar()
+ if ch >= '0' && ch <= '9' {
+ for ; err == nil && (ch >= '0' && ch <= '9'); ch, err = self.readChar() {
+ sb.WriteByte(ch)
+ }
+ }
+ }
+ if ch == 'e' || ch == 'E' {
+ sym = SymFloat
+ sb.WriteByte(ch)
+ if ch, err = self.readChar(); err == nil {
+ if ch == '+' || ch == '-' {
+ sb.WriteByte(ch)
+ ch, err = self.readChar()
+ }
+ if ch >= '0' && ch <= '9' {
+ for ; err == nil && (ch >= '0' && ch <= '9'); ch, err = self.readChar() {
+ sb.WriteByte(ch)
+ }
+ //err = self.sync(err)
+ } else {
+ err = errors.New("expected integer exponent")
+ }
+ }
+ // } else {
+ // err = self.sync(err)
+ }
+
+ if err != nil && err != io.EOF {
+ tk = self.makeErrorToken(err)
+ } else {
+ err = self.sync(err)
+ txt := sb.String()
+ if sym == SymFloat {
+ value, err = strconv.ParseFloat(txt, 64)
+ } else if strings.HasPrefix(txt, "0x") {
+ value, err = strconv.ParseInt(txt, 16, 64)
+ } else if strings.HasPrefix(txt, "0o") {
+ value, err = strconv.ParseInt(txt, 8, 64)
+ } else if strings.HasPrefix(txt, "0b") {
+ value, err = strconv.ParseInt(txt, 2, 64)
+ } else {
+ value, err = strconv.ParseInt(txt, 10, 64)
+ }
+ if err == nil {
+ tk = self.makeValueToken(sym, txt, value)
+ } else {
+ tk = self.makeErrorToken(err)
+ }
+ }
+ return
+}
+
+func (self *scanner) fetchIdentifier(firstCh byte) (tk *Token) {
+ var err error
+ var sb strings.Builder
+ for ch := firstCh; err == nil && (ch == '_' || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9')); ch, err = self.readChar() {
+ sb.WriteByte(ch)
+ }
+
+ if err != nil && err != io.EOF {
+ tk = self.makeErrorToken(err)
+ } else if err = self.sync(err); err != nil && err != io.EOF {
+ tk = self.makeErrorToken(err)
+ } else {
+ txt := sb.String()
+ uptxt := strings.ToUpper(txt)
+ if sym, ok := keywords[uptxt]; ok {
+ tk = self.makeKeywordToken(sym, uptxt)
+ } else if uptxt == `TRUE` {
+ tk = self.makeValueToken(SymBool, txt, true)
+ } else if uptxt == `FALSE` {
+ tk = self.makeValueToken(SymBool, txt, false)
+ } else if ch, _ := self.peek(); ch == '(' {
+ self.readChar()
+ tk = self.makeValueToken(SymFunction, txt+"(", uptxt)
+ } else {
+ tk = self.makeValueToken(SymIdentifier, txt, txt)
+ }
+ }
+
+ // if err != nil && err != io.EOF {
+ // tk = self.makeErrorToken(err)
+ // } else if err = self.sync(err); err != nil && err != io.EOF {
+ // tk = self.makeErrorToken(err)
+ // } else {
+ // txt := sb.String()
+ // uptxt := strings.ToUpper(txt)
+ // if sym, ok := keywords[uptxt]; ok {
+ // tk = self.makeValueToken(sym, txt, "")
+ // } else {
+ // tk = self.makeValueToken(SymIdentifier, txt, txt)
+ // }
+ // }
+ return
+}
+
+func (self *scanner) fetchBlockComment() *Token {
+ return self.fetchUntil(SymComment, false, '*', '/')
+}
+
+func (self *scanner) fetchOnLineComment() *Token {
+ return self.fetchUntil(SymComment, true, '\n')
+}
+
+// func (self *scanner) fetchUntil(sym Symbol, allowEos bool, endings ...byte) (tk *Token) {
+// var err error
+// var ch byte
+// var sb strings.Builder
+// var value string
+// end := string(endings)
+// endReached := false
+// for ch, err = self.readChar(); err == nil && !endReached; {
+// sb.WriteByte(ch)
+// if sb.Len() >= len(end) && strings.HasSuffix(sb.String(), end) {
+// value = sb.String()[0 : sb.Len()-len(end)]
+// endReached = true
+// } else {
+// ch, err = self.readChar()
+// }
+// }
+// if !endReached && allowEos {
+// value = sb.String()
+// endReached = true
+// }
+
+// if endReached {
+// tk = self.makeValueToken(sym, "", value)
+// } else {
+// tk = self.makeErrorToken(err)
+// }
+// return
+// }
+
+// func (self *scanner) fetchUntil(sym Symbol, allowEos bool, endings ...byte) (tk *Token) {
+// var err error
+// var ch byte
+// var sb strings.Builder
+// var value string
+// end := make([]byte, len(endings))
+// length := 0
+// endReached := false
+// for ch, err = self.readChar(); err == nil && !endReached; {
+// sb.WriteByte(ch)
+// if length == len(endings) {
+// for i := 0; i < length-1; i++ {
+// end[i] = end[i+1]
+// }
+// length--
+// }
+// end[length] = ch
+// length++
+// if bytes.Equal(endings, end) {
+// value = sb.String()[0 : sb.Len()-len(end)]
+// endReached = true
+// } else {
+// ch, err = self.readChar()
+// }
+// }
+// if !endReached && allowEos {
+// value = sb.String()
+// endReached = true
+// }
+
+// if endReached {
+// tk = self.makeValueToken(sym, "", value)
+// } else {
+// tk = self.makeErrorToken(err)
+// }
+// return
+// }
+
+func (self *scanner) fetchUntil(sym Symbol, allowEos bool, endings ...byte) (tk *Token) {
+ var err error
+ var ch byte
+ var sb strings.Builder
+ var value string
+ ring := NewByteSlider(len(endings))
+ endReached := false
+ for ch, err = self.readChar(); err == nil && !endReached; {
+ sb.WriteByte(ch)
+ ring.PushEnd(ch)
+ if ring.Equal(endings) {
+ value = sb.String()[0 : sb.Len()-len(endings)]
+ endReached = true
+ } else {
+ ch, err = self.readChar()
+ }
+ }
+ if !endReached && allowEos {
+ value = sb.String()
+ endReached = true
+ }
+
+ if endReached {
+ tk = self.makeValueToken(sym, "", value)
+ } else {
+ tk = self.makeErrorToken(err)
+ }
+ return
+}
+
+func (self *scanner) fetchString() (tk *Token) {
+ var err error
+ var ch, prev byte
+ var sb strings.Builder
+ for ch, err = self.readChar(); err == nil; ch, err = self.readChar() {
+ if prev == '\\' {
+ switch ch {
+ case '"':
+ sb.WriteByte('"')
+ case 'n':
+ sb.WriteByte('\n')
+ case 'r':
+ sb.WriteByte('\r')
+ case 't':
+ sb.WriteByte('\t')
+ case '\\':
+ sb.WriteByte('\\')
+ default:
+ sb.WriteByte(ch)
+ }
+ prev = 0
+ } else if ch == '"' {
+ break
+ } else {
+ prev = ch
+ if ch != '\\' {
+ sb.WriteByte(ch)
+ }
+ }
+ }
+ if err != nil {
+ if err == io.EOF {
+ tk = self.makeErrorToken(errors.New("missing string termination \""))
+ } else {
+ tk = self.makeErrorToken(err)
+ }
+ } else {
+ txt := sb.String()
+ tk = self.makeValueToken(SymString, `"`+txt+`"`, txt)
+ }
+ return
+}
+
+func (self *scanner) peek() (next byte, err error) {
+ var one []byte
+ if one, err = self.stream.Peek(1); err == nil {
+ next = one[0]
+ }
+ return
+}
+
+func (self *scanner) skipBlanks() (err error) {
+ var one []byte
+ for one, err = self.stream.Peek(1); err == nil && one[0] <= 32; one, err = self.stream.Peek(1) {
+ self.readChar()
+ }
+ return
+}
+
+func (self *scanner) translate(sym Symbol) Symbol {
+ if self.translations != nil {
+ if translatedSym, ok := self.translations[sym]; ok {
+ return translatedSym
+ }
+ }
+ return sym
+}
+
+func (self *scanner) moveOn(sym Symbol, chars ...byte) (tk *Token) {
+ tk = NewToken(self.translate(sym), string(chars))
+ for i := 1; i < len(chars); i++ {
+ self.readChar()
+ }
+ return
+}
+
+func (self *scanner) makeToken(sym Symbol, chars ...byte) (tk *Token) {
+ tk = NewToken(self.translate(sym), string(chars))
+ return
+}
+
+func (self *scanner) makeKeywordToken(sym Symbol, upperCaseKeyword string) (tk *Token) {
+ tk = NewToken(self.translate(sym), upperCaseKeyword)
+ return
+}
+
+func (self *scanner) makeValueToken(sym Symbol, source string, value any) (tk *Token) {
+ tk = NewValueToken(self.translate(sym), source, value)
+ return
+}
+
+func (self *scanner) makeErrorToken(err error) *Token {
+ return NewErrorToken(self.row, self.column, err)
+}
diff --git a/scanner_test.go b/scanner_test.go
new file mode 100644
index 0000000..8a732be
--- /dev/null
+++ b/scanner_test.go
@@ -0,0 +1,91 @@
+// scanner_test.go
+package expr
+
+import (
+ "errors"
+ "fmt"
+ "strings"
+ "testing"
+)
+
+func TestScanner(t *testing.T) {
+ type inputType struct {
+ source string
+ wantSym Symbol
+ wantValue any
+ wantErr error
+ }
+
+ inputs := []inputType{
+ /* 1 */ {`123`, SymInteger, int64(123), nil},
+ /* 2 */ {"=", SymEqual, nil, nil},
+ /* 3 */ {`--`, SymDoubleMinus, nil, nil},
+ /* 4 */ {`++`, SymDoublePlus, nil, nil},
+ /* 5 */ {`**`, SymDoubleStar, nil, nil},
+ /* 6 */ {`&&`, SymDoubleAmpersand, nil, nil},
+ /* 7 */ {`||`, SymDoubleVertBar, nil, nil},
+ /* 8 */ {`NOT`, SymKwNot, nil, nil},
+ /* 9 */ {`AND`, SymKwAnd, nil, nil},
+ /* 10 */ {`or`, SymKwOr, nil, nil},
+ /* 11 */ {`+=`, SymPlusEqual, nil, nil},
+ /* 12 */ {`-=`, SymMinusEqual, nil, nil},
+ /* 13 */ {`|`, SymVertBar, nil, nil},
+ /* 14 */ {`:`, SymColon, nil, nil},
+ /* 15 */ {`;`, SymSemiColon, nil, nil},
+ /* 16 */ {`.`, SymDot, nil, nil},
+ /* 17 */ {`.5`, SymFloat, float64(0.5), nil},
+ /* 18 */ {`\\`, SymBackSlash, nil, nil},
+ /* 19 */ {"`", SymBackTick, nil, nil},
+ /* 20 */ {"?", SymQuestion, nil, nil},
+ /* 21 */ {"&", SymAmpersand, nil, nil},
+ /* 22 */ {"@", SymAt, nil, nil},
+ /* 23 */ {`#`, SymHash, nil, nil},
+ /* 24 */ {`%`, SymPercent, nil, nil},
+ /* 25 */ {`'`, SymQuote, nil, nil},
+ /* 26 */ {`\"`, SymDoubleQuote, nil, nil},
+ /* 27 */ {`_`, SymUndescore, nil, nil},
+ /* 28 */ {`<>`, SymLessGreater, nil, nil},
+ /* 29 */ {`[`, SymOpenSquare, nil, nil},
+ /* 30 */ {`]`, SymClosedSquare, nil, nil},
+ /* 31 */ {`{`, SymOpenBrace, nil, nil},
+ /* 32 */ {`}`, SymClosedBrace, nil, nil},
+ /* 33 */ {`(`, SymOpenRound, nil, nil},
+ /* 34 */ {`)`, SymClosedRound, nil, nil},
+ /* 35 */ {`1E+2`, SymFloat, float64(100), nil},
+ /* 36 */ {`1E+x`, SymError, errors.New("expected integer exponent"), nil},
+ /* 37 */ {`$`, SymDollar, nil, nil},
+ /* 38 */ {`\`, SymError, errors.New("incomplete escape sequence"), nil},
+ /* 39 */ {`"string"`, SymString, "string", nil},
+ /* 39 */ {`identifier`, SymIdentifier, "identifier", nil},
+ }
+
+ for i, input := range inputs {
+
+ if input.wantErr == nil {
+ t.Log(fmt.Sprintf("[+]Test nr %2d -- %q", i+1, input.source))
+ } else {
+ t.Log(fmt.Sprintf("[-]Test nr %2d -- %q", i+1, input.source))
+ }
+
+ r := strings.NewReader(input.source)
+ scanner := NewScanner(r, nil)
+
+ if tk := scanner.Next(); tk == nil {
+ t.Errorf("%d: %q -> got = (nil), want %v (value %v [%T)", i+1, input.source, input.wantSym, input.wantValue, input.wantValue)
+ } else if tk.Sym != input.wantSym || tk.Value != input.wantValue {
+ if tk.Sym == SymError && input.wantSym == tk.Sym {
+ if tkErr, tkOk := tk.Value.(error); tkOk {
+ if inputErr, inputOk := input.wantValue.(error); inputOk {
+ if tkErr.Error() != inputErr.Error() {
+ t.Errorf("%d: %q -> got-error = %v, want-error: %v", i+1,
+ input.source, tk.Value, input.wantValue)
+ }
+ }
+ }
+ } else {
+ t.Errorf("%d: %q -> got = %v (value=%v [%T), want %v (value=%v [%T)", i+1,
+ input.source, tk.Sym, tk.Value, tk.Value, input.wantSym, input.wantValue, input.wantValue)
+ }
+ }
+ }
+}
diff --git a/symbol.go b/symbol.go
new file mode 100644
index 0000000..f023b17
--- /dev/null
+++ b/symbol.go
@@ -0,0 +1,82 @@
+// Symbol.go
+package expr
+
+type Symbol int16
+
+const (
+ SymUnknown Symbol = iota - 1 // -1: Unknown symbol
+ SymNone // 0: Null value for variable of type symbol
+ SymError // 1: Error reading from stream
+ SymEos // 2: End of stream
+ SymMinus // 3: '-'
+ SymMinusEqual // 4: '-='
+ SymDoubleMinus // 5: '--'
+ SymPlus // 6: '+'
+ SymPlusEqual // 7: '+='
+ SymDoublePlus // 8: '++'
+ SymStar // 9: '*'
+ SymDoubleStar // 10: '**'
+ SymSlash // 11: '/'
+ SymBackSlash // 12: '\'
+ SymVertBar // 13: '|'
+ SymDoubleVertBar // 14: '||'
+ SymComma // 15: ','
+ SymColon // 16: ':'
+ SymSemiColon // 17: ';'
+ SymDot // 18: '.'
+ SymQuote // 19: '\''
+ SymDoubleQuote // 20: '"'
+ SymBackTick // 0: '`'
+ SymExclamation // 0: '!'
+ SymQuestion // 0: '?'
+ SymAmpersand // 0: '&&'
+ SymDoubleAmpersand // 0: '&&'
+ SymPercent // 0: '%'
+ SymAt // 0: '@'
+ SymUndescore // 0: '_'
+ SymEqual // 0: '='
+ SymDoubleEqual // 0: '=='
+ SymLess // 0: '<'
+ SymLessOrEqual // 0: '<='
+ SymGreater // 0: '>'
+ SymGreaterOrEqual // 0: '>='
+ SymLessGreater // 0: '<>'
+ SymNotEqual // 0: '!='
+ SymDollar // 0: '$'
+ SymHash // 0: '#'
+ SymOpenRound // 0: '('
+ SymClosedRound // 0: ')'
+ SymOpenSquare // 0: '['
+ SymClosedSquare // 0: ']'
+ SymOpenBrace // 0: '{'
+ SymClosedBrace // 0: '}'
+ SymChangeSign
+ SymUnchangeSign
+ SymIdentifier
+ SymBool
+ SymInteger
+ SymFloat
+ SymString
+ SymKwAnd
+ SymKwNot
+ SymKwOr
+ SymOr
+ SymAnd
+ SymNot
+ SymComment
+ SymFunction
+ // SymOpenComment // 0: '/*'
+ // SymClosedComment // 0: '*/'
+ // SymOneLineComment // 0: '//'
+)
+
+var keywords map[string]Symbol
+
+func init() {
+ //keywords = make(map[string]Symbol)
+ keywords = map[string]Symbol{
+ "AND": SymKwAnd,
+ "OR": SymKwOr,
+ "NOT": SymKwNot,
+ }
+}
diff --git a/term-constuctor-registry.go b/term-constuctor-registry.go
new file mode 100644
index 0000000..d7c699a
--- /dev/null
+++ b/term-constuctor-registry.go
@@ -0,0 +1,25 @@
+// op-registry.go
+package expr
+
+const initialRegistryCapacity = 10
+
+type termContructor func(tk *Token) *term
+
+var constructorRegistry map[Symbol]termContructor = nil
+
+func registerTermConstructor(sym Symbol, constructor termContructor) {
+ if constructorRegistry == nil {
+ constructorRegistry = make(map[Symbol]termContructor, initialRegistryCapacity)
+ }
+ constructorRegistry[sym] = constructor
+}
+
+func newTerm(tk *Token, parent *term) (inst *term) {
+ if constructorRegistry != nil {
+ if construct, exists := constructorRegistry[tk.Sym]; exists {
+ inst = construct(tk)
+ inst.setParent(parent)
+ }
+ }
+ return
+}
diff --git a/term.go b/term.go
new file mode 100644
index 0000000..8e8fa3d
--- /dev/null
+++ b/term.go
@@ -0,0 +1,229 @@
+// term.go
+package expr
+
+import (
+ "fmt"
+ "strings"
+)
+
+type termKind uint16
+
+const (
+ kindUnknown termKind = iota
+ kindBool
+ kindInteger
+ kindFloat
+ kindString
+ kindIdentifier
+)
+
+type termClass uint16
+
+const (
+ classNull termClass = iota
+ classConst
+ classVar
+ classFunc
+ classOperator
+)
+
+type termPriority uint32
+
+const (
+ priNone termPriority = iota
+ priOr
+ priAnd
+ priNot
+ priRelational
+ priSum
+ priProduct
+ priSign
+ priFact
+ priValue
+)
+
+type termPosition uint8
+
+const (
+ posLeaf termPosition = iota
+ posInfix
+ posPrefix
+ posPostfix
+)
+
+type evalFuncType func(ctx exprContext, self *term) (v any, err error)
+
+// type iterm interface {
+// getClass() termClass
+// getKind() termKind
+// compute(ctx exprContext) (v any, err error)
+// // isOperator() bool
+// // isOperand() bool
+// source() string
+// setParent(parentNode term)
+// }
+
+type term struct {
+ tk Token
+ class termClass
+ kind termKind
+ parent *term
+ children []*term
+ position termPosition // operator position: infix, prefix, suffix
+ priority termPriority // operator priority: higher value means higher priority
+ evalFunc evalFuncType
+}
+
+func (self *term) String() string {
+ var sb strings.Builder
+ self.toString(&sb)
+ return sb.String()
+}
+
+func (self *term) toString(sb *strings.Builder) {
+ if self.position == posLeaf {
+ sb.WriteString(self.tk.String())
+ } else {
+ sb.WriteByte('[')
+ sb.WriteString(self.tk.String())
+ if self.children != nil {
+ sb.WriteByte('(')
+ for i, c := range self.children {
+ if i > 0 {
+ sb.WriteByte(' ')
+ }
+ c.toString(sb)
+ }
+ sb.WriteByte(')')
+ }
+ sb.WriteByte(']')
+ }
+}
+
+func (self *term) getChildrenCount() (count int) {
+ if self.position == posLeaf || self.children == nil {
+ count = 0
+ } else {
+ count = len(self.children)
+ }
+ return
+}
+
+func (self *term) getRoom() (room int) {
+ switch self.position {
+ case posLeaf:
+ room = 0
+ case posInfix:
+ room = 2
+ case posPostfix, posPrefix:
+ room = 1
+ default:
+ panic("Invalid node position")
+ }
+ return
+}
+
+func (self *term) isComplete() bool {
+ return self.getChildrenCount() == self.getRoom()
+}
+
+func (self *term) removeLastChild() (child *term) {
+ if self.children != nil {
+ child = self.children[len(self.children)-1]
+ self.children = self.children[0 : len(self.children)-1]
+ } else {
+ panic("Can't get last child")
+ }
+ return
+}
+
+func (self *term) isLeaf() bool {
+ return self.position == posLeaf
+}
+
+// func (self *term) getKind() termKind {
+// return self.kind
+// }
+
+// func (self *term) getClass() termClass {
+// return self.class
+// }
+
+func (self *term) getPriority() termPriority {
+ return self.priority
+}
+
+func (self *term) setParent(parent *term) {
+ self.parent = parent
+ if parent != nil && len(parent.children) < cap(parent.children) {
+ parent.children = append(parent.children, self)
+ }
+}
+
+// func (self *term) isOperand() bool {
+// return self.getClass() != classOperator
+// }
+
+// func (self *term) isOperator() bool {
+// return self.getClass() == classOperator
+// }
+
+func (self *term) source() string {
+ return self.tk.source
+}
+
+func (self *term) compute(ctx exprContext) (v any, err error) {
+ if self.evalFunc == nil {
+ err = fmt.Errorf("undfined eval-func for %v term type", self.kind)
+ } else {
+ v, err = self.evalFunc(ctx, self)
+ }
+ return
+}
+
+func (self *term) errIncompatibleTypes(leftValue, rightValue any) error {
+ return fmt.Errorf(
+ "left operand '%v' [%T] is not compatible with right operand '%v' [%T] with respect to operator %q",
+ leftValue, leftValue,
+ rightValue, rightValue,
+ self.source())
+}
+
+func (self *term) errIncompatibleType(value any) error {
+ return fmt.Errorf(
+ "prefix/postfix operator %q do not support operand '%v' [%T]", self.source(), value, value)
+}
+
+func (self *term) checkOperands() (err error) {
+ switch self.position {
+ case posInfix:
+ if self.children == nil || len(self.children) != 2 || self.children[0] == nil || self.children[1] == nil {
+ err = fmt.Errorf("infix operator %q requires two operands, got %d", self.source(), self.getChildrenCount())
+ }
+ case posPrefix:
+ if self.children == nil || len(self.children) != 1 || self.children[0] == nil {
+ err = fmt.Errorf("prefix operator %q requires one operand", self.tk.String())
+ }
+ case posPostfix:
+ if self.children == nil || len(self.children) != 1 || self.children[0] == nil {
+ err = fmt.Errorf("postfix operator %q requires one operand", self.tk.String())
+ }
+ }
+ return
+}
+
+func (self *term) evalInfix(ctx exprContext) (leftValue, rightValue any, err error) {
+ if err = self.checkOperands(); err == nil {
+ if leftValue, err = self.children[0].compute(ctx); err == nil {
+ rightValue, err = self.children[1].compute(ctx)
+ }
+ }
+ return
+}
+
+func (self *term) evalPrefix(ctx exprContext) (rightValue any, err error) {
+ if err = self.checkOperands(); err == nil {
+ rightValue, err = self.children[0].compute(ctx)
+ }
+ return
+}
diff --git a/term_test.go b/term_test.go
new file mode 100644
index 0000000..1363a27
--- /dev/null
+++ b/term_test.go
@@ -0,0 +1,41 @@
+// term_test.go
+package expr
+
+import (
+ "fmt"
+ "testing"
+)
+
+func TestString(t *testing.T) {
+ tk1 := NewValueToken(SymInteger, "100", 100)
+ tk2 := NewToken(SymPlus, "+")
+ tk3 := NewValueToken(SymInteger, "50", 500)
+
+ tree := NewAst()
+ if gotErr := tree.addTokens(tk1, tk2, tk3); gotErr == nil {
+ fmt.Println("Tree:", tree)
+ } else {
+ t.Errorf("err: got <%v>, want <nil>", gotErr)
+ }
+}
+
+func TestGetRoom(t *testing.T) {
+ tk1 := NewValueToken(SymInteger, "100", 100)
+
+ tree := NewAst()
+ if gotErr := tree.addTokens(tk1); gotErr == nil {
+ fmt.Println("Tree-root room:", tree.root.getRoom())
+ } else {
+ t.Errorf("err: got <%v>, want <nil>", gotErr)
+ }
+}
+func TestGetChildrenCount(t *testing.T) {
+ tk1 := NewValueToken(SymInteger, "100", 100)
+
+ tree := NewAst()
+ if gotErr := tree.addTokens(tk1); gotErr == nil {
+ fmt.Println("Tree-root children count:", tree.root.getChildrenCount())
+ } else {
+ t.Errorf("err: got <%v>, want <nil>", gotErr)
+ }
+}
diff --git a/token.go b/token.go
new file mode 100644
index 0000000..dacddb0
--- /dev/null
+++ b/token.go
@@ -0,0 +1,55 @@
+// token.go
+package expr
+
+import (
+ "fmt"
+ "io"
+ "slices"
+)
+
+type Token struct {
+ Sym Symbol
+ source string
+ Value any
+}
+
+func (tk *Token) DevString() string {
+ if tk.Value != nil {
+ return fmt.Sprintf("[%d]%q{%v}", tk.Sym, tk.source, tk.Value)
+ }
+ return fmt.Sprintf("[%d]%q{}", tk.Sym, tk.source)
+}
+
+func (tk *Token) String() string {
+ if tk.Value != nil {
+ return fmt.Sprintf("%#v", tk.Value)
+ }
+ return fmt.Sprintf("%s", tk.source)
+}
+
+func NewToken(sym Symbol, source string) *Token {
+ return &Token{Sym: sym, source: source}
+}
+
+func NewValueToken(sym Symbol, source string, value any) *Token {
+ return &Token{Sym: sym, source: source, Value: value}
+}
+
+func NewErrorToken(row, col int, err error) *Token {
+ if err == io.EOF {
+ return NewToken(SymEos, "")
+ }
+ return NewValueToken(SymError, fmt.Sprintf("[%d:%d]", row, col), err)
+}
+
+func (tk *Token) IsEos() bool {
+ return tk.Sym == SymEos
+}
+
+func (tk *Token) IsError() bool {
+ return tk.Sym == SymError
+}
+
+func (tk *Token) IsTerm(termSymbols []Symbol) bool {
+ return tk.IsEos() || tk.IsError() || (termSymbols != nil && slices.Index(termSymbols, tk.Sym) >= 0)
+}
diff --git a/token_test.go b/token_test.go
new file mode 100644
index 0000000..35bd5d1
--- /dev/null
+++ b/token_test.go
@@ -0,0 +1,15 @@
+// token_test.go
+package expr
+
+import (
+ "fmt"
+ "testing"
+)
+
+func TestDevString(t *testing.T) {
+ tk1 := NewValueToken(SymInteger, "100", 100)
+ tk2 := NewToken(SymPlus, "+")
+
+ fmt.Println("Token '100':", tk1.DevString())
+ fmt.Println("Token '+':", tk2.DevString())
+}
diff --git a/utils.go b/utils.go
new file mode 100644
index 0000000..673c221
--- /dev/null
+++ b/utils.go
@@ -0,0 +1,51 @@
+// utils.go
+package expr
+
+func isString(v any) (ok bool) {
+ _, ok = v.(string)
+ return ok
+}
+
+func isInteger(v any) (ok bool) {
+ _, ok = v.(int64)
+ return ok
+}
+
+func isFloat(v any) (ok bool) {
+ _, ok = v.(float64)
+ return ok
+}
+
+func isNumber(v any) (ok bool) {
+ return isFloat(v) || isInteger(v)
+}
+
+func isNumberString(v any) (ok bool) {
+ return isString(v) || isNumber(v)
+}
+
+func numAsFloat(v any) (f float64) {
+ var ok bool
+ if f, ok = v.(float64); !ok {
+ i, _ := v.(int64)
+ f = float64(i)
+ }
+ return
+}
+
+func toBool(v any) (b bool, ok bool) {
+ ok = true
+ switch x := v.(type) {
+ case string:
+ b = len(x) > 0
+ case float64:
+ b = x != 0.0
+ case int64:
+ b = x != 0
+ case bool:
+ b = x
+ default:
+ ok = false
+ }
+ return
+}