first working implementation of iterators and some fixes to the parser around lists analysis

operand-iterator.go

@@ -0,0 +1,152 @@
+// Copyright (c) 2024 Celestino Amoroso (celestino.amoroso@gmail.com).
+// All rights reserved.
+
+// operand-iterator.go
+package expr
+
+import (
+	"fmt"
+	"io"
+)
+
+// -------- iterator term
+
+const (
+	initName    = "init"
+	nextName    = "next"
+	currentName = "current"
+)
+
+type dataCursor struct {
+	ds          map[any]*term
+	ctx         ExprContext
+	index       int
+	resource    any
+	nextFunc    Functor
+	currentFunc Functor
+}
+
+func (dc *dataCursor) String() string {
+	var s string
+	if item, err := dc.Current(); err == nil {
+		s = fmt.Sprintf("%v", item)
+	} else {
+		s = "(nil)"
+	}
+	return s
+}
+
+func (dc *dataCursor) Current() (item any, err error) { // must return io.EOF at the last item
+	if item, err = dc.currentFunc.Invoke(dc.ctx, currentName, []any{}); err == nil && item == nil {
+		err = io.EOF
+	}
+	return
+}
+
+func (dc *dataCursor) Next() (item any, err error) { // must return io.EOF after the last item
+	if item, err = dc.nextFunc.Invoke(dc.ctx, nextName, []any{}); err == nil {
+		if item == nil {
+			err = io.EOF
+		} else {
+			dc.index++
+		}
+	}
+	return
+}
+
+func (dc *dataCursor) Index() int {
+	return dc.index
+}
+
+func newIteratorTerm(tk *Token, dsTerm *term, args []*term) *term {
+	tk.Sym = SymIterator
+
+	children := make([]*term, 0, 1+len(args))
+	children = append(children, dsTerm)
+	children = append(children, args...)
+	return &term{
+		tk:       *tk,
+		parent:   nil,
+		children: children,
+		position: posLeaf,
+		priority: priValue,
+		evalFunc: evalIterator,
+	}
+}
+
+// -------- eval iterator
+
+func evalTermArray(ctx ExprContext, a []*term) (values []any, err error) {
+	values = make([]any, len(a))
+	for i, t := range a {
+		var value any
+		if value, err = t.compute(ctx); err == nil {
+			values[i] = value
+		} else {
+			break
+		}
+	}
+	return
+}
+
+func getDataSourceDict(ctx ExprContext, self *term) (ds map[string]Functor, err error) {
+	var value any
+	if len(self.children) < 1 || self.children[0] == nil {
+		err = self.Errorf("missing the data-source parameter")
+		return
+	}
+
+	if value, err = self.children[0].compute(ctx); err != nil {
+		return
+	}
+
+	if dictAny, ok := value.(map[any]any); ok {
+		ds = make(map[string]Functor)
+		for _, k := range []string{initName, currentName, nextName} {
+			if item, exists := dictAny[k]; exists && item != nil {
+				if functor, ok := item.(*funcDefFunctor); ok {
+					ds[k] = functor
+				}
+			} else if k != initName {
+				err = fmt.Errorf("the data-source must provide a non-nil %q operator", k)
+				break
+			}
+		}
+	} else {
+		err = self.Errorf("the first param (data-source) of an iterator must be a dict, not a %T", value)
+	}
+	return
+}
+
+func evalIterator(ctx ExprContext, self *term) (v any, err error) {
+	var ds map[string]Functor
+
+	if ds, err = getDataSourceDict(ctx, self); err != nil {
+		return
+	}
+
+	dc := &dataCursor{
+		index: -1,
+		ctx:   ctx.Clone(),
+	}
+
+	if initFunc, exists := ds[initName]; exists && initFunc != nil {
+		var args []any
+		if len(self.children) > 1 {
+			if args, err = evalTermArray(ctx, self.children[1:]); err != nil {
+				return
+			}
+		} else {
+			args = []any{}
+		}
+		if dc.resource, err = initFunc.Invoke(dc.ctx, initName, args); err != nil {
+			return
+		}
+	}
+
+	dc.nextFunc, _ = ds[nextName]
+	dc.currentFunc, _ = ds[currentName]
+	v = dc
+
+	return
+}

operator-iter-value.go

@@ -0,0 +1,38 @@
+// Copyright (c) 2024 Celestino Amoroso (celestino.amoroso@gmail.com).
+// All rights reserved.
+
+// operator-iter-value.go
+package expr
+
+
+//-------- iter value term
+
+func newIterValueTerm(tk *Token) (inst *term) {
+	return &term{
+		tk:       *tk,
+		children: make([]*term, 0, 1),
+		position: posPrefix,
+		priority: priIterValue,
+		evalFunc: evalIterValue,
+	}
+}
+
+func evalIterValue(ctx ExprContext, self *term) (v any, err error) {
+	var leftValue any
+
+	if leftValue, err = self.evalPrefix(ctx); err != nil {
+		return
+	}
+
+	if dc, ok := leftValue.(*dataCursor); ok {
+		v, err = dc.Current()
+	} else {
+		err = self.errIncompatibleType(leftValue)
+	}
+	return
+}
+
+// init
+func init() {
+	registerTermConstructor(SymOpenClosedRound, newIterValueTerm)
+}

parser.go

@@ -58,23 +58,72 @@ func (self *parser) parseFuncCall(scanner *scanner, allowVarRef bool, tk *Token)
 	return
 }
 
+// func (self *parser) parseFuncDef(scanner *scanner) (tree *term, err error) {
+// 	// Example: "add = func(x,y) {x+y}
+// 	var body *ast
+// 	args := make([]*term, 0)
+// 	tk := scanner.Next()
+// 	for tk.Sym != SymClosedRound && tk.Sym != SymEos {
+// 		if tk.Sym == SymIdentifier {
+// 			t := newTerm(tk, nil)
+// 			args = append(args, t)
+// 		} else {
+// 			err = tk.Errorf("invalid param %q, variable identifier expected", tk.source)
+// 			break
+// 		}
+// 		tk = scanner.Next()
+// 	}
+// 	if err == nil && tk.Sym != SymClosedRound {
+// 		err = tk.Errorf("unterminate function params list")
+// 	}
+// 	if err == nil {
+// 		tk = scanner.Next()
+// 		if tk.Sym == SymOpenBrace {
+// 			body, err = self.parseGeneral(scanner, true, true, SymClosedBrace)
+// 		}
+// 	}
+// 	if err == nil {
+// 		// TODO Check arguments
+// 		if scanner.Previous().Sym != SymClosedBrace {
+// 			err = scanner.Previous().Errorf("not properly terminated function body")
+// 		} else {
+// 			tk = scanner.makeValueToken(SymExpression, "", body)
+// 			tree = newFuncDefTerm(tk, args)
+// 		}
+// 	}
+// 	return
+// }
+
 func (self *parser) parseFuncDef(scanner *scanner) (tree *term, err error) {
 	// Example: "add = func(x,y) {x+y}
 	var body *ast
 	args := make([]*term, 0)
-	tk := scanner.Next()
+	lastSym := SymUnknown
-	for tk.Sym != SymClosedRound && tk.Sym != SymEos {
+	itemExpected := false
-		if tk.Sym == SymIdentifier {
+	tk := scanner.Previous()
-			t := newTerm(tk, nil)
+	for lastSym != SymClosedRound && lastSym != SymEos {
-			args = append(args, t)
+		var subTree *ast
+		if subTree, err = self.parseItem(scanner, true, SymComma, SymClosedRound); err == nil {
+			if subTree.root != nil {
+				if subTree.root.symbol() == SymIdentifier {
+					args = append(args, subTree.root)
+				} else {
+					err = tk.Errorf("exptected identifier, got %q", subTree.root)
+				}
+			} else if itemExpected {
+				prev := scanner.Previous()
+				err = prev.Errorf("expected function parameter, got %q", prev)
+				break
+			}
 		} else {
-			err = tk.Errorf("invalid param %q, variable identifier expected", tk.source)
 			break
 		}
-		tk = scanner.Next()
+		lastSym = scanner.Previous().Sym
+		itemExpected = lastSym == SymComma
 	}
-	if err == nil && tk.Sym != SymClosedRound {
+
-		err = tk.Errorf("unterminate function params list")
+	if err == nil && lastSym != SymClosedRound {
+		err = tk.Errorf("unterminated function parameters list")
 	}
 	if err == nil {
 		tk = scanner.Next()
@@ -125,6 +174,51 @@ func (self *parser) parseList(scanner *scanner, allowVarRef bool) (subtree *term
 	return
 }
 
+func (self *parser) parseIterDef(scanner *scanner, allowVarRef bool) (subtree *term, err error) {
+	var ds *term
+	tk := scanner.Previous()
+	args := make([]*term, 0)
+	lastSym := SymUnknown
+	dsExpected := true
+	itemExpected := false
+	for lastSym != SymClosedRound && lastSym != SymEos {
+		var subTree *ast
+		if subTree, err = self.parseItem(scanner, allowVarRef, SymComma, SymClosedRound); err == nil {
+			if subTree.root != nil {
+				if dsExpected {
+					if sym := subTree.root.symbol(); sym == SymDict || sym == SymIdentifier {
+						ds = subTree.root
+					} else {
+						err = subTree.root.Errorf("data-source dictionary expected, got %q", subTree.root.source())
+					}
+					dsExpected = false
+				} else {
+					args = append(args, subTree.root)
+				}
+			} else if itemExpected {
+				prev := scanner.Previous()
+				err = prev.Errorf("expected iterator argument, got %q", prev)
+				break
+			}
+		} else {
+			break
+		}
+		lastSym = scanner.Previous().Sym
+		itemExpected = lastSym == SymComma
+	}
+	if err == nil {
+		// TODO Check arguments
+		if lastSym != SymClosedRound {
+			err = scanner.Previous().Errorf("unterminate iterator param list")
+		} else if ds != nil {
+			subtree = newIteratorTerm(tk, ds, args)
+		} else {
+			tk.Errorf("missing data-source param")
+		}
+	}
+	return
+}
+
 func (self *parser) parseDictKey(scanner *scanner, allowVarRef bool) (key any, err error) {
 	tk := scanner.Next()
 	if tk.Sym == SymError {
@@ -156,9 +250,12 @@ func (self *parser) parseDictionary(scanner *scanner, allowVarRef bool) (subtree
 		var key any
 		if key, err = self.parseDictKey(scanner, allowVarRef); err != nil {
 			break
-		} else if key == nil && itemExpected {
+		} else if key == nil {
 			tk := scanner.Previous()
-			err = tk.Errorf("expected dictionary key, got %q", tk)
+			lastSym = tk.Sym
+			if itemExpected {
+				err = tk.Errorf("expected dictionary key, got %q", tk)
+			}
 			break
 		}
 		if subTree, err = self.parseItem(scanner, allowVarRef, SymComma, SymClosedBrace); err == nil {
@@ -178,7 +275,7 @@ func (self *parser) parseDictionary(scanner *scanner, allowVarRef bool) (subtree
 	if err == nil {
 		// TODO Check arguments
 		if lastSym != SymClosedBrace {
-			err = scanner.Previous().Errorf("unterminate dictionary")
+			err = scanner.Previous().Errorf("unterminated dictionary")
 		} else {
 			subtree = newDictTerm(args)
 		}
@@ -324,6 +421,12 @@ func (self *parser) parseGeneral(scanner *scanner, allowForest bool, allowVarRef
 				err = tree.addTerm(funcDefTerm)
 				currentTerm = funcDefTerm
 			}
+		case SymDollarRound:
+			var iterDefTerm *term
+			if iterDefTerm, err = self.parseIterDef(scanner, allowVarRef); err == nil {
+				err = tree.addTerm(iterDefTerm)
+				currentTerm = iterDefTerm
+			}
 		case SymIdentifier:
 			if tk.source[0] == '@' && !allowVarRef {
 				err = tk.Errorf("variable references are not allowed in top level expressions: %q", tk.source)

scanner.go

@@ -239,9 +239,18 @@ func (self *scanner) fetchNextToken() (tk *Token) {
 				tk = self.makeToken(SymGreater, ch)
 			}
 		case '$':
-			tk = self.makeToken(SymDollar, ch)
+			if next, _ := self.peek(); next == '(' {
+				tk = self.moveOn(SymDollarRound, ch, next)
+				tk.source += ")"
+			} else {
+				tk = self.makeToken(SymDollar, ch)
+			}
 		case '(':
+			if next, _ := self.peek(); next == ')' {
+				tk = self.moveOn(SymOpenClosedRound, ch, next)
+			} else {
 			tk = self.makeToken(SymOpenRound, ch)
+		}
 		case ')':
 			tk = self.makeToken(SymClosedRound, ch)
 		case '[':

symbol.go

@@ -62,6 +62,8 @@ const (
 	SymInsert                            //  51: '>>'
 	SymAppend                            //  52: '<<'
 	SymCaret                             //  53: '^'
+	SymDollarRound                       //  54: '$('
+	SymOpenClosedRound //  55: '()'
 	SymChangeSign
 	SymUnchangeSign
 	SymIdentifier
@@ -69,6 +71,7 @@ const (
 	SymInteger
 	SymFloat
 	SymString
+	SymIterator
 	SymOr
 	SymAnd
 	SymNot