new operator 'map' ans general variable access by ${}

iter-factory.go

@@ -16,9 +16,7 @@ func NewIterator(value any) (it Iterator, err error) {
 		it, err = NewDictIterator(v, nil)
 	case []any:
 		it = NewArrayIterator(v)
-	case *ListIterator:
-		it = v
-	case *DictIterator:
+	case Iterator:
 		it = v
 	default:
 		it = NewArrayIterator([]any{value})

list-type.go

@@ -26,9 +26,9 @@ func newList(listAny []any) (list *ListType) {
 func NewList(listAny []any) (list *ListType) {
 	if listAny != nil {
 		ls := make(ListType, len(listAny))
-//		for i, item := range listAny {
-//			ls[i] = item
-//		}
+		//		for i, item := range listAny {
+		//			ls[i] = item
+		//		}
 		copy(ls, listAny)
 		list = &ls
 	}
@@ -53,14 +53,14 @@ func ListFromStrings(stringList []string) (list *ListType) {
 }
 
 func (ls *ListType) ToString(opt FmtOpt) (s string) {
-        indent := GetFormatIndent(opt)
-        flags := GetFormatFlags(opt)
+	indent := GetFormatIndent(opt)
+	flags := GetFormatFlags(opt)
 
 	var sb strings.Builder
 	sb.WriteByte('[')
 	if len(*ls) > 0 {
-                innerOpt := MakeFormatOptions(flags, indent+1)
-                nest := strings.Repeat("  ", indent+1)
+		innerOpt := MakeFormatOptions(flags, indent+1)
+		nest := strings.Repeat("  ", indent+1)
 
 		if flags&MultiLine != 0 {
 			sb.WriteByte('\n')
@@ -129,6 +129,20 @@ func (ls *ListType) contains(t *ListType) (answer bool) {
 	return
 }
 
+func (ls1 *ListType) Equals(ls2 ListType) (answer bool) {
+	if ls2 != nil && len(*ls1) == len(ls2) {
+		answer = true
+		for index, i1 := range *ls1 {
+			// if i1 != (ls2)[index] {
+			if reflect.DeepEqual(i1, ls2[index]) {
+				answer = false
+				break
+			}
+		}
+	}
+	return
+}
+
 func (list *ListType) indexDeepSameCmp(target any) (index int) {
 	var eq bool
 	var err error
@@ -193,3 +207,6 @@ func (list *ListType) setItem(index int64, value any) (err error) {
 	return
 }
 
+func (list *ListType) appendItem(value any) {
+	*list = append(*list, value)
+}

operator-map.go

@@ -0,0 +1,64 @@
+// Copyright (c) 2024 Celestino Amoroso (celestino.amoroso@gmail.com).
+// All rights reserved.
+
+// operator-map.go
+package expr
+
+import (
+	"fmt"
+	"io"
+)
+
+//-------- map term
+
+func newMapTerm(tk *Token) (inst *term) {
+	return &term{
+		tk:       *tk,
+		children: make([]*term, 0, 2),
+		position: posInfix,
+		priority: priIterOp,
+		evalFunc: evalMap,
+	}
+}
+
+func evalMap(ctx ExprContext, opTerm *term) (v any, err error) {
+	var leftValue, rightValue any
+	var it Iterator
+	var item any
+
+	if err = opTerm.checkOperands(); err != nil {
+		return
+	}
+
+	if leftValue, err = opTerm.children[0].compute(ctx); err != nil {
+		return
+	}
+
+	if it, err = NewIterator(leftValue); err != nil {
+		return nil, fmt.Errorf("left operand of MAP must be an iterable data-source; got %s", TypeName(leftValue))
+	}
+
+	values := newListA()
+	for item, err = it.Next(); err == nil; item, err = it.Next() {
+		ctx.SetVar("_", item)
+		ctx.SetVar("_index", it.Index())
+		ctx.SetVar("_count", it.Count())
+		if rightValue, err = opTerm.children[1].compute(ctx); err == nil {
+			values.appendItem(rightValue)
+		}
+		ctx.DeleteVar("_")
+		if err != nil {
+			break
+		}
+	}
+	if err == io.EOF {
+		err = nil
+	}
+	v = values
+	return
+}
+
+// init
+func init() {
+	registerTermConstructor(SymKwMap, newMapTerm)
+}

scanner.go

@@ -210,14 +210,14 @@ func (scanner *scanner) fetchNextToken() (tk *Token) {
 				tk = scanner.makeToken(SymQuote, ch)
 				escape = false
 			} else {
-				tk = scanner.fetchString(ch)
+				tk = scanner.fetchString(ch, true)
 			}
 		case '"':
 			if escape {
 				tk = scanner.makeToken(SymDoubleQuote, ch)
 				escape = false
 			} else {
-				tk = scanner.fetchString(ch)
+				tk = scanner.fetchString(ch, true)
 			}
 		case '`':
 			tk = scanner.makeToken(SymBackTick, ch)
@@ -315,6 +315,14 @@ func (scanner *scanner) fetchNextToken() (tk *Token) {
 				tk.source += ")"
 			} else if next == '$' {
 				tk = scanner.moveOn(SymDoubleDollar, ch, next)
+			} else if next == '{' {
+				scanner.readChar()
+				if tk = scanner.fetchString('}', false); tk != nil {
+					tk.Sym = SymIdentifier
+				}
+			} else if next == '_' || (next >= 'a' && next <= 'z') || (next >= 'A' && next <= 'Z') {
+				scanner.readChar()
+				tk = scanner.fetchIdentifier(next)
 			} else {
 				tk = scanner.makeToken(SymDollar, ch)
 			}
@@ -590,7 +598,7 @@ func (scanner *scanner) fetchUntil(sym Symbol, allowEos bool, endings ...byte) (
 	return
 }
 
-func (scanner *scanner) fetchString(termCh byte) (tk *Token) {
+func (scanner *scanner) fetchString(termCh byte, addQuote bool) (tk *Token) {
 	var err error
 	var ch, prev byte
 	var sb strings.Builder
@@ -628,7 +636,11 @@ func (scanner *scanner) fetchString(termCh byte) (tk *Token) {
 		}
 	} else {
 		txt := sb.String()
-		tk = scanner.makeValueToken(SymString, `"`+txt+`"`, txt)
+		if addQuote {
+			tk = scanner.makeValueToken(SymString, `"`+txt+`"`, txt)
+		} else {
+			tk = scanner.makeValueToken(SymString, txt, txt)
+		}
 	}
 	return
 }

symbol.go

@@ -119,6 +119,7 @@ const (
 	SymKwPlugin
 	SymKwIn
 	SymKwInclude
+	SymKwMap
 	SymKwNil
 	SymKwUnset
 )
@@ -135,6 +136,7 @@ func init() {
 		"FUNC":    SymKwFunc,
 		"IN":      SymKwIn,
 		"INCLUDE": SymKwInclude,
+		"MAP":     SymKwMap,
 		"NOT":     SymKwNot,
 		"OR":      SymKwOr,
 		"NIL":     SymKwNil,

t_common_test.go

@@ -72,6 +72,7 @@ func doTest(t *testing.T, ctx ExprContext, section string, input *inputType, cou
 	var expr Expr
 	var gotResult any
 	var gotErr error
+	var eq, eqDone bool
 
 	wantErr := getWantedError(input)
 
@@ -90,7 +91,18 @@ func doTest(t *testing.T, ctx ExprContext, section string, input *inputType, cou
 		gotResult, gotErr = expr.Eval(ctx)
 	}
 
-	eq := reflect.DeepEqual(gotResult, input.wantResult)
+	if input.wantResult != nil && gotResult != nil {
+		if ls1, ok := input.wantResult.(*ListType); ok {
+			if ls2, ok := gotResult.(*ListType); ok {
+				eq = ls1.Equals(*ls2)
+				eqDone = true
+			}
+		}
+	}
+
+	if !eqDone {
+		eq = reflect.DeepEqual(gotResult, input.wantResult)
+	}
 
 	if !eq /*gotResult != input.wantResult*/ {
 		t.Errorf("%d: `%s` -> result = %v [%s], want = %v [%s]", count, input.source, gotResult, TypeName(gotResult), input.wantResult, TypeName(input.wantResult))

t_expr_test.go

@@ -43,9 +43,11 @@ func TestExpr(t *testing.T) {
 		it++;
 		it++
 `, int64(1), nil},
+		/*  20 */ {`a=2; ${a}`, int64(2), nil},
+		/*  21 */ {`$_=2; $_`, int64(2), nil},
 	}
 	// t.Setenv("EXPR_PATH", ".")
 
-	// runTestSuiteSpec(t, section, inputs, 18)
+	// runTestSuiteSpec(t, section, inputs, 21)
 	runTestSuite(t, section, inputs)
 }

t_iterator_test.go

@@ -31,8 +31,9 @@ func TestIteratorParser(t *testing.T) {
 		/*  20 */ {`it=$({1:"one",2:"two",3:"three"}, "default", "value"); it++`, "one", nil},
 		/*  21 */ {`it=$({1:"one",2:"two",3:"three"}, "desc", "key"); it++`, int64(3), nil},
 		/*  22 */ {`it=$({1:"one",2:"two",3:"three"}, "asc", "item"); it++`, NewList([]any{int64(1), "one"}), nil},
+		/*  23 */ {`builtin "os.file"; fileReadIterator("test-file.txt") map ${_index}`, NewList([]any{int64(0), int64(1)}), nil},
 	}
 
-	// runTestSuiteSpec(t, section, inputs, 20)
-	runTestSuite(t, section, inputs)
+	runTestSuiteSpec(t, section, inputs, 23)
+	// runTestSuite(t, section, inputs)
 }

t_operator_test.go

@@ -30,13 +30,15 @@ func TestOperator(t *testing.T) {
 		/*  17 */ {`~true`, nil, `[1:2] prefix/postfix operator "~" do not support operand 'true' [bool]`},
 		/*  18 */ {`1^2`, int64(3), nil},
 		/*  19 */ {`3^2`, int64(1), nil},
-		/*  19 */ {`a=1; a^=2`, int64(3), nil},
-		/*  20 */ {`a=1; ++a`, int64(2), nil},
-		/*  21 */ {`a=1; --a`, int64(0), nil},
+		/*  20 */ {`a=1; a^=2`, int64(3), nil},
+		/*  21 */ {`a=1; ++a`, int64(2), nil},
+		/*  22 */ {`a=1; --a`, int64(0), nil},
+		/*  23 */ {`[1,2,3] map var("_")`, []any{1, 2, 3}, nil},
+		/*  24 */ {`[1,2,3] map $_`, newList([]any{1, 2, 3}), nil},
 	}
 
 	// t.Setenv("EXPR_PATH", ".")
 
-	// runTestSuiteSpec(t, section, inputs, 4)
-	runTestSuite(t, section, inputs)
+	runTestSuiteSpec(t, section, inputs, 24)
+	// runTestSuite(t, section, inputs)
 }

term.go

@@ -13,6 +13,7 @@ type termPriority uint32
 const (
 	priNone termPriority = iota
 	priRange
+	priIterOp // map, filter, digest, etc
 	priBut
 	priAssign
 	priInsert