Compare commits

..

16 Commits

Author SHA1 Message Date
camoroso 79889cd8e1 New builtin module 'iterator' 2024-07-14 16:53:32 +02:00
camoroso 234759158c scanner: disabled prefix operator '()' 2024-07-13 18:10:04 +02:00
camoroso 00fda29606 operator-iter-value.go: prefix operator () used to get the current value of an iterator replaced by ^ 2024-07-13 17:17:16 +02:00
camoroso 2a2840bdf2 ListIterator now implements next and current command (e.g it.next) 2024-07-13 17:15:53 +02:00
camoroso 06373f5126 Impemented Typer interface 2024-07-13 17:14:25 +02:00
camoroso e69dad5fb5 Rename priority label priCoalesce as priDefault 2024-07-13 17:13:16 +02:00
camoroso 7459057df9 Expr embeds Typer and ast implements it 2024-07-13 17:11:39 +02:00
camoroso 176969c956 New iterator tests 2024-07-13 16:19:04 +02:00
camoroso cde2efacf1 Test on iterator filter and map 2024-07-13 15:30:04 +02:00
camoroso d7a7b3218c Merge branch 'main' into enhance_iterators 2024-07-13 09:08:16 +02:00
camoroso be3bb12f28 operator-unset.go: fixed function removal 2024-07-13 09:07:33 +02:00
camoroso 032916d4fa New operator unset to delete variables and functions from current context 2024-07-13 09:01:59 +02:00
camoroso f3cc0cc7ad operator-include.go: Fixed inclusion of a list of files.
Now returns the value of the last evaluated expression.
2024-07-13 09:00:53 +02:00
camoroso 905337f963 ExprContext: new functions VarCount(), DeleteVar(), FuncCount(), DeleteFunc() 2024-07-13 08:59:15 +02:00
camoroso 9a95a837f6 data-cursor.go: add item mapping support 2024-07-13 06:44:00 +02:00
camoroso 8547248ea2 data-cursor.go: fixed filter 2024-07-13 00:12:08 +02:00
24 changed files with 388 additions and 51 deletions
+4
View File
@@ -19,6 +19,10 @@ func NewAst() *ast {
return &ast{}
}
func (expr *ast) TypeName() string {
return "Expression"
}
func (expr *ast) ToForest() {
if expr.root != nil {
if expr.forest == nil {
+4
View File
@@ -29,6 +29,10 @@ func newExprFunctor(e Expr, params []ExprFuncParam, ctx ExprContext) *exprFuncto
return &exprFunctor{expr: e, params: params, defCtx: defCtx}
}
func (functor *exprFunctor) TypeName() string {
return "ExprFunctor"
}
func (functor *exprFunctor) GetDefinitionContext() ExprContext {
return functor.defCtx
}
+4
View File
@@ -14,6 +14,10 @@ func NewGolangFunctor(f FuncTemplate) *golangFunctor {
return &golangFunctor{f: f}
}
func (functor *golangFunctor) TypeName() string {
return "GoFunctor"
}
func (functor *golangFunctor) Invoke(ctx ExprContext, name string, args []any) (result any, err error) {
return functor.f(ctx, name, args)
}
+102
View File
@@ -0,0 +1,102 @@
// Copyright (c) 2024 Celestino Amoroso (celestino.amoroso@gmail.com).
// All rights reserved.
// builtin-iterator.go
package expr
import (
"fmt"
"io"
)
const (
iterParamOperator = "operator"
iterParamVars = "vars"
)
func parseRunArgs(localCtx ExprContext, args []any) (it Iterator, op Functor, err error) {
var ok bool
if it, ok = args[0].(Iterator); !ok {
err = fmt.Errorf("paramter %q must be an iterator, passed %v [%s]", ParamIterator, args[0], TypeName(args[0]))
return
}
if len(args) > 1 {
if op, ok = args[1].(Functor); !ok || op == nil {
err = fmt.Errorf("paramter %q must be a function, passed %v [%s]", iterParamOperator, args[1], TypeName(args[1]))
return
}
if len(args) > 2 {
var vars *DictType
if vars, ok = args[2].(*DictType); !ok || vars == nil {
err = fmt.Errorf("paramter %q must be a dictionary, passed %v [%s]", iterParamVars, args[2], TypeName(args[2]))
return
}
for key, value := range *vars {
var varName string
if varName, ok = key.(string); ok {
localCtx.UnsafeSetVar(varName, value)
}
}
}
}
return
}
func runFunc(ctx ExprContext, name string, args []any) (result any, err error) {
var it Iterator
var ok bool
var op Functor
var v any
var usingDefaultOp = false
var params []any
var item any
localCtx := ctx.Clone()
localCtx.UnsafeSetVar("it_status", nil)
if it, op, err = parseRunArgs(localCtx, args); err != nil {
return
} else if op == nil {
op = NewGolangFunctor(printLnFunc)
usingDefaultOp = true
}
for item, err = it.Next(); err == nil; item, err = it.Next() {
if usingDefaultOp {
params = []any{item}
} else {
params = []any{it.Index(), item}
}
if v, err = op.Invoke(localCtx, iterParamOperator, params); err != nil {
break
} else {
var success bool
if success, ok = ToBool(v); !success || !ok {
break
}
}
}
if err == io.EOF {
err = nil
}
if err == nil {
result, _ = localCtx.GetVar("it_status")
}
return
}
func ImportIterFuncs(ctx ExprContext) {
ctx.RegisterFunc("run", NewGolangFunctor(runFunc), TypeAny, []ExprFuncParam{
NewFuncParam(ParamIterator),
NewFuncParamFlag(iterParamOperator, PfOptional),
NewFuncParamFlag(iterParamVars, PfOptional),
})
}
func init() {
RegisterBuiltinModule("iterator", ImportIterFuncs, "Iterator helper functions")
}
+2 -1
View File
@@ -19,9 +19,10 @@ const (
ParamEllipsis = "..."
ParamFilepath = "filepath"
ParamDirpath = "dirpath"
ParamIterator = "iterator"
)
// to be moved in its own source file
const (
ConstLastIndex = 0xFFFF_FFFF
)
)
+56 -31
View File
@@ -133,54 +133,79 @@ func (dc *dataCursor) Current() (item any, err error) { // must return io.EOF at
return
}
func (dc *dataCursor) _Next() (item any, err error) { // must return io.EOF after the last item
if dc.resource != nil {
ctx := cloneContext(dc.ctx)
// fmt.Printf("Entering Inner-Ctx [%p]: %s\n", ctx, CtxToString(ctx, 0))
if item, err = dc.nextFunc.Invoke(ctx, nextName, []any{dc.resource}); err == nil {
if item == nil {
err = io.EOF
} else {
dc.index++
}
}
// fmt.Printf("Exiting Inner-Ctx [%p]: %s\n", ctx, CtxToString(ctx, 0))
exportObjects(dc.ctx, ctx)
// fmt.Printf("Outer-Ctx [%p]: %s\n", dc.ctx, CtxToString(dc.ctx, 0))
} else {
err = errInvalidDataSource()
// func (dc *dataCursor) _Next() (item any, err error) { // must return io.EOF after the last item
// if dc.resource != nil {
// ctx := cloneContext(dc.ctx)
// // fmt.Printf("Entering Inner-Ctx [%p]: %s\n", ctx, CtxToString(ctx, 0))
// if item, err = dc.nextFunc.Invoke(ctx, nextName, []any{dc.resource}); err == nil {
// if item == nil {
// err = io.EOF
// } else {
// dc.index++
// }
// }
// // fmt.Printf("Exiting Inner-Ctx [%p]: %s\n", ctx, CtxToString(ctx, 0))
// exportObjects(dc.ctx, ctx)
// // fmt.Printf("Outer-Ctx [%p]: %s\n", dc.ctx, CtxToString(dc.ctx, 0))
// } else {
// err = errInvalidDataSource()
// }
// return
// }
// func (dc *dataCursor) _filter(item any) (filterdItem any, err error) {
// if filter, ok := dc.ds[filterName]; ok {
// ctx := cloneContext(dc.ctx)
// filterdItem, err = filter.Invoke(ctx, filterName, []any{item, dc.index})
// } else {
// filterdItem = item
// }
// return
// }
func (dc *dataCursor) checkFilter(filter Functor, item any) (accepted bool, err error) {
var v any
var ok bool
ctx := cloneContext(dc.ctx)
if v, err = filter.Invoke(ctx, filterName, []any{item, dc.index}); err == nil && v != nil {
if accepted, ok = v.(bool); !ok {
accepted = true // NOTE: A non-boolean value that is not nil means the item has been accepted
}
}
return
}
const filterName = "filter"
func (dc *dataCursor) filter(item any) (filterdItem any, err error) {
if filter, ok := dc.ds[filterName]; ok {
ctx := cloneContext(dc.ctx)
filterdItem, err = filter.Invoke(ctx, filterName, []any{item, dc.index});
} else {
filterdItem = item
}
func (dc *dataCursor) mapItem(mapper Functor, item any) (mappedItem any, err error) {
ctx := cloneContext(dc.ctx)
mappedItem, err = mapper.Invoke(ctx, mapName, []any{item, dc.index});
return
}
func (dc *dataCursor) Next() (item any, err error) { // must return io.EOF after the last item
var accepted bool
if dc.resource != nil {
ctx := cloneContext(dc.ctx)
// fmt.Printf("Entering Inner-Ctx [%p]: %s\n", ctx, CtxToString(ctx, 0))
filter := dc.ds[filterName]
mapper := dc.ds[mapName]
for item == nil && err == nil {
ctx := cloneContext(dc.ctx)
if item, err = dc.nextFunc.Invoke(ctx, nextName, []any{dc.resource}); err == nil {
if item == nil {
err = io.EOF
} else {
dc.index++
item, err = dc.filter(item)
if filter != nil {
if accepted, err = dc.checkFilter(filter, item); err != nil || !accepted {
item = nil
}
}
if item != nil && mapper != nil {
item, err = dc.mapItem(mapper, item)
}
}
}
exportObjects(dc.ctx, ctx)
}
// fmt.Printf("Exiting Inner-Ctx [%p]: %s\n", ctx, CtxToString(ctx, 0))
exportObjects(dc.ctx, ctx)
// fmt.Printf("Outer-Ctx [%p]: %s\n", dc.ctx, CtxToString(dc.ctx, 0))
} else {
err = errInvalidDataSource()
}
+7
View File
@@ -14,8 +14,15 @@ type ExprContext interface {
GetLast() any
SetVar(varName string, value any)
UnsafeSetVar(varName string, value any)
EnumVars(func(name string) (accept bool)) (varNames []string)
VarCount() int
DeleteVar(varName string)
EnumFuncs(func(name string) (accept bool)) (funcNames []string)
FuncCount() int
DeleteFunc(funcName string)
GetFuncInfo(name string) (item ExprFunc, exists bool)
Call(name string, args []any) (result any, err error)
RegisterFuncInfo(info ExprFunc)
+1
View File
@@ -6,6 +6,7 @@ package expr
// ---- Functor interface
type Functor interface {
Typer
Invoke(ctx ExprContext, name string, args []any) (result any, err error)
SetFunc(info ExprFunc)
GetFunc() ExprFunc
+1
View File
@@ -6,6 +6,7 @@ package expr
// ----Expression interface
type Expr interface {
Typer
Eval(ctx ExprContext) (result any, err error)
String() string
}
+2
View File
@@ -19,6 +19,8 @@ const (
currentName = "current"
indexName = "index"
countName = "count"
filterName = "filter"
mapName = "map"
)
type Iterator interface {
+5 -1
View File
@@ -90,16 +90,20 @@ func (it *ListIterator) TypeName() string {
}
func (it *ListIterator) HasOperation(name string) bool {
yes := name == resetName || name == indexName || name == countName
yes := name == nextName || name == resetName || name == indexName || name == countName || name == currentName
return yes
}
func (it *ListIterator) CallOperation(name string, args []any) (v any, err error) {
switch name {
case nextName:
v, err = it.Next()
case resetName:
v, err = it.Reset()
case indexName:
v = int64(it.Index())
case currentName:
v, err = it.Current()
case countName:
v = it.count
default:
+3 -3
View File
@@ -11,7 +11,7 @@ func newDefaultTerm(tk *Token) (inst *term) {
tk: *tk,
children: make([]*term, 0, 2),
position: posInfix,
priority: priCoalesce,
priority: priDefault,
evalFunc: evalDefault,
}
}
@@ -45,7 +45,7 @@ func newAlternateTerm(tk *Token) (inst *term) {
tk: *tk,
children: make([]*term, 0, 2),
position: posInfix,
priority: priCoalesce,
priority: priDefault,
evalFunc: evalAlternate,
}
}
@@ -81,7 +81,7 @@ func newDefaultAssignTerm(tk *Token) (inst *term) {
tk: *tk,
children: make([]*term, 0, 2),
position: posInfix,
priority: priCoalesce,
priority: priDefault,
evalFunc: evalAssignDefault,
}
}
+5 -4
View File
@@ -25,8 +25,8 @@ func evalInclude(ctx ExprContext, opTerm *term) (v any, err error) {
count := 0
if IsList(childValue) {
list, _ := childValue.([]any)
for i, filePathSpec := range list {
list, _ := childValue.(*ListType)
for i, filePathSpec := range *list {
if filePath, ok := filePathSpec.(string); ok {
if v, err = EvalFile(ctx, filePath); err == nil {
count++
@@ -47,8 +47,9 @@ func evalInclude(ctx ExprContext, opTerm *term) (v any, err error) {
} else {
err = opTerm.errIncompatibleType(childValue)
}
if err == nil {
v = count
if err != nil {
//v = count
v = nil
}
return
}
+2 -1
View File
@@ -33,5 +33,6 @@ func evalIterValue(ctx ExprContext, opTerm *term) (v any, err error) {
// init
func init() {
registerTermConstructor(SymOpenClosedRound, newIterValueTerm)
// registerTermConstructor(SymOpenClosedRound, newIterValueTerm)
registerTermConstructor(SymCaret, newIterValueTerm)
}
+69
View File
@@ -0,0 +1,69 @@
// Copyright (c) 2024 Celestino Amoroso (celestino.amoroso@gmail.com).
// All rights reserved.
// operator-unset.go
package expr
import "strings"
//-------- unset term
func newUnsetTerm(tk *Token) (inst *term) {
return &term{
tk: *tk,
children: make([]*term, 0, 1),
position: posPrefix,
priority: priSign,
evalFunc: evalUnset,
}
}
func deleteContextItem(ctx ExprContext, opTerm *term, item any) (deleted bool, err error) {
if name, ok := item.(string); ok {
var size int
if strings.HasSuffix(name, "()") {
size = ctx.FuncCount()
ctx.DeleteFunc(strings.TrimRight(name, "()"))
deleted = ctx.FuncCount() < size
} else {
size = ctx.VarCount()
ctx.DeleteVar(name)
deleted = ctx.VarCount() < size
}
} else {
err = opTerm.errIncompatibleType(item)
}
return
}
func evalUnset(ctx ExprContext, opTerm *term) (v any, err error) {
var childValue any
var deleted bool
if childValue, err = opTerm.evalPrefix(ctx); err != nil {
return
}
count := 0
if IsList(childValue) {
list, _ := childValue.(*ListType)
for _, item := range *list {
if deleted, err = deleteContextItem(ctx, opTerm, item); err != nil {
break
} else if deleted {
count++
}
}
} else if deleted, err = deleteContextItem(ctx, opTerm, childValue); err == nil && deleted {
count++
}
if err == nil {
v = int64(count)
}
return
}
// init
func init() {
registerTermConstructor(SymKwUnset, newUnsetTerm)
}
+4 -4
View File
@@ -269,11 +269,11 @@ func (scanner *scanner) fetchNextToken() (tk *Token) {
tk = scanner.makeToken(SymDollar, ch)
}
case '(':
if next, _ := scanner.peek(); next == ')' {
tk = scanner.moveOn(SymOpenClosedRound, ch, next)
} else {
// if next, _ := scanner.peek(); next == ')' {
// tk = scanner.moveOn(SymOpenClosedRound, ch, next)
// } else {
tk = scanner.makeToken(SymOpenRound, ch)
}
// }
case ')':
tk = scanner.makeToken(SymClosedRound, ch)
case '[':
+16
View File
@@ -132,6 +132,14 @@ func (ctx *SimpleStore) EnumVars(acceptor func(name string) (accept bool)) (varN
return
}
func (ctx *SimpleStore) VarCount() int {
return len(ctx.varStore)
}
func (ctx *SimpleStore) DeleteVar(varName string) {
delete(ctx.varStore, varName)
}
func (ctx *SimpleStore) GetFuncInfo(name string) (info ExprFunc, exists bool) {
info, exists = ctx.funcStore[name]
return
@@ -163,6 +171,14 @@ func (ctx *SimpleStore) EnumFuncs(acceptor func(name string) (accept bool)) (fun
return
}
func (ctx *SimpleStore) FuncCount() int {
return len(ctx.funcStore)
}
func (ctx *SimpleStore) DeleteFunc(funcName string) {
delete(ctx.funcStore, funcName)
}
func (ctx *SimpleStore) Call(name string, args []any) (result any, err error) {
if info, exists := GetLocalFuncInfo(ctx, name); exists {
functor := info.Functor()
+2
View File
@@ -106,6 +106,7 @@ const (
SymKwIn
SymKwInclude
SymKwNil
SymKwUnset
)
var keywords map[string]Symbol
@@ -123,5 +124,6 @@ func init() {
"NOT": SymKwNot,
"OR": SymKwOr,
"NIL": SymKwNil,
"UNSET": SymKwUnset,
}
}
+50
View File
@@ -0,0 +1,50 @@
// Copyright (c) 2024 Celestino Amoroso (celestino.amoroso@gmail.com).
// All rights reserved.
// t_builtin-iterator.go
package expr
import (
"testing"
)
func TestFuncRun(t *testing.T) {
section := "Builtin-Iterator"
inputs := []inputType{
/* 1 */ {`builtin "iterator"; it=$(1,2,3); run(it)`, nil, nil},
/* 2 */ {`builtin "iterator"; run($(1,2,3), func(index,item){item+10})`, nil, nil},
/* 3 */ {`builtin "iterator"; run($(1,2,3), func(index,item){it_status=it_status+item; true}, {"it_status":0})`, int64(6), nil},
/* 4 */ {`builtin ["iterator", "fmt"]; run($(1,2,3), func(index,item){println(item+10)})`, nil, nil},
/* 5 */ {`builtin "iterator"; run(nil)`, nil, `paramter "iterator" must be an iterator, passed <nil> [nil]`},
/* 6 */ {`builtin "iterator"; run($(1,2,3), nil)`, nil, `paramter "operator" must be a function, passed <nil> [nil]`},
/* 7 */ {`builtin "iterator"; run($(1,2,3), func(){1}, "prrr")`, nil, `paramter "vars" must be a dictionary, passed prrr [string]`},
}
//t.Setenv("EXPR_PATH", ".")
//runTestSuiteSpec(t, section, inputs, 3)
runTestSuite(t, section, inputs)
}
// func TestFmt(t *testing.T) {
// section := "Builtin-Fmt"
// text := "ciao mondo"
// inputs := []inputType{
// /* 1 */ {fmt.Sprintf(`println("%s")`, text), int64(11), nil},
// }
// // t.Setenv("EXPR_PATH", ".")
// var b bytes.Buffer
// ctx := NewSimpleStore()
// currentStdout := SetCtrl(ctx, ControlStdout, &b)
// runCtxTestSuite(t, ctx, section, inputs)
// SetCtrl(ctx, ControlStdout, currentStdout)
// if b.String() != text+"\n" {
// t.Errorf("println(): Got: %q, Want: %q", b.String(), text+"\n")
// }
// }
+2 -1
View File
@@ -30,13 +30,14 @@ func TestFuncs(t *testing.T) {
/* 17 */ {`f=func(x,n){1}; f(3,4,)`, nil, `[1:24] expected "function-param-value", got ")"`},
/* 18 */ {`factory=func(base){func(){@base=base+1}}; inc10=factory(10); inc5=factory(5); inc10(); inc5(); inc10()`, int64(12), nil},
/* 19 */ {`f=func(a,y=1,z="sos"){}; string(f)`, `f(a, y=1, z="sos"):any{}`, nil},
// /* 20 */ {`a=[func(){3}]; a[0]()`, int64(3), nil},
// /* 20 */ {`m={}; m["f"]=func(){3}; m["f"]()`, int64(3), nil},
// /* 18 */ {`f=func(a){a*2}`, nil, errors.New(`[1:24] expected "function-param-value", got ")"`)},
}
// t.Setenv("EXPR_PATH", ".")
// runTestSuiteSpec(t, section, inputs, 20)
//runTestSuiteSpec(t, section, inputs, 20)
runTestSuite(t, section, inputs)
}
+8 -4
View File
@@ -9,10 +9,10 @@ import "testing"
func TestIteratorParser(t *testing.T) {
section := "Iterator"
inputs := []inputType{
/* 1 */ {`include "test-resources/iterator.expr"; it=$(ds,3); ()it`, int64(0), nil},
/* 1 */ {`include "test-resources/iterator.expr"; it=$(ds,3); ^it`, int64(0), nil},
/* 2 */ {`include "test-resources/iterator.expr"; it=$(ds,3); it++; it++`, int64(1), nil},
/* 3 */ {`include "test-resources/iterator.expr"; it=$(ds,3); it++; it++; #it`, int64(2), nil},
/* 4 */ {`include "test-resources/iterator.expr"; it=$(ds,3); it++; it++; it.reset; ()it`, int64(0), nil},
/* 4 */ {`include "test-resources/iterator.expr"; it=$(ds,3); it++; it++; it.reset; ^it`, int64(0), nil},
/* 5 */ {`builtin "math.arith"; include "test-resources/iterator.expr"; it=$(ds,3); add(it)`, int64(6), nil},
/* 6 */ {`builtin "math.arith"; include "test-resources/iterator.expr"; it=$(ds,3); mul(it)`, int64(0), nil},
/* 7 */ {`builtin "math.arith"; include "test-resources/file-reader.expr"; it=$(ds,"test-resources/int.list"); mul(it)`, int64(12000), nil},
@@ -20,9 +20,13 @@ func TestIteratorParser(t *testing.T) {
/* 9 */ {`include "test-resources/file-reader.expr"; it=$(ds,"test-resources/int.list"); it.clean`, true, nil},
/* 10 */ {`it=$(1,2,3); it++`, int64(1), nil},
/* 11 */ {`it=$(1,2,3); it++; it.reset; it++`, int64(1), nil},
// /* 12 */ {`include "test-resources/filter.expr"; it=$(ds,10); it++`, int64(1), nil},
/* 12 */ {`it=$([1,2,3,4],1); it++`, int64(2), nil},
/* 13 */ {`it=$([1,2,3,4],1,3); it++; it++;`, int64(3), nil},
/* 14 */ {`it=$([1,2,3,4],1,3,2); it++; it++;`, int64(4), nil},
/* 15 */ {`it=$([1,2,3,4],1,2,2); it++; it++;`, nil, `EOF`},
/* 16 */ {`include "test-resources/filter.expr"; it=$(ds,10); it++`, int64(2), nil},
}
//runTestSuiteSpec(t, section, inputs, 12)
//runTestSuiteSpec(t, section, inputs, 4)
runTestSuite(t, section, inputs)
}
+23
View File
@@ -0,0 +1,23 @@
// Copyright (c) 2024 Celestino Amoroso (celestino.amoroso@gmail.com).
// All rights reserved.
// t_operator_test.go
package expr
import (
"testing"
)
func TestOperator(t *testing.T) {
section := "Operator"
inputs := []inputType{
/* 1 */ {`a=1; unset "a"; a`, nil, `undefined variable or function "a"`},
/* 2 */ {`a=1; unset ["a", "b"]`, int64(1), nil},
/* 3 */ {`f=func(){3}; unset "f()"`, int64(1), nil},
}
// t.Setenv("EXPR_PATH", ".")
//runTestSuiteSpec(t, section, inputs, 3)
runTestSuite(t, section, inputs)
}
+1 -1
View File
@@ -26,7 +26,7 @@ const (
priSign
priFact
priIterValue
priCoalesce
priDefault
priIncDec
priDot
priValue
+15
View File
@@ -0,0 +1,15 @@
ds={
"init":func(end){@end=end; @current=0 but true},
"current":func(){current},
"next":func(){
((next=current+1) <= end) ? [true] {@current=next but current} :: {nil}
}
, "filter":func(item,index) { item > 0 }
, "map": func(item,index) { item * 2 }
}
/* Example
;
it=$(ds,3);
add(it)
*/