// Copyright (c) 2024-2026 Celestino Amoroso (celestino.amoroso@gmail.com). // All rights reserved. // parser.go package expr import ( "errors" "fmt" "slices" "git.portale-stac.it/go-pkg/expr/scan" "git.portale-stac.it/go-pkg/expr/sym" "golang.org/x/exp/constraints" ) //-------- parser type parserContext uint16 const ( parserNoFlags = 0 allowMultiExpr parserContext = 1 << iota allowVarRef selectorContext listContext // squareContext for list indexContext // squareContext for index allowIndex // allow index in squareContext squareContext = listContext | indexContext // Square parenthesis for list or index ) func hasFlag[T constraints.Unsigned](set T, singleFlag T) bool { return (set & singleFlag) != 0 } func addFlags[T constraints.Unsigned](set T, flags T) T { return set | flags } func addFlagsCond[T constraints.Unsigned](set T, flags T, cond bool) (newSet T) { if cond { newSet = set | flags } else { newSet = set } return } func remFlags[T constraints.Unsigned](set T, flags T) T { return set & (^flags) } type parser struct { } func NewParser() (p *parser) { p = &parser{} return p } func (parser *parser) Next(scanner *scan.Scanner) (tk *scan.Token) { for tk = scanner.Next(); tk.IsSymbol(sym.SymComment); tk = scanner.Next() { } return } func (parser *parser) parseFuncCall(scanner *scan.Scanner, ctx parserContext, tk *scan.Token) (tree *scan.Term, err error) { args := make([]*scan.Term, 0, 10) itemExpected := false lastSym := sym.SymUnknown for lastSym != sym.SymClosedRound && lastSym != sym.SymEos { var subTree *scan.Ast if subTree, err = parser.parseItem(scanner, ctx, sym.SymComma, sym.SymClosedRound); err != nil { break } prev := scanner.Previous() if subTree.Root() != nil { args = append(args, subTree.Root()) } else if itemExpected { err = prev.ErrorExpectedGot("function-param-value") break } itemExpected = prev.Sym == sym.SymComma lastSym = scanner.Previous().Sym } if err == nil { if lastSym != sym.SymClosedRound { err = errors.New("unterminated arguments list") } else { tree = newFuncCallTerm(tk, args) } } return } func (parser *parser) parseFuncDef(scanner *scan.Scanner) (tree *scan.Term, err error) { // Example: "add = func(x,y) {x+y} var body *scan.Ast args := make([]*scan.Term, 0) lastSym := sym.SymUnknown defaultParamsStarted := false itemExpected := false tk := scanner.Previous() for lastSym != sym.SymClosedRound && lastSym != sym.SymEos { tk = parser.Next(scanner) if tk.IsSymbol(sym.SymIdentifier) { param := scan.NewTerm(tk) if len(args) > 0 { if pos := paramAlreadyDefined(args, param); pos > 0 { err = tk.Errorf("parameter %q at position %d already defined at position %d", param.Source(), len(args)+1, pos) break } } args = append(args, param) tk = parser.Next(scanner) if tk.Sym == sym.SymEqual { var paramExpr *scan.Ast defaultParamsStarted = true if paramExpr, err = parser.parseItem(scanner, parserNoFlags, sym.SymComma, sym.SymClosedRound); err != nil { break } param.ForceChild(paramExpr.Root()) } else if defaultParamsStarted { err = tk.Errorf("can't mix default and non-default parameters") break } } else if itemExpected { prev := scanner.Previous() err = prev.ErrorExpectedGot("function-param-spec") break } lastSym = scanner.Previous().Sym itemExpected = lastSym == sym.SymComma } if err == nil && lastSym != sym.SymClosedRound { err = tk.ErrorExpectedGot(")") } if err == nil { tk = parser.Next(scanner) if tk.IsSymbol(sym.SymOpenBrace) { body, err = parser.parseGeneral(scanner, allowMultiExpr|allowVarRef, sym.SymClosedBrace) } else { err = tk.ErrorExpectedGot("{") } } if err == nil { if scanner.Previous().Sym != sym.SymClosedBrace { err = scanner.Previous().ErrorExpectedGot("}") } else { tk = scanner.MakeValueToken(sym.SymExpression, "", body) tree = newFuncDefTerm(tk, args) } } return } func paramAlreadyDefined(args []*scan.Term, param *scan.Term) (position int) { position = 0 for i, arg := range args { if arg.Source() == param.Source() { position = i + 1 } } return } func (parser *parser) parseList(scanner *scan.Scanner, ctx parserContext, termSym sym.Symbol) (listTerm *scan.Term, err error) { r, c := scanner.LastPos() args := make([]*scan.Term, 0) lastSym := sym.SymUnknown itemExpected := false itemCtx := remFlags(ctx, allowIndex) for lastSym != termSym && lastSym != sym.SymEos { var itemTree *scan.Ast if itemTree, err = parser.parseItem(scanner, itemCtx, sym.SymComma, termSym); err == nil { root := itemTree.Root() if root != nil { if root.Symbol() == sym.SymDoubleDot { if hasFlag(ctx, allowIndex) { changeColonToInterval(root) } else { // err = root.Errorf("unexpected range expression") err = errIntervalUnexpectedExpression(root) break } } // if hasFlag(ctx, allowIndex) && root.Symbol() == sym.SymDoubleDot { // changeColonToInterval(root) // } // if !hasFlag(ctx, allowIndex) && root.Symbol() == sym.SymInterval { // // err = root.Errorf("unexpected range expression") // err = errIntervalUnexpectedExpression(root) // break // } args = append(args, root) } else if itemExpected { prev := scanner.Previous() err = prev.ErrorExpectedGot("list-item") break } } else { break } lastSym = scanner.Previous().Sym if itemExpected = lastSym == sym.SymComma; itemExpected { remFlags(ctx, allowIndex) } } if err == nil { if lastSym != termSym { err = scanner.Previous().ErrorExpectedGot("]") } else if termSym == sym.SymClosedSquare { listTerm = newListTerm(r, c, args) } else if termSym == sym.SymGreaterClosedSquare { listTerm = newLinkedListTerm(r, c, args) } else { err = fmt.Errorf("[%d:%d] unknown list type", r, c) } } return } func (parser *parser) parseIterDef(scanner *scan.Scanner, ctx parserContext) (subtree *scan.Term, err error) { tk := scanner.Previous() args := make([]*scan.Term, 0) lastSym := sym.SymUnknown itemExpected := false for lastSym != sym.SymClosedRound && lastSym != sym.SymEos { var subTree *scan.Ast if subTree, err = parser.parseItem(scanner, ctx, sym.SymComma, sym.SymClosedRound); err == nil { if subTree.Root() != nil { args = append(args, subTree.Root()) } else if itemExpected { prev := scanner.Previous() err = prev.ErrorExpectedGot("iterator-param") break } } else { break } lastSym = scanner.Previous().Sym itemExpected = lastSym == sym.SymComma } if err == nil { if lastSym != sym.SymClosedRound { err = scanner.Previous().ErrorExpectedGot(")") } else { subtree = newIteratorTerm(tk, args) } } return } // func (parser *parser) parseDictKey(scanner *scan.Scanner) (key any, err error) { // tk := parser.Next(scanner) // if tk.Sym == sym.SymError { // err = tk.Error() // return // } // if tk.Sym == sym.SymClosedBrace || tk.Sym == sym.SymEos { // return // } // if tk.Sym == sym.SymInteger || tk.Sym == sym.SymString || tk.Sym == sym.SymIdentifier { // tkSep := parser.Next(scanner) // if tkSep.Sym != sym.SymColon { // err = tkSep.ErrorExpectedGot(":") // } else { // key = tk.Value // } // } else { // err = tk.ErrorExpectedGot("dictionary-key or }") // } // return // } func (parser *parser) parseDictKey(scanner *scan.Scanner) (key *scan.Term, err error) { var keyTree *scan.Ast if keyTree, err = parser.parseItem(scanner, parserNoFlags, sym.SymColon, sym.SymClosedBrace); err == nil { key = keyTree.Root() tkSep := scanner.Previous() symbol := tkSep.Sym if symbol == sym.SymClosedBrace || symbol == sym.SymEos { if key != nil { err = tkSep.ErrorExpectedGot(":") } } else if symbol != sym.SymColon { err = tkSep.ErrorExpectedGot(":") } } return } func (parser *parser) parseDictionary(scanner *scan.Scanner, ctx parserContext) (subtree *scan.Term, err error) { args := make(map[any]*scan.Term, 0) lastSym := sym.SymUnknown itemExpected := false for lastSym != sym.SymClosedBrace && lastSym != sym.SymEos { var valueTree *scan.Ast var key any if key, err = parser.parseDictKey(scanner); err != nil { break } else if key.(*scan.Term) == nil { tk := scanner.Previous() lastSym = tk.Sym if itemExpected { err = tk.ErrorExpectedGot("dictionary-key") } break } if valueTree, err = parser.parseItem(scanner, ctx, sym.SymComma, sym.SymClosedBrace); err == nil { if valueTree.Root() != nil { args[key] = valueTree.Root() } else /*if key != nil*/ { prev := scanner.Previous() err = prev.ErrorExpectedGot("dictionary-value") break } } else { break } lastSym = scanner.Previous().Sym itemExpected = lastSym == sym.SymComma } if err == nil { if lastSym != sym.SymClosedBrace { err = scanner.Previous().ErrorExpectedGot("}") } else { subtree = newDictTerm(args) } } return } func (parser *parser) parseSelectorCase(scanner *scan.Scanner, ctx parserContext, defaultCase bool) (caseTerm *scan.Term, err error) { var filterList *scan.Term var caseExpr *scan.Ast ctx = remFlags(ctx, allowIndex) tk := parser.Next(scanner) startRow := tk.Row() startCol := tk.Col() if tk.Sym == sym.SymOpenSquare { if defaultCase { err = tk.Errorf("case list in default clause") return } if filterList, err = parser.parseList(scanner, remFlags(ctx, allowIndex), sym.SymClosedSquare); err != nil { return } tk = parser.Next(scanner) startRow = tk.Row() startCol = tk.Col() } else if !defaultCase { filterList = newListTerm(startRow, startCol, make([]*scan.Term, 0)) } if tk.Sym == sym.SymOpenBrace { if caseExpr, err = parser.parseGeneral(scanner, ctx|allowMultiExpr, sym.SymClosedBrace); err != nil { return } } else { err = tk.ErrorExpectedGot("{") } if err == nil { caseTerm = newSelectorCaseTerm(startRow, startCol, filterList, caseExpr) } return } func addSelectorCase(selectorTerm, caseTerm *scan.Term) { if len(selectorTerm.Children) < 2 { caseListTerm := newListTermA(caseTerm) selectorTerm.Children = append(selectorTerm.Children, caseListTerm) } else { caseListTerm := selectorTerm.Children[1] caseList, _ := caseListTerm.Value().([]*scan.Term) caseList = append(caseList, caseTerm) caseListTerm.Tk.Value = caseList } caseTerm.Parent = selectorTerm } func (parser *parser) parseSelector(scanner *scan.Scanner, tree *scan.Ast, ctx parserContext) (selectorTerm *scan.Term, err error) { var caseTerm *scan.Term ctx = remFlags(ctx, allowIndex) tk := scanner.MakeToken(sym.SymSelector, '?') if selectorTerm, err = tree.AddToken(tk); err != nil { return } if caseTerm, err = parser.parseSelectorCase(scanner, ctx|allowVarRef, false); err == nil { addSelectorCase(selectorTerm, caseTerm) } return } func (parser *parser) parseItem(scanner *scan.Scanner, ctx parserContext, termSymbols ...sym.Symbol) (tree *scan.Ast, err error) { return parser.parseGeneral(scanner, ctx|allowVarRef, termSymbols...) } func (parser *parser) Parse(scanner *scan.Scanner, termSymbols ...sym.Symbol) (tree *scan.Ast, err error) { defer func() { if r := recover(); r != nil { if errVal, ok := r.(error); ok { err = errVal } else { err = errors.New("unexpected error while parsing the expression") } } }() termSymbols = append(termSymbols, sym.SymEos) return parser.parseGeneral(scanner, allowMultiExpr, termSymbols...) } func couldBeACollection(t *scan.Term) bool { var symbol = sym.SymUnknown if t != nil { symbol = t.Symbol() } // return sym == sym.SymList || sym == sym.SymString || sym == sym.SymDict || sym == sym.SymExpression || sym == sym.SymVariable return slices.Contains([]sym.Symbol{sym.SymList, sym.SymLinkedList, sym.SymString, sym.SymDict, sym.SymExpression, sym.SymVariable, sym.SymIndex}, symbol) } func listSubTree(tree *scan.Ast, listTerm *scan.Term, allowIndeces bool) (root *scan.Term, err error) { var tk *scan.Token if allowIndeces { tk = scan.NewToken(listTerm.Tk.Row(), listTerm.Tk.Col(), sym.SymIndex, listTerm.Source()) root = scan.NewTerm(tk) if err = tree.AddTerm(root); err == nil { err = tree.AddTerm(listTerm) } } else { root = listTerm err = tree.AddTerm(listTerm) } return } func changePrefix(tk *scan.Token) { switch tk.Sym { case sym.SymMinus: tk.SetSymbol(sym.SymChangeSign) case sym.SymPlus: tk.SetSymbol(sym.SymUnchangeSign) case sym.SymStar: tk.SetSymbol(sym.SymDereference) case sym.SymExclamation: tk.SetSymbol(sym.SymNot) case sym.SymDoublePlus: tk.SetSymbol(sym.SymPreInc) case sym.SymDoubleMinus: tk.SetSymbol(sym.SymPreDec) } } func (parser *parser) parseGeneral(scanner *scan.Scanner, ctx parserContext, termSymbols ...sym.Symbol) (tree *scan.Ast, err error) { var selectorTerm *scan.Term = nil var currentTerm *scan.Term = nil var tk *scan.Token tree = scan.NewAst() firstToken := true // lastSym := SymUnknown for tk = parser.Next(scanner); err == nil && tk != nil && !tk.IsTerm(termSymbols); tk = parser.Next(scanner) { // if tk.Sym == SymComment { // continue // } if tk.Sym == sym.SymSemiColon { if hasFlag(ctx, allowMultiExpr) { tree.ToForest() firstToken = true currentTerm = nil selectorTerm = nil continue } else { err = tk.ErrorExpectedOneOfGot(termSymbols...) break } } //fmt.Println("Token:", tk) if firstToken { changePrefix(tk) firstToken = false } switch tk.Sym { case sym.SymOpenRound: var subTree *scan.Ast if subTree, err = parser.parseGeneral(scanner, ctx, sym.SymClosedRound); err == nil { if subTree.Root() == nil { err = tk.ErrorExpectedGotString("expression", "()") } else { exprTerm := newExprTerm(subTree.Root()) err = tree.AddTerm(exprTerm) currentTerm = exprTerm } } case sym.SymFuncCall: var funcCallTerm *scan.Term if funcCallTerm, err = parser.parseFuncCall(scanner, ctx, tk); err == nil { err = tree.AddTerm(funcCallTerm) currentTerm = funcCallTerm } case sym.SymOpenSquare: var listTerm *scan.Term newCtx := addFlagsCond(addFlags(ctx, squareContext), allowIndex, couldBeACollection(currentTerm)) if listTerm, err = parser.parseList(scanner, newCtx, sym.SymClosedSquare); err == nil { currentTerm, err = listSubTree(tree, listTerm, hasFlag(newCtx, allowIndex)) } case sym.SymOpenSquareLess: var listTerm *scan.Term newCtx := addFlagsCond(addFlags(ctx, listContext), allowIndex, false) if listTerm, err = parser.parseList(scanner, newCtx, sym.SymGreaterClosedSquare); err == nil { currentTerm, err = listSubTree(tree, listTerm, hasFlag(newCtx, allowIndex)) } case sym.SymOpenBrace: if currentTerm != nil && currentTerm.Symbol() == sym.SymColon { err = currentTerm.Errorf(`selector-case outside of a selector context`) } else { var mapTerm *scan.Term if mapTerm, err = parser.parseDictionary(scanner, ctx); err == nil { err = tree.AddTerm(mapTerm) currentTerm = mapTerm } } case sym.SymEqual, sym.SymPlusEqual, sym.SymMinusEqual, sym.SymStarEqual, sym.SymSlashEqual, sym.SymPercEqual, sym.SymAmpersandEqual, sym.SymVertBarEqual, sym.SymDoubleLessEqual, sym.SymDoubleGreaterEqual, sym.SymCaretEqual: currentTerm, err = tree.AddToken(tk) firstToken = true case sym.SymFuncDef: var funcDefTerm *scan.Term if funcDefTerm, err = parser.parseFuncDef(scanner); err == nil { err = tree.AddTerm(funcDefTerm) currentTerm = funcDefTerm } case sym.SymDollarRound: var iterDefTerm *scan.Term if iterDefTerm, err = parser.parseIterDef(scanner, ctx); err == nil { err = tree.AddTerm(iterDefTerm) currentTerm = iterDefTerm } case sym.SymIdentifier: if tk.Source()[0] == '@' && !hasFlag(ctx, allowVarRef) { err = tk.Errorf("variable references are not allowed in top level expressions: %q", tk.Source()) } else { currentTerm, err = tree.AddToken(tk) } case sym.SymQuestion: if selectorTerm, err = parser.parseSelector(scanner, tree, ctx); err == nil { currentTerm = selectorTerm addFlags(ctx, selectorContext) } case sym.SymColon, sym.SymDoubleColon: var caseTerm *scan.Term if selectorTerm != nil { if caseTerm, err = parser.parseSelectorCase(scanner, ctx, tk.Sym == sym.SymDoubleColon); err == nil { addSelectorCase(selectorTerm, caseTerm) currentTerm = caseTerm if tk.Sym == sym.SymDoubleColon { selectorTerm = nil } } } else { // err = tk.Errorf("colon outside a selector context") currentTerm, err = tree.AddToken(tk) // Colon outside a selector term acts like a separator firstToken = tk.IsOneOfA(sym.SymColon, sym.SymInterval) } case sym.SymDoubleDot: currentTerm, err = tree.AddToken(tk) if currentTerm.GetLeftChild() == nil { zeroTk := scan.NewValueToken(tk.Row(), tk.Col(), sym.SymInteger, "0", int64(0)) tree.AddToken(zeroTk) } // Double dot acts like a separator firstToken = tk.IsOneOfA(sym.SymDoubleDot, sym.SymInterval) default: currentTerm, err = tree.AddToken(tk) } if currentTerm != nil && currentTerm.Tk.Sym != sym.SymSelector && currentTerm.Parent != nil && currentTerm.Parent.Tk.Sym != sym.SymSelector { selectorTerm = nil remFlags(ctx, selectorContext) } // lastSym = tk.Sym } if err == nil { if !tk.IsOneOf(termSymbols) { var symDesc string if tk.IsSymbol(sym.SymError) { symDesc = tk.ErrorText() } else { symDesc = scan.SymToString(tk.Sym) } err = tk.ErrorExpectedGotStringWithPrefix("expected one of", scan.SymListToString(termSymbols, true), symDesc) } else { err = tk.Error() } } return }