// Copyright (c) 2024-2026 Celestino Amoroso (celestino.amoroso@gmail.com). // All rights reserved. // parser.go package expr import ( "errors" "slices" "git.portale-stac.it/go-pkg/expr/scan" "golang.org/x/exp/constraints" ) //-------- parser type parserContext uint16 const ( parserNoFlags = 0 allowMultiExpr parserContext = 1 << iota allowVarRef selectorContext listContext // squareContext for list indexContext // squareContext for index allowIndex // allow index in squareContext squareContext = listContext | indexContext // Square parenthesis for list or index ) func hasFlag[T constraints.Unsigned](set T, singleFlag T) bool { return (set & singleFlag) != 0 } func addFlags[T constraints.Unsigned](set T, flags T) T { return set | flags } func addFlagsCond[T constraints.Unsigned](set T, flags T, cond bool) (newSet T) { if cond { newSet = set | flags } else { newSet = set } return } func remFlags[T constraints.Unsigned](set T, flags T) T { return set & (^flags) } type parser struct { } func NewParser() (p *parser) { p = &parser{} return p } func (parser *parser) Next(scanner *scan.Scanner) (tk *scan.Token) { for tk = scanner.Next(); tk.IsSymbol(scan.SymComment); tk = scanner.Next() { } return } func (parser *parser) parseFuncCall(scanner *scan.Scanner, ctx parserContext, tk *scan.Token) (tree *scan.Term, err error) { args := make([]*scan.Term, 0, 10) itemExpected := false lastSym := scan.SymUnknown for lastSym != scan.SymClosedRound && lastSym != scan.SymEos { var subTree *scan.Ast if subTree, err = parser.parseItem(scanner, ctx, scan.SymComma, scan.SymClosedRound); err != nil { break } prev := scanner.Previous() if subTree.Root() != nil { args = append(args, subTree.Root()) } else if itemExpected { err = prev.ErrorExpectedGot("function-param-value") break } itemExpected = prev.Sym == scan.SymComma lastSym = scanner.Previous().Sym } if err == nil { if lastSym != scan.SymClosedRound { err = errors.New("unterminated arguments list") } else { tree = newFuncCallTerm(tk, args) } } return } func (parser *parser) parseFuncDef(scanner *scan.Scanner) (tree *scan.Term, err error) { // Example: "add = func(x,y) {x+y} var body *scan.Ast args := make([]*scan.Term, 0) lastSym := scan.SymUnknown defaultParamsStarted := false itemExpected := false tk := scanner.Previous() for lastSym != scan.SymClosedRound && lastSym != scan.SymEos { tk = parser.Next(scanner) if tk.IsSymbol(scan.SymIdentifier) { param := scan.NewTerm(tk) if len(args) > 0 { if pos := paramAlreadyDefined(args, param); pos > 0 { err = tk.Errorf("parameter %q at position %d already defined at position %d", param.Source(), len(args)+1, pos) break } } args = append(args, param) tk = parser.Next(scanner) if tk.Sym == scan.SymEqual { var paramExpr *scan.Ast defaultParamsStarted = true if paramExpr, err = parser.parseItem(scanner, parserNoFlags, scan.SymComma, scan.SymClosedRound); err != nil { break } param.ForceChild(paramExpr.Root()) } else if defaultParamsStarted { err = tk.Errorf("can't mix default and non-default parameters") break } } else if itemExpected { prev := scanner.Previous() err = prev.ErrorExpectedGot("function-param-spec") break } lastSym = scanner.Previous().Sym itemExpected = lastSym == scan.SymComma } if err == nil && lastSym != scan.SymClosedRound { err = tk.ErrorExpectedGot(")") } if err == nil { tk = parser.Next(scanner) if tk.IsSymbol(scan.SymOpenBrace) { body, err = parser.parseGeneral(scanner, allowMultiExpr|allowVarRef, scan.SymClosedBrace) } else { err = tk.ErrorExpectedGot("{") } } if err == nil { if scanner.Previous().Sym != scan.SymClosedBrace { err = scanner.Previous().ErrorExpectedGot("}") } else { tk = scanner.MakeValueToken(scan.SymExpression, "", body) tree = newFuncDefTerm(tk, args) } } return } func paramAlreadyDefined(args []*scan.Term, param *scan.Term) (position int) { position = 0 for i, arg := range args { if arg.Source() == param.Source() { position = i + 1 } } return } func (parser *parser) parseList(scanner *scan.Scanner, ctx parserContext) (listTerm *scan.Term, err error) { r, c := scanner.LastPos() args := make([]*scan.Term, 0) lastSym := scan.SymUnknown itemExpected := false itemCtx := remFlags(ctx, allowIndex) for lastSym != scan.SymClosedSquare && lastSym != scan.SymEos { zeroRequired := scanner.Current().Sym == scan.SymColon var itemTree *scan.Ast if itemTree, err = parser.parseItem(scanner, itemCtx, scan.SymComma, scan.SymClosedSquare); err == nil { root := itemTree.Root() if root != nil { if hasFlag(ctx, allowIndex) && root.Symbol() == scan.SymColon { changeColonToRange(root) } if !hasFlag(ctx, allowIndex) && root.Symbol() == scan.SymRange { // err = root.Errorf("unexpected range expression") err = errRangeUnexpectedExpression(root) break } args = append(args, root) if hasFlag(ctx, allowIndex) && root.Symbol() == scan.SymRange && zeroRequired { //len(root.children) == 0 { if len(root.Children) == 1 { root.Children = append(root.Children, root.Children[0]) } else if len(root.Children) > 1 { // err = root.Errorf("invalid range specification") err = errRangeInvalidSpecification(root) break } zeroTk := scan.NewValueToken(root.Tk.Row(), root.Tk.Col(), scan.SymInteger, "0", int64(0)) zeroTerm := scan.NewTerm(zeroTk) zeroTerm.SetParent(root) root.Children[0] = zeroTerm } } else if itemExpected { prev := scanner.Previous() err = prev.ErrorExpectedGot("list-item") break } } else { break } lastSym = scanner.Previous().Sym if itemExpected = lastSym == scan.SymComma; itemExpected { remFlags(ctx, allowIndex) } } if err == nil { if lastSym != scan.SymClosedSquare { err = scanner.Previous().ErrorExpectedGot("]") } else { listTerm = newListTerm(r, c, args) } } return } func (parser *parser) parseIterDef(scanner *scan.Scanner, ctx parserContext) (subtree *scan.Term, err error) { tk := scanner.Previous() args := make([]*scan.Term, 0) lastSym := scan.SymUnknown itemExpected := false for lastSym != scan.SymClosedRound && lastSym != scan.SymEos { var subTree *scan.Ast if subTree, err = parser.parseItem(scanner, ctx, scan.SymComma, scan.SymClosedRound); err == nil { if subTree.Root() != nil { args = append(args, subTree.Root()) } else if itemExpected { prev := scanner.Previous() err = prev.ErrorExpectedGot("iterator-param") break } } else { break } lastSym = scanner.Previous().Sym itemExpected = lastSym == scan.SymComma } if err == nil { if lastSym != scan.SymClosedRound { err = scanner.Previous().ErrorExpectedGot(")") } else { subtree = newIteratorTerm(tk, args) } } return } func (parser *parser) parseDictKey(scanner *scan.Scanner) (key any, err error) { tk := parser.Next(scanner) if tk.Sym == scan.SymError { err = tk.Error() return } if tk.Sym == scan.SymClosedBrace || tk.Sym == scan.SymEos { return } if tk.Sym == scan.SymInteger || tk.Sym == scan.SymString { tkSep := parser.Next(scanner) if tkSep.Sym != scan.SymColon { err = tkSep.ErrorExpectedGot(":") } else { key = tk.Value } } else { err = tk.ErrorExpectedGot("dictionary-key or }") } return } func (parser *parser) parseDictionary(scanner *scan.Scanner, ctx parserContext) (subtree *scan.Term, err error) { args := make(map[any]*scan.Term, 0) lastSym := scan.SymUnknown itemExpected := false for lastSym != scan.SymClosedBrace && lastSym != scan.SymEos { var subTree *scan.Ast var key any if key, err = parser.parseDictKey(scanner); err != nil { break } else if key == nil { tk := scanner.Previous() lastSym = tk.Sym if itemExpected { err = tk.ErrorExpectedGot("dictionary-key") } break } if subTree, err = parser.parseItem(scanner, ctx, scan.SymComma, scan.SymClosedBrace); err == nil { if subTree.Root() != nil { args[key] = subTree.Root() } else /*if key != nil*/ { prev := scanner.Previous() err = prev.ErrorExpectedGot("dictionary-value") break } } else { break } lastSym = scanner.Previous().Sym itemExpected = lastSym == scan.SymComma } if err == nil { if lastSym != scan.SymClosedBrace { err = scanner.Previous().ErrorExpectedGot("}") } else { subtree = newDictTerm(args) } } return } func (parser *parser) parseSelectorCase(scanner *scan.Scanner, ctx parserContext, defaultCase bool) (caseTerm *scan.Term, err error) { var filterList *scan.Term var caseExpr *scan.Ast ctx = remFlags(ctx, allowIndex) tk := parser.Next(scanner) startRow := tk.Row() startCol := tk.Col() if tk.Sym == scan.SymOpenSquare { if defaultCase { err = tk.Errorf("case list in default clause") return } if filterList, err = parser.parseList(scanner, remFlags(ctx, allowIndex)); err != nil { return } tk = parser.Next(scanner) startRow = tk.Row() startCol = tk.Col() } else if !defaultCase { filterList = newListTerm(startRow, startCol, make([]*scan.Term, 0)) } if tk.Sym == scan.SymOpenBrace { if caseExpr, err = parser.parseGeneral(scanner, ctx|allowMultiExpr, scan.SymClosedBrace); err != nil { return } } else { err = tk.ErrorExpectedGot("{") } if err == nil { caseTerm = newSelectorCaseTerm(startRow, startCol, filterList, caseExpr) } return } func addSelectorCase(selectorTerm, caseTerm *scan.Term) { if len(selectorTerm.Children) < 2 { caseListTerm := newListTermA(caseTerm) selectorTerm.Children = append(selectorTerm.Children, caseListTerm) } else { caseListTerm := selectorTerm.Children[1] caseList, _ := caseListTerm.Value().([]*scan.Term) caseList = append(caseList, caseTerm) caseListTerm.Tk.Value = caseList } caseTerm.Parent = selectorTerm } func (parser *parser) parseSelector(scanner *scan.Scanner, tree *scan.Ast, ctx parserContext) (selectorTerm *scan.Term, err error) { var caseTerm *scan.Term ctx = remFlags(ctx, allowIndex) tk := scanner.MakeToken(scan.SymSelector, '?') if selectorTerm, err = tree.AddToken(tk); err != nil { return } if caseTerm, err = parser.parseSelectorCase(scanner, ctx|allowVarRef, false); err == nil { addSelectorCase(selectorTerm, caseTerm) } return } func (parser *parser) parseItem(scanner *scan.Scanner, ctx parserContext, termSymbols ...scan.Symbol) (tree *scan.Ast, err error) { return parser.parseGeneral(scanner, ctx|allowVarRef, termSymbols...) } func (parser *parser) Parse(scanner *scan.Scanner, termSymbols ...scan.Symbol) (tree *scan.Ast, err error) { defer func() { if r := recover(); r != nil { if errVal, ok := r.(error); ok { err = errVal } else { err = errors.New("unexpected error while parsing the expression") } } }() termSymbols = append(termSymbols, scan.SymEos) return parser.parseGeneral(scanner, allowMultiExpr, termSymbols...) } func couldBeACollection(t *scan.Term) bool { var sym = scan.SymUnknown if t != nil { sym = t.Symbol() } // return sym == scan.SymList || sym == scan.SymString || sym == scan.SymDict || sym == scan.SymExpression || sym == scan.SymVariable return slices.Contains([]scan.Symbol{scan.SymList, scan.SymString, scan.SymDict, scan.SymExpression, scan.SymVariable, scan.SymIndex}, sym) } func listSubTree(tree *scan.Ast, listTerm *scan.Term, allowIndeces bool) (root *scan.Term, err error) { var tk *scan.Token if allowIndeces { tk = scan.NewToken(listTerm.Tk.Row(), listTerm.Tk.Col(), scan.SymIndex, listTerm.Source()) root = scan.NewTerm(tk) if err = tree.AddTerm(root); err == nil { err = tree.AddTerm(listTerm) } } else { root = listTerm err = tree.AddTerm(listTerm) } return } func changePrefix(tk *scan.Token) { switch tk.Sym { case scan.SymMinus: tk.SetSymbol(scan.SymChangeSign) case scan.SymPlus: tk.SetSymbol(scan.SymUnchangeSign) case scan.SymStar: tk.SetSymbol(scan.SymDereference) case scan.SymExclamation: tk.SetSymbol(scan.SymNot) case scan.SymDoublePlus: tk.SetSymbol(scan.SymPreInc) case scan.SymDoubleMinus: tk.SetSymbol(scan.SymPreDec) } } func (parser *parser) parseGeneral(scanner *scan.Scanner, ctx parserContext, termSymbols ...scan.Symbol) (tree *scan.Ast, err error) { var selectorTerm *scan.Term = nil var currentTerm *scan.Term = nil var tk *scan.Token tree = scan.NewAst() firstToken := true // lastSym := SymUnknown for tk = parser.Next(scanner); err == nil && tk != nil && !tk.IsTerm(termSymbols); /*&& !areSymbolsOutOfCtx(tk, selectorTerm, SymColon, SymDoubleColon)*/ tk = parser.Next(scanner) { // if tk.Sym == SymComment { // continue // } if tk.Sym == scan.SymSemiColon { if hasFlag(ctx, allowMultiExpr) { tree.ToForest() firstToken = true currentTerm = nil selectorTerm = nil continue } else { err = tk.Errorf(`unexpected token %q, expected ",", "]", or ")"`, tk.Source()) break } } //fmt.Println("Token:", tk) if firstToken { changePrefix(tk) firstToken = false } switch tk.Sym { case scan.SymOpenRound: var subTree *scan.Ast if subTree, err = parser.parseGeneral(scanner, ctx, scan.SymClosedRound); err == nil { if subTree.Root() == nil { err = tk.ErrorExpectedGotString("expression", "()") } else { exprTerm := newExprTerm(subTree.Root()) err = tree.AddTerm(exprTerm) currentTerm = exprTerm } } case scan.SymFuncCall: var funcCallTerm *scan.Term if funcCallTerm, err = parser.parseFuncCall(scanner, ctx, tk); err == nil { err = tree.AddTerm(funcCallTerm) currentTerm = funcCallTerm } case scan.SymOpenSquare: var listTerm *scan.Term newCtx := addFlagsCond(addFlags(ctx, squareContext), allowIndex, couldBeACollection(currentTerm)) if listTerm, err = parser.parseList(scanner, newCtx); err == nil { currentTerm, err = listSubTree(tree, listTerm, hasFlag(newCtx, allowIndex)) } case scan.SymOpenBrace: if currentTerm != nil && currentTerm.Symbol() == scan.SymColon { err = currentTerm.Errorf(`selector-case outside of a selector context`) } else { var mapTerm *scan.Term if mapTerm, err = parser.parseDictionary(scanner, ctx); err == nil { err = tree.AddTerm(mapTerm) currentTerm = mapTerm } } case scan.SymEqual, scan.SymPlusEqual, scan.SymMinusEqual, scan.SymStarEqual, scan.SymSlashEqual, scan.SymPercEqual, scan.SymAmpersandEqual, scan.SymVertBarEqual, scan.SymDoubleLessEqual, scan.SymDoubleGreaterEqual, scan.SymCaretEqual: currentTerm, err = tree.AddToken(tk) firstToken = true case scan.SymFuncDef: var funcDefTerm *scan.Term if funcDefTerm, err = parser.parseFuncDef(scanner); err == nil { err = tree.AddTerm(funcDefTerm) currentTerm = funcDefTerm } case scan.SymDollarRound: var iterDefTerm *scan.Term if iterDefTerm, err = parser.parseIterDef(scanner, ctx); err == nil { err = tree.AddTerm(iterDefTerm) currentTerm = iterDefTerm } case scan.SymIdentifier: if tk.Source()[0] == '@' && !hasFlag(ctx, allowVarRef) { err = tk.Errorf("variable references are not allowed in top level expressions: %q", tk.Source()) } else { currentTerm, err = tree.AddToken(tk) } case scan.SymQuestion: if selectorTerm, err = parser.parseSelector(scanner, tree, ctx); err == nil { currentTerm = selectorTerm addFlags(ctx, selectorContext) } case scan.SymColon, scan.SymDoubleColon: var caseTerm *scan.Term if selectorTerm != nil { if caseTerm, err = parser.parseSelectorCase(scanner, ctx, tk.Sym == scan.SymDoubleColon); err == nil { addSelectorCase(selectorTerm, caseTerm) currentTerm = caseTerm if tk.Sym == scan.SymDoubleColon { selectorTerm = nil } } } else { currentTerm, err = tree.AddToken(tk) if tk.IsOneOfA(scan.SymColon, scan.SymRange) { // Colon outside a selector term acts like a separator firstToken = true } } default: currentTerm, err = tree.AddToken(tk) } if currentTerm != nil && currentTerm.Tk.Sym != scan.SymSelector && currentTerm.Parent != nil && currentTerm.Parent.Tk.Sym != scan.SymSelector { selectorTerm = nil remFlags(ctx, selectorContext) } // lastSym = tk.Sym } if err == nil { if !tk.IsOneOf(termSymbols) { var symDesc string if tk.IsSymbol(scan.SymError) { symDesc = tk.ErrorText() } else { symDesc = scan.SymToString(tk.Sym) } err = tk.ErrorExpectedGotStringWithPrefix("expected one of", scan.SymListToString(termSymbols, true), symDesc) } else { err = tk.Error() } } return }