New, more flexible, parser context datatype that includes and extends

the previous flags allowForest and allowVarRef. Added binary operator (work in progress). Better implementation of +=,-=,*=, and /= (new) operators.
2024-12-19 14:48:27 +01:00
parent 5c44532790
commit 6ee21e10af
16 changed files with 537 additions and 140 deletions
@@ -6,10 +6,46 @@ package expr

 import (
 	"errors"
+
+	"golang.org/x/exp/constraints"
 )

 //-------- parser

+type parserContext uint16
+
+const (
+	parserNoFlags                = 0
+	allowMultiExpr parserContext = 1 << iota
+	allowVarRef
+	selectorContext
+	listContext                                // squareContext for list
+	indexContext                               // squareContext for index
+	allowIndex                                 // allow index in squareContext
+	squareContext = listContext | indexContext // Square parenthesis for list or index
+)
+
+func hasFlag[T constraints.Unsigned](set T, singleFlag T) bool {
+	return (set & singleFlag) != 0
+}
+
+func addFlags[T constraints.Unsigned](set T, flags T) T {
+	return set | flags
+}
+
+func addFlagsCond[T constraints.Unsigned](set T, flags T, cond bool) (newSet T) {
+	if cond {
+		newSet = set | flags
+	} else {
+		newSet = set
+	}
+	return
+}
+
+func remFlags[T constraints.Unsigned](set T, flags T) T {
+	return set & (^flags)
+}
+
 type parser struct {
 }

@@ -24,13 +60,13 @@ func (parser *parser) Next(scanner *scanner) (tk *Token) {
 	return
 }

-func (parser *parser) parseFuncCall(scanner *scanner, allowVarRef bool, tk *Token) (tree *term, err error) {
+func (parser *parser) parseFuncCall(scanner *scanner, ctx parserContext, tk *Token) (tree *term, err error) {
 	args := make([]*term, 0, 10)
 	itemExpected := false
 	lastSym := SymUnknown
 	for lastSym != SymClosedRound && lastSym != SymEos {
 		var subTree *ast
-		if subTree, err = parser.parseItem(scanner, allowVarRef, SymComma, SymClosedRound); err != nil {
+		if subTree, err = parser.parseItem(scanner, ctx, SymComma, SymClosedRound); err != nil {
 			break
 		}
 		prev := scanner.Previous()
@@ -77,7 +113,7 @@ func (parser *parser) parseFuncDef(scanner *scanner) (tree *term, err error) {
 			if tk.Sym == SymEqual {
 				var paramExpr *ast
 				defaultParamsStarted = true
-				if paramExpr, err = parser.parseItem(scanner, false, SymComma, SymClosedRound); err != nil {
+				if paramExpr, err = parser.parseItem(scanner, parserNoFlags, SymComma, SymClosedRound); err != nil {
 					break
 				}
 				param.forceChild(paramExpr.root)
@@ -100,7 +136,7 @@ func (parser *parser) parseFuncDef(scanner *scanner) (tree *term, err error) {
 	if err == nil {
 		tk = parser.Next(scanner)
 		if tk.IsSymbol(SymOpenBrace) {
-			body, err = parser.parseGeneral(scanner, true, true, SymClosedBrace)
+			body, err = parser.parseGeneral(scanner, allowMultiExpr|allowVarRef, SymClosedBrace)
 		} else {
 			err = tk.ErrorExpectedGot("{")
 		}
@@ -126,7 +162,7 @@ func paramAlreadyDefined(args []*term, param *term) (position int) {
 	return
 }

-func (parser *parser) parseList(scanner *scanner, parsingIndeces bool, allowVarRef bool) (subtree *term, err error) {
+func (parser *parser) parseList(scanner *scanner, ctx parserContext) (subtree *term, err error) {
 	r, c := scanner.lastPos()
 	args := make([]*term, 0)
 	lastSym := SymUnknown
@@ -134,15 +170,17 @@ func (parser *parser) parseList(scanner *scanner, parsingIndeces bool, allowVarR
 	for lastSym != SymClosedSquare && lastSym != SymEos {
 		var subTree *ast
 		zeroRequired := scanner.current.Sym == SymColon
-		if subTree, err = parser.parseItem(scanner, allowVarRef, SymComma, SymClosedSquare); err == nil {
+		if subTree, err = parser.parseItem(scanner, ctx, SymComma, SymClosedSquare); err == nil {
 			root := subTree.root
 			if root != nil {
-				if !parsingIndeces && root.symbol() == SymColon {
+				//if !parsingIndeces && root.symbol() == SymColon {
+				if !hasFlag(ctx, allowIndex) && root.symbol() == SymColon {
 					err = root.Errorf("unexpected range expression")
 					break
 				}
 				args = append(args, root)
-				if parsingIndeces && root.symbol() == SymColon && zeroRequired { //len(root.children) == 0 {
+				// if parsingIndeces && root.symbol() == SymColon && zeroRequired { //len(root.children) == 0 {
+				if hasFlag(ctx, allowIndex) && root.symbol() == SymColon && zeroRequired { //len(root.children) == 0 {
 					if len(root.children) == 1 {
 						root.children = append(root.children, root.children[0])
 					} else if len(root.children) > 1 {
@@ -175,14 +213,14 @@ func (parser *parser) parseList(scanner *scanner, parsingIndeces bool, allowVarR
 	return
 }

-func (parser *parser) parseIterDef(scanner *scanner, allowVarRef bool) (subtree *term, err error) {
+func (parser *parser) parseIterDef(scanner *scanner, ctx parserContext) (subtree *term, err error) {
 	tk := scanner.Previous()
 	args := make([]*term, 0)
 	lastSym := SymUnknown
 	itemExpected := false
 	for lastSym != SymClosedRound && lastSym != SymEos {
 		var subTree *ast
-		if subTree, err = parser.parseItem(scanner, allowVarRef, SymComma, SymClosedRound); err == nil {
+		if subTree, err = parser.parseItem(scanner, ctx, SymComma, SymClosedRound); err == nil {
 			if subTree.root != nil {
 				args = append(args, subTree.root)
 			} else if itemExpected {
@@ -228,7 +266,7 @@ func (parser *parser) parseDictKey(scanner *scanner) (key any, err error) {
 	return
 }

-func (parser *parser) parseDictionary(scanner *scanner, allowVarRef bool) (subtree *term, err error) {
+func (parser *parser) parseDictionary(scanner *scanner, ctx parserContext) (subtree *term, err error) {
 	args := make(map[any]*term, 0)
 	lastSym := SymUnknown
 	itemExpected := false
@@ -245,7 +283,7 @@ func (parser *parser) parseDictionary(scanner *scanner, allowVarRef bool) (subtr
 			}
 			break
 		}
-		if subTree, err = parser.parseItem(scanner, allowVarRef, SymComma, SymClosedBrace); err == nil {
+		if subTree, err = parser.parseItem(scanner, ctx, SymComma, SymClosedBrace); err == nil {
 			if subTree.root != nil {
 				args[key] = subTree.root
 			} else /*if key != nil*/ {
@@ -270,7 +308,7 @@ func (parser *parser) parseDictionary(scanner *scanner, allowVarRef bool) (subtr
 	return
 }

-func (parser *parser) parseSelectorCase(scanner *scanner, allowVarRef bool, defaultCase bool) (caseTerm *term, err error) {
+func (parser *parser) parseSelectorCase(scanner *scanner, ctx parserContext, defaultCase bool) (caseTerm *term, err error) {
 	var filterList *term
 	var caseExpr *ast
 	tk := parser.Next(scanner)
@@ -281,7 +319,7 @@ func (parser *parser) parseSelectorCase(scanner *scanner, allowVarRef bool, defa
 			err = tk.Errorf("case list in default clause")
 			return
 		}
-		if filterList, err = parser.parseList(scanner, false, allowVarRef); err != nil {
+		if filterList, err = parser.parseList(scanner, remFlags(ctx, allowIndex)); err != nil {
 			return
 		}
 		tk = parser.Next(scanner)
@@ -292,7 +330,7 @@ func (parser *parser) parseSelectorCase(scanner *scanner, allowVarRef bool, defa
 	}

 	if tk.Sym == SymOpenBrace {
-		if caseExpr, err = parser.parseGeneral(scanner, true, allowVarRef, SymClosedBrace); err != nil {
+		if caseExpr, err = parser.parseGeneral(scanner, ctx|allowMultiExpr, SymClosedBrace); err != nil {
 			return
 		}
 	} else {
@@ -318,26 +356,26 @@ func addSelectorCase(selectorTerm, caseTerm *term) {
 	caseTerm.parent = selectorTerm
 }

-func (parser *parser) parseSelector(scanner *scanner, tree *ast, allowVarRef bool) (selectorTerm *term, err error) {
+func (parser *parser) parseSelector(scanner *scanner, tree *ast, ctx parserContext) (selectorTerm *term, err error) {
 	var caseTerm *term
 	tk := scanner.makeToken(SymSelector, '?')
 	if selectorTerm, err = tree.addToken(tk); err != nil {
 		return
 	}

-	if caseTerm, err = parser.parseSelectorCase(scanner, allowVarRef, false); err == nil {
+	if caseTerm, err = parser.parseSelectorCase(scanner, ctx|allowVarRef, false); err == nil {
 		addSelectorCase(selectorTerm, caseTerm)
 	}
 	return
 }

-func (parser *parser) parseItem(scanner *scanner, allowVarRef bool, termSymbols ...Symbol) (tree *ast, err error) {
-	return parser.parseGeneral(scanner, false, allowVarRef, termSymbols...)
+func (parser *parser) parseItem(scanner *scanner, ctx parserContext, termSymbols ...Symbol) (tree *ast, err error) {
+	return parser.parseGeneral(scanner, ctx|allowVarRef, termSymbols...)
 }

 func (parser *parser) Parse(scanner *scanner, termSymbols ...Symbol) (tree *ast, err error) {
 	termSymbols = append(termSymbols, SymEos)
-	return parser.parseGeneral(scanner, true, false, termSymbols...)
+	return parser.parseGeneral(scanner, allowMultiExpr, termSymbols...)
 }

 func couldBeACollection(t *term) bool {
@@ -356,7 +394,7 @@ func couldBeACollection(t *term) bool {
 // 	return areOut
 // }

-func (parser *parser) parseGeneral(scanner *scanner, allowForest bool, allowVarRef bool, termSymbols ...Symbol) (tree *ast, err error) {
+func (parser *parser) parseGeneral(scanner *scanner, ctx parserContext, termSymbols ...Symbol) (tree *ast, err error) {
 	var selectorTerm *term = nil
 	var currentTerm *term = nil
 	var tk *Token
@@ -370,7 +408,7 @@ func (parser *parser) parseGeneral(scanner *scanner, allowForest bool, allowVarR
 		// }

 		if tk.Sym == SymSemiColon {
-			if allowForest {
+			if hasFlag(ctx, allowMultiExpr) {
 				tree.ToForest()
 				firstToken = true
 				currentTerm = nil
@@ -395,22 +433,22 @@ func (parser *parser) parseGeneral(scanner *scanner, allowForest bool, allowVarR
 		switch tk.Sym {
 		case SymOpenRound:
 			var subTree *ast
-			if subTree, err = parser.parseGeneral(scanner, false, allowVarRef, SymClosedRound); err == nil {
+			if subTree, err = parser.parseGeneral(scanner, ctx, SymClosedRound); err == nil {
 				subTree.root.priority = priValue
 				err = tree.addTerm(newExprTerm(subTree.root))
 				currentTerm = subTree.root
 			}
 		case SymFuncCall:
 			var funcCallTerm *term
-			if funcCallTerm, err = parser.parseFuncCall(scanner, allowVarRef, tk); err == nil {
+			if funcCallTerm, err = parser.parseFuncCall(scanner, ctx, tk); err == nil {
 				err = tree.addTerm(funcCallTerm)
 				currentTerm = funcCallTerm
 			}
 		case SymOpenSquare:
 			var listTerm *term
-			parsingIndeces := couldBeACollection(currentTerm)
-			if listTerm, err = parser.parseList(scanner, parsingIndeces, allowVarRef); err == nil {
-				if parsingIndeces {
+			newCtx := addFlagsCond(addFlags(ctx, squareContext), allowIndex, couldBeACollection(currentTerm))
+			if listTerm, err = parser.parseList(scanner, newCtx); err == nil {
+				if hasFlag(newCtx, allowIndex) {
 					indexTk := NewToken(listTerm.tk.row, listTerm.tk.col, SymIndex, listTerm.source())
 					indexTerm := newTerm(indexTk)
 					if err = tree.addTerm(indexTerm); err == nil {
@@ -426,7 +464,7 @@ func (parser *parser) parseGeneral(scanner *scanner, allowForest bool, allowVarR
 				err = currentTerm.Errorf(`selector-case outside of a selector context`)
 			} else {
 				var mapTerm *term
-				if mapTerm, err = parser.parseDictionary(scanner, allowVarRef); err == nil {
+				if mapTerm, err = parser.parseDictionary(scanner, ctx); err == nil {
 					err = tree.addTerm(mapTerm)
 					currentTerm = mapTerm
 				}
@@ -444,24 +482,24 @@ func (parser *parser) parseGeneral(scanner *scanner, allowForest bool, allowVarR
 			}
 		case SymDollarRound:
 			var iterDefTerm *term
-			if iterDefTerm, err = parser.parseIterDef(scanner, allowVarRef); err == nil {
+			if iterDefTerm, err = parser.parseIterDef(scanner, ctx); err == nil {
 				err = tree.addTerm(iterDefTerm)
 				currentTerm = iterDefTerm
 			}
 		case SymIdentifier:
-			if tk.source[0] == '@' && !allowVarRef {
+			if tk.source[0] == '@' && !hasFlag(ctx, allowVarRef) {
 				err = tk.Errorf("variable references are not allowed in top level expressions: %q", tk.source)
 			} else {
 				currentTerm, err = tree.addToken(tk)
 			}
 		case SymQuestion:
-			if selectorTerm, err = parser.parseSelector(scanner, tree, allowVarRef); err == nil {
+			if selectorTerm, err = parser.parseSelector(scanner, tree, ctx); err == nil {
 				currentTerm = selectorTerm
 			}
 		case SymColon, SymDoubleColon:
 			var caseTerm *term
 			if selectorTerm != nil {
-				if caseTerm, err = parser.parseSelectorCase(scanner, allowVarRef, tk.Sym == SymDoubleColon); err == nil {
+				if caseTerm, err = parser.parseSelectorCase(scanner, ctx, tk.Sym == SymDoubleColon); err == nil {
 					addSelectorCase(selectorTerm, caseTerm)
 					currentTerm = caseTerm
 					if tk.Sym == SymDoubleColon {
@@ -475,8 +513,16 @@ func (parser *parser) parseGeneral(scanner *scanner, allowForest bool, allowVarR
 				// Colon outside a selector term acts like a separator
 				firstToken = true
 			}
-		case SymPlusEqual, SymMinusEqual, SymStarEqual:
-			currentTerm, err = parser.expandOpAssign(scanner, tree, tk, allowVarRef)
+		// case SymPlusEqual:
+		// 	opTk := NewToken(tk.row, tk.col, SymEqual, "=")
+		// 	if _, err = tree.addToken(opTk); err == nil {
+		// 		if err = tree.addTerm(tree.root.Clone()); err == nil {
+		// 			opTk = NewToken(tk.row, tk.col, SymPlus, "+")
+		// 			currentTerm, err = tree.addToken(opTk)
+		// 		}
+		// 	}
+		// case SymStarEqual:
+		// 	currentTerm, err = parser.expandOpAssign(scanner, tree, tk, ctx, termSymbols)
 		default:
 			currentTerm, err = tree.addToken(tk)
 		}
@@ -488,13 +534,23 @@ func (parser *parser) parseGeneral(scanner *scanner, allowForest bool, allowVarR
 		// lastSym = tk.Sym
 	}

-	if err == nil && !tk.IsOneOf(termSymbols) {
-		err = tk.ErrorExpectedGotStringWithPrefix("expected one of", SymListToString(termSymbols, false), SymToString(tk.Sym))
+	if err == nil {
+		if !tk.IsOneOf(termSymbols) {
+			var symDesc string
+			if tk.IsSymbol(SymError) {
+				symDesc = tk.ErrorText()
+			} else {
+				symDesc = SymToString(tk.Sym)
+			}
+			err = tk.ErrorExpectedGotStringWithPrefix("expected one of", SymListToString(termSymbols, true), symDesc)
+		} else {
+			err = tk.Error()
+		}
 	}

-	if err == nil {
-		err = tk.Error()
-	}
+	// if err == nil {
+	// 	err = tk.Error()
+	// }
 	return
 }

@@ -505,48 +561,49 @@ func (parser *parser) parseGeneral(scanner *scanner, allowForest bool, allowVarR
 // 	return
 // }

-func (parser *parser) expandOpAssign(scanner *scanner, tree *ast, tk *Token, allowVarRef bool) (t *term, err error) {
-	var opSym Symbol
-	var opString string
+// func (parser *parser) expandOpAssign(scanner *scanner, tree *ast, tk *Token, ctx parserContext, termSymbols []Symbol) (t *term, err error) {
+// 	var opSym Symbol
+// 	var opString string

-	if tree.root != nil {
-		switch tk.Sym {
-		case SymPlusEqual:
-			opString = "+"
-			opSym = SymPlus
-		case SymMinusEqual:
-			opString = "-"
-			opSym = SymMinus
-		case SymStarEqual:
-			opString = "*"
-			opSym = SymStar
-		default:
-			err = tk.Errorf("unsopported operator %q", tk.source)
-			return
-		}
-		leftExpr := tree.root.Clone()
-		leftExpr.setParent(nil)
-		if t, err = tree.addToken(NewToken(tk.row, tk.col, SymEqual, "=")); err == nil {
-			t = leftExpr
-			if err = tree.addTerm(leftExpr); err == nil {
-				if t, err = tree.addToken(NewToken(tk.row, tk.col, opSym, opString)); err == nil {
+// 	if tree.root != nil {
+// 		switch tk.Sym {
+// 		case SymPlusEqual:
+// 			opString = "+"
+// 			opSym = SymPlus
+// 		case SymMinusEqual:
+// 			opString = "-"
+// 			opSym = SymMinus
+// 		case SymStarEqual:
+// 			opString = "*"
+// 			opSym = SymStar
+// 		default:
+// 			err = tk.Errorf("unsopported operator %q", tk.source)
+// 			return
+// 		}
+// 		leftExpr := tree.root.Clone()
+// 		leftExpr.setParent(nil)
+// 		if t, err = tree.addToken(NewToken(tk.row, tk.col, SymEqual, "=")); err == nil {
+// 			t = leftExpr
+// 			if err = tree.addTerm(leftExpr); err == nil {
+// 				if t, err = tree.addToken(NewToken(tk.row, tk.col, opSym, opString)); err == nil {

-					var subTree *ast
-					if subTree, err = parser.parseGeneral(scanner, false, allowVarRef, SymSemiColon, SymClosedRound, SymClosedBrace, SymClosedSquare); err == nil {
-						if scanner.Previous().IsOneOfA(SymSemiColon, SymClosedRound, SymClosedBrace, SymClosedSquare) {
-							if err = scanner.UnreadToken(); err != nil {
-								return
-							}
-						}
-						subTree.root.priority = priValue
-						err = tree.addTerm(newExprTerm(subTree.root))
-						t = subTree.root
-					}
-				}
-			}
-		}
-	} else {
-		err = tk.Errorf("left operand of %q must be a variable or a variable expression", tk)
-	}
-	return
-}
+// 					var subTree *ast
+// 					// if subTree, err = parser.parseGeneral(scanner, ctx, SymEos, SymSemiColon, SymClosedRound, SymClosedBrace, SymClosedSquare); err == nil {
+// 					if subTree, err = parser.parseGeneral(scanner, ctx, termSymbols...); err == nil {
+// 						if scanner.Previous().IsOneOfA(SymSemiColon, SymClosedRound, SymClosedBrace, SymClosedSquare) {
+// 							if err = scanner.UnreadToken(); err != nil {
+// 								return
+// 							}
+// 						}
+// 						subTree.root.priority = priValue
+// 						err = tree.addTerm(newExprTerm(subTree.root))
+// 						t = subTree.root
+// 					}
+// 				}
+// 			}
+// 		}
+// 	} else {
+// 		err = tk.Errorf("left operand of %q must be a variable or a variable expression", tk)
+// 	}
+// 	return
+// }