moved scanner sources to package 'scan'

parser.go

@@ -7,6 +7,7 @@ package expr
 import (
 	"errors"
 
+	"git.portale-stac.it/go-pkg/expr/scan"
 	"golang.org/x/exp/constraints"
 )
 
@@ -54,34 +55,34 @@ func NewParser() (p *parser) {
 	return p
 }
 
-func (parser *parser) Next(scanner *scanner) (tk *Token) {
-	for tk = scanner.Next(); tk.IsSymbol(SymComment); tk = scanner.Next() {
+func (parser *parser) Next(scanner *scan.Scanner) (tk *scan.Token) {
+	for tk = scanner.Next(); tk.IsSymbol(scan.SymComment); tk = scanner.Next() {
 	}
 	return
 }
 
-func (parser *parser) parseFuncCall(scanner *scanner, ctx parserContext, tk *Token) (tree *term, err error) {
-	args := make([]*term, 0, 10)
+func (parser *parser) parseFuncCall(scanner *scan.Scanner, ctx parserContext, tk *scan.Token) (tree *scan.Term, err error) {
+	args := make([]*scan.Term, 0, 10)
 	itemExpected := false
-	lastSym := SymUnknown
-	for lastSym != SymClosedRound && lastSym != SymEos {
-		var subTree *ast
-		if subTree, err = parser.parseItem(scanner, ctx, SymComma, SymClosedRound); err != nil {
+	lastSym := scan.SymUnknown
+	for lastSym != scan.SymClosedRound && lastSym != scan.SymEos {
+		var subTree *scan.Ast
+		if subTree, err = parser.parseItem(scanner, ctx, scan.SymComma, scan.SymClosedRound); err != nil {
 			break
 		}
 		prev := scanner.Previous()
-		if subTree.root != nil {
-			args = append(args, subTree.root)
+		if subTree.Root() != nil {
+			args = append(args, subTree.Root())
 		} else if itemExpected {
 			err = prev.ErrorExpectedGot("function-param-value")
 			break
 		}
 
-		itemExpected = prev.Sym == SymComma
+		itemExpected = prev.Sym == scan.SymComma
 		lastSym = scanner.Previous().Sym
 	}
 	if err == nil {
-		if lastSym != SymClosedRound {
+		if lastSym != scan.SymClosedRound {
 			err = errors.New("unterminated arguments list")
 		} else {
 			tree = newFuncCallTerm(tk, args)
@@ -90,18 +91,18 @@ func (parser *parser) parseFuncCall(scanner *scanner, ctx parserContext, tk *Tok
 	return
 }
 
-func (parser *parser) parseFuncDef(scanner *scanner) (tree *term, err error) {
+func (parser *parser) parseFuncDef(scanner *scan.Scanner) (tree *scan.Term, err error) {
 	// Example: "add = func(x,y) {x+y}
-	var body *ast
-	args := make([]*term, 0)
-	lastSym := SymUnknown
+	var body *scan.Ast
+	args := make([]*scan.Term, 0)
+	lastSym := scan.SymUnknown
 	defaultParamsStarted := false
 	itemExpected := false
 	tk := scanner.Previous()
-	for lastSym != SymClosedRound && lastSym != SymEos {
+	for lastSym != scan.SymClosedRound && lastSym != scan.SymEos {
 		tk = parser.Next(scanner)
-		if tk.IsSymbol(SymIdentifier) {
-			param := newTerm(tk)
+		if tk.IsSymbol(scan.SymIdentifier) {
+			param := scan.NewTerm(tk)
 			if len(args) > 0 {
 				if pos := paramAlreadyDefined(args, param); pos > 0 {
 					err = tk.Errorf("parameter %q at position %d already defined at position %d", param.Source(), len(args)+1, pos)
@@ -110,13 +111,13 @@ func (parser *parser) parseFuncDef(scanner *scanner) (tree *term, err error) {
 			}
 			args = append(args, param)
 			tk = parser.Next(scanner)
-			if tk.Sym == SymEqual {
-				var paramExpr *ast
+			if tk.Sym == scan.SymEqual {
+				var paramExpr *scan.Ast
 				defaultParamsStarted = true
-				if paramExpr, err = parser.parseItem(scanner, parserNoFlags, SymComma, SymClosedRound); err != nil {
+				if paramExpr, err = parser.parseItem(scanner, parserNoFlags, scan.SymComma, scan.SymClosedRound); err != nil {
 					break
 				}
-				param.forceChild(paramExpr.root)
+				param.ForceChild(paramExpr.Root())
 			} else if defaultParamsStarted {
 				err = tk.Errorf("can't mix default and non-default parameters")
 				break
@@ -127,32 +128,32 @@ func (parser *parser) parseFuncDef(scanner *scanner) (tree *term, err error) {
 			break
 		}
 		lastSym = scanner.Previous().Sym
-		itemExpected = lastSym == SymComma
+		itemExpected = lastSym == scan.SymComma
 	}
 
-	if err == nil && lastSym != SymClosedRound {
+	if err == nil && lastSym != scan.SymClosedRound {
 		err = tk.ErrorExpectedGot(")")
 	}
 	if err == nil {
 		tk = parser.Next(scanner)
-		if tk.IsSymbol(SymOpenBrace) {
-			body, err = parser.parseGeneral(scanner, allowMultiExpr|allowVarRef, SymClosedBrace)
+		if tk.IsSymbol(scan.SymOpenBrace) {
+			body, err = parser.parseGeneral(scanner, allowMultiExpr|allowVarRef, scan.SymClosedBrace)
 		} else {
 			err = tk.ErrorExpectedGot("{")
 		}
 	}
 	if err == nil {
-		if scanner.Previous().Sym != SymClosedBrace {
+		if scanner.Previous().Sym != scan.SymClosedBrace {
 			err = scanner.Previous().ErrorExpectedGot("}")
 		} else {
-			tk = scanner.makeValueToken(SymExpression, "", body)
+			tk = scanner.MakeValueToken(scan.SymExpression, "", body)
 			tree = newFuncDefTerm(tk, args)
 		}
 	}
 	return
 }
 
-func paramAlreadyDefined(args []*term, param *term) (position int) {
+func paramAlreadyDefined(args []*scan.Term, param *scan.Term) (position int) {
 	position = 0
 	for i, arg := range args {
 		if arg.Source() == param.Source() {
@@ -162,39 +163,39 @@ func paramAlreadyDefined(args []*term, param *term) (position int) {
 	return
 }
 
-func (parser *parser) parseList(scanner *scanner, ctx parserContext) (listTerm *term, err error) {
-	r, c := scanner.lastPos()
-	args := make([]*term, 0)
-	lastSym := SymUnknown
+func (parser *parser) parseList(scanner *scan.Scanner, ctx parserContext) (listTerm *scan.Term, err error) {
+	r, c := scanner.LastPos()
+	args := make([]*scan.Term, 0)
+	lastSym := scan.SymUnknown
 	itemExpected := false
 	itemCtx := remFlags(ctx, allowIndex)
-	for lastSym != SymClosedSquare && lastSym != SymEos {
-		zeroRequired := scanner.current.Sym == SymColon
-		var itemTree *ast
-		if itemTree, err = parser.parseItem(scanner, itemCtx, SymComma, SymClosedSquare); err == nil {
-			root := itemTree.root
+	for lastSym != scan.SymClosedSquare && lastSym != scan.SymEos {
+		zeroRequired := scanner.Current().Sym == scan.SymColon
+		var itemTree *scan.Ast
+		if itemTree, err = parser.parseItem(scanner, itemCtx, scan.SymComma, scan.SymClosedSquare); err == nil {
+			root := itemTree.Root()
 			if root != nil {
-				if hasFlag(ctx, allowIndex) && root.symbol() == SymColon {
+				if hasFlag(ctx, allowIndex) && root.Symbol() == scan.SymColon {
 					changeColonToRange(root)
 				}
-				if !hasFlag(ctx, allowIndex) && root.symbol() == SymRange {
+				if !hasFlag(ctx, allowIndex) && root.Symbol() == scan.SymRange {
 					// err = root.Errorf("unexpected range expression")
 					err = errRangeUnexpectedExpression(root)
 					break
 				}
 				args = append(args, root)
-				if hasFlag(ctx, allowIndex) && root.symbol() == SymRange && zeroRequired { //len(root.children) == 0 {
-					if len(root.children) == 1 {
-						root.children = append(root.children, root.children[0])
-					} else if len(root.children) > 1 {
+				if hasFlag(ctx, allowIndex) && root.Symbol() == scan.SymRange && zeroRequired { //len(root.children) == 0 {
+					if len(root.Children) == 1 {
+						root.Children = append(root.Children, root.Children[0])
+					} else if len(root.Children) > 1 {
 						// err = root.Errorf("invalid range specification")
 						err = errRangeInvalidSpecification(root)
 						break
 					}
-					zeroTk := NewValueToken(root.tk.row, root.tk.col, SymInteger, "0", int64(0))
-					zeroTerm := newTerm(zeroTk)
-					zeroTerm.setParent(root)
-					root.children[0] = zeroTerm
+					zeroTk := scan.NewValueToken(root.Tk.Row(), root.Tk.Col(), scan.SymInteger, "0", int64(0))
+					zeroTerm := scan.NewTerm(zeroTk)
+					zeroTerm.SetParent(root)
+					root.Children[0] = zeroTerm
 				}
 			} else if itemExpected {
 				prev := scanner.Previous()
@@ -205,12 +206,12 @@ func (parser *parser) parseList(scanner *scanner, ctx parserContext) (listTerm *
 			break
 		}
 		lastSym = scanner.Previous().Sym
-		if itemExpected = lastSym == SymComma; itemExpected {
+		if itemExpected = lastSym == scan.SymComma; itemExpected {
 			remFlags(ctx, allowIndex)
 		}
 	}
 	if err == nil {
-		if lastSym != SymClosedSquare {
+		if lastSym != scan.SymClosedSquare {
 			err = scanner.Previous().ErrorExpectedGot("]")
 		} else {
 			listTerm = newListTerm(r, c, args)
@@ -219,16 +220,16 @@ func (parser *parser) parseList(scanner *scanner, ctx parserContext) (listTerm *
 	return
 }
 
-func (parser *parser) parseIterDef(scanner *scanner, ctx parserContext) (subtree *term, err error) {
+func (parser *parser) parseIterDef(scanner *scan.Scanner, ctx parserContext) (subtree *scan.Term, err error) {
 	tk := scanner.Previous()
-	args := make([]*term, 0)
-	lastSym := SymUnknown
+	args := make([]*scan.Term, 0)
+	lastSym := scan.SymUnknown
 	itemExpected := false
-	for lastSym != SymClosedRound && lastSym != SymEos {
-		var subTree *ast
-		if subTree, err = parser.parseItem(scanner, ctx, SymComma, SymClosedRound); err == nil {
-			if subTree.root != nil {
-				args = append(args, subTree.root)
+	for lastSym != scan.SymClosedRound && lastSym != scan.SymEos {
+		var subTree *scan.Ast
+		if subTree, err = parser.parseItem(scanner, ctx, scan.SymComma, scan.SymClosedRound); err == nil {
+			if subTree.Root() != nil {
+				args = append(args, subTree.Root())
 			} else if itemExpected {
 				prev := scanner.Previous()
 				err = prev.ErrorExpectedGot("iterator-param")
@@ -238,10 +239,10 @@ func (parser *parser) parseIterDef(scanner *scanner, ctx parserContext) (subtree
 			break
 		}
 		lastSym = scanner.Previous().Sym
-		itemExpected = lastSym == SymComma
+		itemExpected = lastSym == scan.SymComma
 	}
 	if err == nil {
-		if lastSym != SymClosedRound {
+		if lastSym != scan.SymClosedRound {
 			err = scanner.Previous().ErrorExpectedGot(")")
 		} else {
 			subtree = newIteratorTerm(tk, args)
@@ -250,18 +251,18 @@ func (parser *parser) parseIterDef(scanner *scanner, ctx parserContext) (subtree
 	return
 }
 
-func (parser *parser) parseDictKey(scanner *scanner) (key any, err error) {
+func (parser *parser) parseDictKey(scanner *scan.Scanner) (key any, err error) {
 	tk := parser.Next(scanner)
-	if tk.Sym == SymError {
+	if tk.Sym == scan.SymError {
 		err = tk.Error()
 		return
 	}
-	if tk.Sym == SymClosedBrace || tk.Sym == SymEos {
+	if tk.Sym == scan.SymClosedBrace || tk.Sym == scan.SymEos {
 		return
 	}
-	if tk.Sym == SymInteger || tk.Sym == SymString {
+	if tk.Sym == scan.SymInteger || tk.Sym == scan.SymString {
 		tkSep := parser.Next(scanner)
-		if tkSep.Sym != SymColon {
+		if tkSep.Sym != scan.SymColon {
 			err = tkSep.ErrorExpectedGot(":")
 		} else {
 			key = tk.Value
@@ -272,12 +273,12 @@ func (parser *parser) parseDictKey(scanner *scanner) (key any, err error) {
 	return
 }
 
-func (parser *parser) parseDictionary(scanner *scanner, ctx parserContext) (subtree *term, err error) {
-	args := make(map[any]*term, 0)
-	lastSym := SymUnknown
+func (parser *parser) parseDictionary(scanner *scan.Scanner, ctx parserContext) (subtree *scan.Term, err error) {
+	args := make(map[any]*scan.Term, 0)
+	lastSym := scan.SymUnknown
 	itemExpected := false
-	for lastSym != SymClosedBrace && lastSym != SymEos {
-		var subTree *ast
+	for lastSym != scan.SymClosedBrace && lastSym != scan.SymEos {
+		var subTree *scan.Ast
 		var key any
 		if key, err = parser.parseDictKey(scanner); err != nil {
 			break
@@ -289,9 +290,9 @@ func (parser *parser) parseDictionary(scanner *scanner, ctx parserContext) (subt
 			}
 			break
 		}
-		if subTree, err = parser.parseItem(scanner, ctx, SymComma, SymClosedBrace); err == nil {
-			if subTree.root != nil {
-				args[key] = subTree.root
+		if subTree, err = parser.parseItem(scanner, ctx, scan.SymComma, scan.SymClosedBrace); err == nil {
+			if subTree.Root() != nil {
+				args[key] = subTree.Root()
 			} else /*if key != nil*/ {
 				prev := scanner.Previous()
 				err = prev.ErrorExpectedGot("dictionary-value")
@@ -301,10 +302,10 @@ func (parser *parser) parseDictionary(scanner *scanner, ctx parserContext) (subt
 			break
 		}
 		lastSym = scanner.Previous().Sym
-		itemExpected = lastSym == SymComma
+		itemExpected = lastSym == scan.SymComma
 	}
 	if err == nil {
-		if lastSym != SymClosedBrace {
+		if lastSym != scan.SymClosedBrace {
 			err = scanner.Previous().ErrorExpectedGot("}")
 		} else {
 			subtree = newDictTerm(args)
@@ -313,14 +314,14 @@ func (parser *parser) parseDictionary(scanner *scanner, ctx parserContext) (subt
 	return
 }
 
-func (parser *parser) parseSelectorCase(scanner *scanner, ctx parserContext, defaultCase bool) (caseTerm *term, err error) {
-	var filterList *term
-	var caseExpr *ast
+func (parser *parser) parseSelectorCase(scanner *scan.Scanner, ctx parserContext, defaultCase bool) (caseTerm *scan.Term, err error) {
+	var filterList *scan.Term
+	var caseExpr *scan.Ast
 	ctx = remFlags(ctx, allowIndex)
 	tk := parser.Next(scanner)
-	startRow := tk.row
-	startCol := tk.col
-	if tk.Sym == SymOpenSquare {
+	startRow := tk.Row()
+	startCol := tk.Col()
+	if tk.Sym == scan.SymOpenSquare {
 		if defaultCase {
 			err = tk.Errorf("case list in default clause")
 			return
@@ -329,14 +330,14 @@ func (parser *parser) parseSelectorCase(scanner *scanner, ctx parserContext, def
 			return
 		}
 		tk = parser.Next(scanner)
-		startRow = tk.row
-		startCol = tk.col
+		startRow = tk.Row()
+		startCol = tk.Col()
 	} else if !defaultCase {
-		filterList = newListTerm(startRow, startCol, make([]*term, 0))
+		filterList = newListTerm(startRow, startCol, make([]*scan.Term, 0))
 	}
 
-	if tk.Sym == SymOpenBrace {
-		if caseExpr, err = parser.parseGeneral(scanner, ctx|allowMultiExpr, SymClosedBrace); err != nil {
+	if tk.Sym == scan.SymOpenBrace {
+		if caseExpr, err = parser.parseGeneral(scanner, ctx|allowMultiExpr, scan.SymClosedBrace); err != nil {
 			return
 		}
 	} else {
@@ -349,25 +350,25 @@ func (parser *parser) parseSelectorCase(scanner *scanner, ctx parserContext, def
 	return
 }
 
-func addSelectorCase(selectorTerm, caseTerm *term) {
-	if len(selectorTerm.children) < 2 {
+func addSelectorCase(selectorTerm, caseTerm *scan.Term) {
+	if len(selectorTerm.Children) < 2 {
 		caseListTerm := newListTermA(caseTerm)
-		selectorTerm.children = append(selectorTerm.children, caseListTerm)
+		selectorTerm.Children = append(selectorTerm.Children, caseListTerm)
 	} else {
-		caseListTerm := selectorTerm.children[1]
-		caseList, _ := caseListTerm.value().([]*term)
+		caseListTerm := selectorTerm.Children[1]
+		caseList, _ := caseListTerm.Value().([]*scan.Term)
 		caseList = append(caseList, caseTerm)
-		caseListTerm.tk.Value = caseList
+		caseListTerm.Tk.Value = caseList
 	}
-	caseTerm.parent = selectorTerm
+	caseTerm.Parent = selectorTerm
 }
 
-func (parser *parser) parseSelector(scanner *scanner, tree *ast, ctx parserContext) (selectorTerm *term, err error) {
-	var caseTerm *term
+func (parser *parser) parseSelector(scanner *scan.Scanner, tree *scan.Ast, ctx parserContext) (selectorTerm *scan.Term, err error) {
+	var caseTerm *scan.Term
 
 	ctx = remFlags(ctx, allowIndex)
-	tk := scanner.makeToken(SymSelector, '?')
-	if selectorTerm, err = tree.addToken(tk); err != nil {
+	tk := scanner.MakeToken(scan.SymSelector, '?')
+	if selectorTerm, err = tree.AddToken(tk); err != nil {
 		return
 	}
 
@@ -377,11 +378,11 @@ func (parser *parser) parseSelector(scanner *scanner, tree *ast, ctx parserConte
 	return
 }
 
-func (parser *parser) parseItem(scanner *scanner, ctx parserContext, termSymbols ...Symbol) (tree *ast, err error) {
+func (parser *parser) parseItem(scanner *scan.Scanner, ctx parserContext, termSymbols ...scan.Symbol) (tree *scan.Ast, err error) {
 	return parser.parseGeneral(scanner, ctx|allowVarRef, termSymbols...)
 }
 
-func (parser *parser) Parse(scanner *scanner, termSymbols ...Symbol) (tree *ast, err error) {
+func (parser *parser) Parse(scanner *scan.Scanner, termSymbols ...scan.Symbol) (tree *scan.Ast, err error) {
 	defer func() {
 		if r := recover(); r != nil {
 			if errVal, ok := r.(error); ok {
@@ -391,56 +392,56 @@ func (parser *parser) Parse(scanner *scanner, termSymbols ...Symbol) (tree *ast,
 			}
 		}
 	}()
-	termSymbols = append(termSymbols, SymEos)
+	termSymbols = append(termSymbols, scan.SymEos)
 	return parser.parseGeneral(scanner, allowMultiExpr, termSymbols...)
 }
 
-func couldBeACollection(t *term) bool {
-	var sym = SymUnknown
+func couldBeACollection(t *scan.Term) bool {
+	var sym = scan.SymUnknown
 	if t != nil {
-		sym = t.symbol()
+		sym = t.Symbol()
 	}
-	return sym == SymList || sym == SymString || sym == SymDict || sym == SymExpression || sym == SymVariable
+	return sym == scan.SymList || sym == scan.SymString || sym == scan.SymDict || sym == scan.SymExpression || sym == scan.SymVariable
 }
 
-func listSubTree(tree *ast, listTerm *term, allowIndeces bool) (root *term, err error) {
-	var tk *Token
+func listSubTree(tree *scan.Ast, listTerm *scan.Term, allowIndeces bool) (root *scan.Term, err error) {
+	var tk *scan.Token
 	if allowIndeces {
-		tk = NewToken(listTerm.tk.row, listTerm.tk.col, SymIndex, listTerm.Source())
-		root = newTerm(tk)
-		if err = tree.addTerm(root); err == nil {
-			err = tree.addTerm(listTerm)
+		tk = scan.NewToken(listTerm.Tk.Row(), listTerm.Tk.Col(), scan.SymIndex, listTerm.Source())
+		root = scan.NewTerm(tk)
+		if err = tree.AddTerm(root); err == nil {
+			err = tree.AddTerm(listTerm)
 		}
 	} else {
 		root = listTerm
-		err = tree.addTerm(listTerm)
+		err = tree.AddTerm(listTerm)
 	}
 	return
 }
 
-func changePrefix(tk *Token) {
+func changePrefix(tk *scan.Token) {
 	switch tk.Sym {
-	case SymMinus:
-		tk.SetSymbol(SymChangeSign)
-	case SymPlus:
-		tk.SetSymbol(SymUnchangeSign)
-	case SymStar:
-		tk.SetSymbol(SymDereference)
-	case SymExclamation:
-		tk.SetSymbol(SymNot)
-	case SymDoublePlus:
-		tk.SetSymbol(SymPreInc)
-	case SymDoubleMinus:
-		tk.SetSymbol(SymPreDec)
+	case scan.SymMinus:
+		tk.SetSymbol(scan.SymChangeSign)
+	case scan.SymPlus:
+		tk.SetSymbol(scan.SymUnchangeSign)
+	case scan.SymStar:
+		tk.SetSymbol(scan.SymDereference)
+	case scan.SymExclamation:
+		tk.SetSymbol(scan.SymNot)
+	case scan.SymDoublePlus:
+		tk.SetSymbol(scan.SymPreInc)
+	case scan.SymDoubleMinus:
+		tk.SetSymbol(scan.SymPreDec)
 	}
 }
 
-func (parser *parser) parseGeneral(scanner *scanner, ctx parserContext, termSymbols ...Symbol) (tree *ast, err error) {
-	var selectorTerm *term = nil
-	var currentTerm *term = nil
-	var tk *Token
+func (parser *parser) parseGeneral(scanner *scan.Scanner, ctx parserContext, termSymbols ...scan.Symbol) (tree *scan.Ast, err error) {
+	var selectorTerm *scan.Term = nil
+	var currentTerm *scan.Term = nil
+	var tk *scan.Token
 
-	tree = NewAst()
+	tree = scan.NewAst()
 	firstToken := true
 	// lastSym := SymUnknown
 	for tk = parser.Next(scanner); err == nil && tk != nil && !tk.IsTerm(termSymbols); /*&& !areSymbolsOutOfCtx(tk, selectorTerm, SymColon, SymDoubleColon)*/ tk = parser.Next(scanner) {
@@ -448,7 +449,7 @@ func (parser *parser) parseGeneral(scanner *scanner, ctx parserContext, termSymb
 		// 	continue
 		// }
 
-		if tk.Sym == SymSemiColon {
+		if tk.Sym == scan.SymSemiColon {
 			if hasFlag(ctx, allowMultiExpr) {
 				tree.ToForest()
 				firstToken = true
@@ -456,7 +457,7 @@ func (parser *parser) parseGeneral(scanner *scanner, ctx parserContext, termSymb
 				selectorTerm = nil
 				continue
 			} else {
-				err = tk.Errorf(`unexpected token %q, expected ",", "]", or ")"`, tk.source)
+				err = tk.Errorf(`unexpected token %q, expected ",", "]", or ")"`, tk.Source())
 				break
 			}
 		}
@@ -468,87 +469,87 @@ func (parser *parser) parseGeneral(scanner *scanner, ctx parserContext, termSymb
 		}
 
 		switch tk.Sym {
-		case SymOpenRound:
-			var subTree *ast
-			if subTree, err = parser.parseGeneral(scanner, ctx, SymClosedRound); err == nil {
-				if subTree.root == nil {
+		case scan.SymOpenRound:
+			var subTree *scan.Ast
+			if subTree, err = parser.parseGeneral(scanner, ctx, scan.SymClosedRound); err == nil {
+				if subTree.Root() == nil {
 					err = tk.ErrorExpectedGotString("expression", "()")
 				} else {
-					exprTerm := newExprTerm(subTree.root)
-					err = tree.addTerm(exprTerm)
+					exprTerm := newExprTerm(subTree.Root())
+					err = tree.AddTerm(exprTerm)
 					currentTerm = exprTerm
 				}
 			}
-		case SymFuncCall:
-			var funcCallTerm *term
+		case scan.SymFuncCall:
+			var funcCallTerm *scan.Term
 			if funcCallTerm, err = parser.parseFuncCall(scanner, ctx, tk); err == nil {
-				err = tree.addTerm(funcCallTerm)
+				err = tree.AddTerm(funcCallTerm)
 				currentTerm = funcCallTerm
 			}
-		case SymOpenSquare:
-			var listTerm *term
+		case scan.SymOpenSquare:
+			var listTerm *scan.Term
 			newCtx := addFlagsCond(addFlags(ctx, squareContext), allowIndex, couldBeACollection(currentTerm))
 			if listTerm, err = parser.parseList(scanner, newCtx); err == nil {
 				currentTerm, err = listSubTree(tree, listTerm, hasFlag(newCtx, allowIndex))
 			}
-		case SymOpenBrace:
-			if currentTerm != nil && currentTerm.symbol() == SymColon {
+		case scan.SymOpenBrace:
+			if currentTerm != nil && currentTerm.Symbol() == scan.SymColon {
 				err = currentTerm.Errorf(`selector-case outside of a selector context`)
 			} else {
-				var mapTerm *term
+				var mapTerm *scan.Term
 				if mapTerm, err = parser.parseDictionary(scanner, ctx); err == nil {
-					err = tree.addTerm(mapTerm)
+					err = tree.AddTerm(mapTerm)
 					currentTerm = mapTerm
 				}
 			}
-		case SymEqual, SymPlusEqual, SymMinusEqual, SymStarEqual, SymSlashEqual, SymPercEqual, SymAmpersandEqual, SymVertBarEqual, SymDoubleLessEqual, SymDoubleGreaterEqual, SymCaretEqual:
-			currentTerm, err = tree.addToken(tk)
+		case scan.SymEqual, scan.SymPlusEqual, scan.SymMinusEqual, scan.SymStarEqual, scan.SymSlashEqual, scan.SymPercEqual, scan.SymAmpersandEqual, scan.SymVertBarEqual, scan.SymDoubleLessEqual, scan.SymDoubleGreaterEqual, scan.SymCaretEqual:
+			currentTerm, err = tree.AddToken(tk)
 			firstToken = true
-		case SymFuncDef:
-			var funcDefTerm *term
+		case scan.SymFuncDef:
+			var funcDefTerm *scan.Term
 			if funcDefTerm, err = parser.parseFuncDef(scanner); err == nil {
-				err = tree.addTerm(funcDefTerm)
+				err = tree.AddTerm(funcDefTerm)
 				currentTerm = funcDefTerm
 			}
-		case SymDollarRound:
-			var iterDefTerm *term
+		case scan.SymDollarRound:
+			var iterDefTerm *scan.Term
 			if iterDefTerm, err = parser.parseIterDef(scanner, ctx); err == nil {
-				err = tree.addTerm(iterDefTerm)
+				err = tree.AddTerm(iterDefTerm)
 				currentTerm = iterDefTerm
 			}
-		case SymIdentifier:
-			if tk.source[0] == '@' && !hasFlag(ctx, allowVarRef) {
-				err = tk.Errorf("variable references are not allowed in top level expressions: %q", tk.source)
+		case scan.SymIdentifier:
+			if tk.Source()[0] == '@' && !hasFlag(ctx, allowVarRef) {
+				err = tk.Errorf("variable references are not allowed in top level expressions: %q", tk.Source())
 			} else {
-				currentTerm, err = tree.addToken(tk)
+				currentTerm, err = tree.AddToken(tk)
 			}
-		case SymQuestion:
+		case scan.SymQuestion:
 			if selectorTerm, err = parser.parseSelector(scanner, tree, ctx); err == nil {
 				currentTerm = selectorTerm
 				addFlags(ctx, selectorContext)
 			}
-		case SymColon, SymDoubleColon:
-			var caseTerm *term
+		case scan.SymColon, scan.SymDoubleColon:
+			var caseTerm *scan.Term
 			if selectorTerm != nil {
-				if caseTerm, err = parser.parseSelectorCase(scanner, ctx, tk.Sym == SymDoubleColon); err == nil {
+				if caseTerm, err = parser.parseSelectorCase(scanner, ctx, tk.Sym == scan.SymDoubleColon); err == nil {
 					addSelectorCase(selectorTerm, caseTerm)
 					currentTerm = caseTerm
-					if tk.Sym == SymDoubleColon {
+					if tk.Sym == scan.SymDoubleColon {
 						selectorTerm = nil
 					}
 				}
 			} else {
-				currentTerm, err = tree.addToken(tk)
-				if tk.IsOneOfA(SymColon, SymRange) {
+				currentTerm, err = tree.AddToken(tk)
+				if tk.IsOneOfA(scan.SymColon, scan.SymRange) {
 					// Colon outside a selector term acts like a separator
 					firstToken = true
 				}
 			}
 		default:
-			currentTerm, err = tree.addToken(tk)
+			currentTerm, err = tree.AddToken(tk)
 		}
 
-		if currentTerm != nil && currentTerm.tk.Sym != SymSelector && currentTerm.parent != nil && currentTerm.parent.tk.Sym != SymSelector {
+		if currentTerm != nil && currentTerm.Tk.Sym != scan.SymSelector && currentTerm.Parent != nil && currentTerm.Parent.Tk.Sym != scan.SymSelector {
 			selectorTerm = nil
 			remFlags(ctx, selectorContext)
 		}
@@ -558,12 +559,12 @@ func (parser *parser) parseGeneral(scanner *scanner, ctx parserContext, termSymb
 	if err == nil {
 		if !tk.IsOneOf(termSymbols) {
 			var symDesc string
-			if tk.IsSymbol(SymError) {
+			if tk.IsSymbol(scan.SymError) {
 				symDesc = tk.ErrorText()
 			} else {
-				symDesc = SymToString(tk.Sym)
+				symDesc = scan.SymToString(tk.Sym)
 			}
-			err = tk.ErrorExpectedGotStringWithPrefix("expected one of", SymListToString(termSymbols, true), symDesc)
+			err = tk.ErrorExpectedGotStringWithPrefix("expected one of", scan.SymListToString(termSymbols, true), symDesc)
 		} else {
 			err = tk.Error()
 		}