go-pkg/expr
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

several changes:

Browse Source 
1. moved scan/symbol.go to new package sym and adapted all files that reference Symbol type and its values.
2. new type interval defined by begin, end and step values.
3. replaced operator-range.go with operator-interval.go.
4. replaced range operator begin:end with begin..end..step.
5. new implementation of sub-collection extraction based on new interval literal.
This commit is contained in:
Celestino Amoroso
2026-07-27 16:01:52 +02:00
parent 07aecfc4e7
commit 8e596d5979
63 changed files with 1125 additions and 755 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scan/scanner.go
+154 -127
View File
@@ -12,20 +12,23 @@ import (
"strconv"
"strings"
"git.portale-stac.it/go-pkg/expr/sym"
"git.portale-stac.it/go-pkg/expr/util"
)
type Scanner struct {
current *Token
prev *Token
stage *Token
stream *bufio.Reader
row int
column int
translations map[Symbol]Symbol
current *Token
prev *Token
stage *Token
stream *bufio.Reader
row int
column int
translations map[sym.Symbol]sym.Symbol
unreadByte byte
prevRowLength int
}
func NewScanner(s io.Reader, translations map[Symbol]Symbol) (inst *Scanner) {
func NewScanner(s io.Reader, translations map[sym.Symbol]sym.Symbol) (inst *Scanner) {
inst = &Scanner{
stream: bufio.NewReader(s),
row: 1,
@@ -36,15 +39,15 @@ func NewScanner(s io.Reader, translations map[Symbol]Symbol) (inst *Scanner) {
return inst
}
func DefaultTranslations() map[Symbol]Symbol {
return map[Symbol]Symbol{
SymDoubleAmpersand: SymAnd,
SymKwAnd: SymAnd,
SymDoubleVertBar: SymOr,
SymKwOr: SymOr,
// SymTilde: SymNot,
SymKwNot: SymNot,
SymLessGreater: SymNotEqual,
func DefaultTranslations() map[sym.Symbol]sym.Symbol {
return map[sym.Symbol]sym.Symbol{
sym.SymDoubleAmpersand: sym.SymAnd,
sym.SymKwAnd: sym.SymAnd,
sym.SymDoubleVertBar: sym.SymOr,
sym.SymKwOr: sym.SymOr,
sym.SymKwNot: sym.SymNot,
sym.SymLessGreater: sym.SymNotEqual,
// SymTilde: sym.SymNot,
}
}
@@ -56,9 +59,25 @@ func (scanner *Scanner) Current() *Token {
return scanner.current
}
func (scanner *Scanner) setUnreadChar(ch byte) {
scanner.column--
if scanner.column == 0 {
scanner.column = scanner.prevRowLength
scanner.row--
}
scanner.unreadByte = ch
}
func (scanner *Scanner) readChar() (ch byte, err error) {
if ch, err = scanner.stream.ReadByte(); err == nil {
if scanner.unreadByte != 0 {
ch = scanner.unreadByte
scanner.unreadByte = 0
} else {
ch, err = scanner.stream.ReadByte()
}
if err == nil {
if ch == '\n' {
scanner.prevRowLength = scanner.column
scanner.row++
scanner.column = 0
} else {
@@ -127,237 +146,237 @@ func (scanner *Scanner) fetchNextToken() (tk *Token) {
switch ch {
case '+':
if next, _ := scanner.peek(); next == '+' {
tk = scanner.moveOn(SymDoublePlus, ch, next)
tk = scanner.moveOn(sym.SymDoublePlus, ch, next)
} else if next == '=' {
tk = scanner.moveOn(SymPlusEqual, ch, next)
tk = scanner.moveOn(sym.SymPlusEqual, ch, next)
} else if next == '>' {
tk = scanner.moveOn(SymPlusGreater, ch, next)
tk = scanner.moveOn(sym.SymPlusGreater, ch, next)
} else {
tk = scanner.MakeToken(SymPlus, ch)
tk = scanner.MakeToken(sym.SymPlus, ch)
}
case '-':
if next, _ := scanner.peek(); next == '-' {
tk = scanner.moveOn(SymDoubleMinus, ch, next)
tk = scanner.moveOn(sym.SymDoubleMinus, ch, next)
} else if next == '=' {
tk = scanner.moveOn(SymMinusEqual, ch, next)
tk = scanner.moveOn(sym.SymMinusEqual, ch, next)
} else {
tk = scanner.MakeToken(SymMinus, ch)
tk = scanner.MakeToken(sym.SymMinus, ch)
}
case '*':
if next, _ := scanner.peek(); next == '*' {
tk = scanner.moveOn(SymDoubleStar, ch, next)
tk = scanner.moveOn(sym.SymDoubleStar, ch, next)
// } else if next == '/' {
// tk = self.moveOn(SymClosedComment, ch, next)
} else if next, _ = scanner.peek(); next == '=' {
tk = scanner.moveOn(SymStarEqual, ch, next)
tk = scanner.moveOn(sym.SymStarEqual, ch, next)
} else {
tk = scanner.MakeToken(SymStar, ch)
tk = scanner.MakeToken(sym.SymStar, ch)
}
case '/':
if next, _ := scanner.peek(); next == '*' {
scanner.readChar()
tk = scanner.fetchBlockComment()
} else if next, _ = scanner.peek(); next == '=' {
tk = scanner.moveOn(SymSlashEqual, ch, next)
tk = scanner.moveOn(sym.SymSlashEqual, ch, next)
} else if next == '/' {
scanner.readChar()
tk = scanner.fetchOnLineComment()
} else {
tk = scanner.MakeToken(SymSlash, ch)
tk = scanner.MakeToken(sym.SymSlash, ch)
}
case '\\':
if escape {
tk = scanner.MakeToken(SymBackSlash, ch)
tk = scanner.MakeToken(sym.SymBackSlash, ch)
escape = false
} else {
escape = true
}
case '|':
if next, _ := scanner.peek(); next == '|' {
tk = scanner.moveOn(SymDoubleVertBar, ch, next)
tk = scanner.moveOn(sym.SymDoubleVertBar, ch, next)
} else if next, _ = scanner.peek(); next == '=' {
tk = scanner.moveOn(SymVertBarEqual, ch, next)
tk = scanner.moveOn(sym.SymVertBarEqual, ch, next)
} else {
tk = scanner.MakeToken(SymVertBar, ch)
tk = scanner.MakeToken(sym.SymVertBar, ch)
}
case ',':
tk = scanner.MakeToken(SymComma, ch)
tk = scanner.MakeToken(sym.SymComma, ch)
case '^':
if next, _ := scanner.peek(); next == '=' {
tk = scanner.moveOn(SymCaretEqual, ch, next)
tk = scanner.moveOn(sym.SymCaretEqual, ch, next)
} else {
tk = scanner.MakeToken(SymCaret, ch)
tk = scanner.MakeToken(sym.SymCaret, ch)
}
case ':':
if next, _ := scanner.peek(); next == ':' {
tk = scanner.moveOn(SymDoubleColon, ch, next)
tk = scanner.moveOn(sym.SymDoubleColon, ch, next)
} else if next == '=' {
tk = scanner.moveOn(SymColonEqual, ch, next)
tk = scanner.moveOn(sym.SymColonEqual, ch, next)
} else {
tk = scanner.MakeToken(SymColon, ch)
tk = scanner.MakeToken(sym.SymColon, ch)
}
case ';':
tk = scanner.MakeToken(SymSemiColon, ch)
tk = scanner.MakeToken(sym.SymSemiColon, ch)
case '.':
//if next, _ := self.peek(); next >= '0' && next <= '9' {
// tk = self.parseNumber(ch)
//} else if next == '/' {
if next, _ := scanner.peek(); next == '/' {
tk = scanner.moveOn(SymDotSlash, ch, next)
tk = scanner.moveOn(sym.SymDotSlash, ch, next)
} else if next == '.' {
tk = scanner.moveOn(sym.SymDoubleDot, ch, next)
// TODO: this section needs verification
if next1, _ := scanner.peek(); next1 == '.' {
tk = scanner.moveOn(SymTripleDot, ch, next, next1)
} else {
tk = scanner.moveOn(SymDoubleDot, ch, next)
tk = scanner.moveOn(sym.SymTripleDot, next, next1)
}
} else {
tk = scanner.MakeToken(SymDot, ch)
tk = scanner.MakeToken(sym.SymDot, ch)
}
case '\'':
if escape {
tk = scanner.MakeToken(SymQuote, ch)
tk = scanner.MakeToken(sym.SymQuote, ch)
escape = false
} else {
tk = scanner.fetchString(ch, true)
}
case '"':
if escape {
tk = scanner.MakeToken(SymDoubleQuote, ch)
tk = scanner.MakeToken(sym.SymDoubleQuote, ch)
escape = false
} else {
tk = scanner.fetchString(ch, true)
}
case '`':
tk = scanner.MakeToken(SymBackTick, ch)
tk = scanner.MakeToken(sym.SymBackTick, ch)
case '!':
if next, _ := scanner.peek(); next == '=' {
tk = scanner.moveOn(SymNotEqual, ch, next)
tk = scanner.moveOn(sym.SymNotEqual, ch, next)
} else {
tk = scanner.MakeToken(SymExclamation, ch)
tk = scanner.MakeToken(sym.SymExclamation, ch)
}
case '?':
if next, _ := scanner.peek(); next == '?' {
tk = scanner.moveOn(SymDoubleQuestion, ch, next)
tk = scanner.moveOn(sym.SymDoubleQuestion, ch, next)
} else if next == '=' {
tk = scanner.moveOn(SymQuestionEqual, ch, next)
tk = scanner.moveOn(sym.SymQuestionEqual, ch, next)
} else if next == '!' {
tk = scanner.moveOn(SymQuestionExclam, ch, next)
tk = scanner.moveOn(sym.SymQuestionExclam, ch, next)
} else {
tk = scanner.MakeToken(SymQuestion, ch)
tk = scanner.MakeToken(sym.SymQuestion, ch)
}
case '&':
if next, _ := scanner.peek(); next == '&' {
tk = scanner.moveOn(SymDoubleAmpersand, ch, next)
tk = scanner.moveOn(sym.SymDoubleAmpersand, ch, next)
} else if next, _ = scanner.peek(); next == '=' {
tk = scanner.moveOn(SymAmpersandEqual, ch, next)
tk = scanner.moveOn(sym.SymAmpersandEqual, ch, next)
} else {
tk = scanner.MakeToken(SymAmpersand, ch)
tk = scanner.MakeToken(sym.SymAmpersand, ch)
}
case '%':
if next, _ := scanner.peek(); next == '=' {
tk = scanner.moveOn(SymPercEqual, ch, next)
tk = scanner.moveOn(sym.SymPercEqual, ch, next)
} else {
tk = scanner.MakeToken(SymPercent, ch)
tk = scanner.MakeToken(sym.SymPercent, ch)
}
case '#':
tk = scanner.MakeToken(SymHash, ch)
tk = scanner.MakeToken(sym.SymHash, ch)
case '@':
if next, _ := scanner.peek(); (next >= 'a' && next <= 'z') || (next >= 'A' && next <= 'Z') {
scanner.readChar()
if tk = scanner.fetchIdentifier(next); tk.Sym == SymIdentifier {
if tk = scanner.fetchIdentifier(next); tk.Sym == sym.SymIdentifier {
//tk.Sym = SymIdRef
tk.source = "@" + tk.source
} else {
tk = scanner.makeErrorToken(fmt.Errorf("invalid variable reference %q", tk.source))
}
} else if next == '@' {
tk = scanner.moveOn(SymDoubleAt, ch, next)
tk = scanner.moveOn(sym.SymDoubleAt, ch, next)
} else {
tk = scanner.MakeToken(SymAt, ch)
tk = scanner.MakeToken(sym.SymAt, ch)
}
case '_':
tk = scanner.MakeToken(SymUndescore, ch)
tk = scanner.MakeToken(sym.SymUndescore, ch)
case '=':
if next, _ := scanner.peek(); next == '=' {
tk = scanner.moveOn(SymDoubleEqual, ch, next)
tk = scanner.moveOn(sym.SymDoubleEqual, ch, next)
} else {
tk = scanner.MakeToken(SymEqual, ch)
tk = scanner.MakeToken(sym.SymEqual, ch)
}
case '<':
if next, _ := scanner.peek(); next == '=' {
tk = scanner.moveOn(SymLessOrEqual, ch, next)
tk = scanner.moveOn(sym.SymLessOrEqual, ch, next)
} else if next == '<' {
scanner.readChar()
next2, _ := scanner.readChar()
scanner.unreadChar()
if next2 == '=' {
tk = scanner.moveOn(SymDoubleLessEqual, ch, next, next2)
tk = scanner.moveOn(sym.SymDoubleLessEqual, ch, next, next2)
} else {
tk = scanner.accept(SymDoubleLess, ch, next)
tk = scanner.accept(sym.SymDoubleLess, ch, next)
}
} else if next == '>' {
tk = scanner.moveOn(SymLessGreater, ch, next)
tk = scanner.moveOn(sym.SymLessGreater, ch, next)
} else if next == '+' {
tk = scanner.moveOn(SymLessPlus, ch, next)
tk = scanner.moveOn(sym.SymLessPlus, ch, next)
} else {
tk = scanner.MakeToken(SymLess, ch)
tk = scanner.MakeToken(sym.SymLess, ch)
}
case '>':
if next, _ := scanner.peek(); next == '=' {
tk = scanner.moveOn(SymGreaterOrEqual, ch, next)
tk = scanner.moveOn(sym.SymGreaterOrEqual, ch, next)
} else if next == '>' {
scanner.readChar()
next2, _ := scanner.readChar()
scanner.unreadChar()
if next2 == '=' {
tk = scanner.moveOn(SymDoubleGreaterEqual, ch, next, next2)
tk = scanner.moveOn(sym.SymDoubleGreaterEqual, ch, next, next2)
} else {
tk = scanner.accept(SymDoubleGreater, ch, next)
tk = scanner.accept(sym.SymDoubleGreater, ch, next)
}
} else if next == ']' {
tk = scanner.moveOn(SymGreaterClosedSquare, ch, next)
tk = scanner.moveOn(sym.SymGreaterClosedSquare, ch, next)
} else {
tk = scanner.MakeToken(SymGreater, ch)
tk = scanner.MakeToken(sym.SymGreater, ch)
}
case '$':
if next, _ := scanner.peek(); next == '(' {
tk = scanner.moveOn(SymDollarRound, ch, next)
tk = scanner.moveOn(sym.SymDollarRound, ch, next)
tk.source += ")"
} else if next == '$' {
tk = scanner.moveOn(SymDoubleDollar, ch, next)
tk = scanner.moveOn(sym.SymDoubleDollar, ch, next)
} else if next == '{' {
scanner.readChar()
if tk = scanner.fetchString('}', false); tk != nil {
tk.Sym = SymIdentifier
tk.Sym = sym.SymIdentifier
}
} else if next == '_' || (next >= 'a' && next <= 'z') || (next >= 'A' && next <= 'Z') {
scanner.readChar()
tk = scanner.fetchIdentifier(next)
} else {
tk = scanner.MakeToken(SymDollar, ch)
tk = scanner.MakeToken(sym.SymDollar, ch)
}
case '(':
// if next, _ := scanner.peek(); next == ')' {
// tk = scanner.moveOn(SymOpenClosedRound, ch, next)
// } else {
tk = scanner.MakeToken(SymOpenRound, ch)
tk = scanner.MakeToken(sym.SymOpenRound, ch)
// }
case ')':
tk = scanner.MakeToken(SymClosedRound, ch)
tk = scanner.MakeToken(sym.SymClosedRound, ch)
case '[':
if next, _ := scanner.peek(); next == '<' {
tk = scanner.moveOn(SymOpenSquareLess, ch, next)
tk = scanner.moveOn(sym.SymOpenSquareLess, ch, next)
} else {
tk = scanner.MakeToken(SymOpenSquare, ch)
tk = scanner.MakeToken(sym.SymOpenSquare, ch)
}
case ']':
tk = scanner.MakeToken(SymClosedSquare, ch)
tk = scanner.MakeToken(sym.SymClosedSquare, ch)
case '{':
tk = scanner.MakeToken(SymOpenBrace, ch)
tk = scanner.MakeToken(sym.SymOpenBrace, ch)
case '}':
tk = scanner.MakeToken(SymClosedBrace, ch)
tk = scanner.MakeToken(sym.SymClosedBrace, ch)
case '~':
tk = scanner.MakeToken(SymTilde, ch)
tk = scanner.MakeToken(sym.SymTilde, ch)
case 0:
if escape {
tk = scanner.makeErrorToken(errors.New("incomplete escape sequence"))
@@ -365,9 +384,9 @@ func (scanner *Scanner) fetchNextToken() (tk *Token) {
escape = false
default:
if /*ch == '_' ||*/ (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') {
if tk = scanner.fetchIdentifier(ch); tk.Sym == SymKwFunc {
if tk = scanner.fetchIdentifier(ch); tk.Sym == sym.SymKwFunc {
if next, _ := scanner.peek(); next == '(' {
tk = scanner.moveOn(SymFuncDef, ch, next)
tk = scanner.moveOn(sym.SymFuncDef, ch, next)
}
}
} else if ch >= '0' && ch <= '9' {
@@ -415,19 +434,20 @@ func (scanner *Scanner) initBase(currentFirstCh byte) (firstCh byte, numBase int
numBase = 10
if ch, err = scanner.peek(); err == nil {
if ch == 'b' || ch == 'B' {
switch ch {
case 'b', 'B':
numBase = 2
digitType = "binary"
scanner.readChar()
digitFunc = isBinaryDigit
firstCh, err = scanner.readChar()
} else if ch == 'o' || ch == 'O' {
case 'o', 'O':
numBase = 8
digitType = "octal"
scanner.readChar()
digitFunc = isOctalDigit
firstCh, err = scanner.readChar()
} else if ch == 'x' || ch == 'X' {
case 'x', 'X':
numBase = 16
digitType = "hex"
scanner.readChar()
@@ -449,7 +469,7 @@ func (scanner *Scanner) initBase(currentFirstCh byte) (firstCh byte, numBase int
func (scanner *Scanner) parseNumber(firstCh byte) (tk *Token) {
var err error
var ch byte
var sym Symbol = SymInteger
var symbol sym.Symbol = sym.SymInteger
var sb strings.Builder
var isDigit func(byte) bool = isDecimalDigit
var numBase = 10
@@ -463,18 +483,24 @@ func (scanner *Scanner) parseNumber(firstCh byte) (tk *Token) {
if numBase == 10 {
if err == nil && ch == '.' {
sym = SymFloat
sb.WriteByte(ch)
ch, err = scanner.readChar()
if ch >= '0' && ch <= '9' {
for ; err == nil && (ch >= '0' && ch <= '9'); ch, err = scanner.readChar() {
sb.WriteByte(ch)
if next, _ := scanner.peek(); next == '.' {
// do nothing, this is an interval operator
scanner.setUnreadChar('.')
} else {
symbol = sym.SymFloat
sb.WriteByte(ch)
ch, err = scanner.readChar()
if ch >= '0' && ch <= '9' {
for ; err == nil && (ch >= '0' && ch <= '9'); ch, err = scanner.readChar() {
sb.WriteByte(ch)
}
}
}
}
if err == nil {
if ch == 'e' || ch == 'E' {
sym = SymFloat
switch ch {
case 'e', 'E':
symbol = sym.SymFloat
sb.WriteByte(ch)
if ch, err = scanner.readChar(); err == nil {
if ch == '+' || ch == '-' {
@@ -489,8 +515,8 @@ func (scanner *Scanner) parseNumber(firstCh byte) (tk *Token) {
err = fmt.Errorf("[%d:%d] expected integer exponent, got %c", scanner.row, scanner.column, ch)
}
}
} else if ch == '(' {
sym = SymFraction
case '(':
symbol = sym.SymFraction
sb.WriteByte(ch)
ch, err = scanner.readChar()
for ; err == nil && (ch >= '0' && ch <= '9'); ch, err = scanner.readChar() {
@@ -514,15 +540,16 @@ func (scanner *Scanner) parseNumber(firstCh byte) (tk *Token) {
var value any
_ = scanner.sync(err) // TODO: Check this function
txt := sb.String()
if sym == SymFloat {
switch symbol {
case sym.SymFloat:
value, err = strconv.ParseFloat(txt, 64)
} else if sym == SymFraction {
case sym.SymFraction:
// value, err = kern.MakeGeneratingFraction(txt)
value = txt
} else {
default:
value, err = strconv.ParseInt(txt, numBase, 64)
}
tk = scanner.MakeValueToken(sym, txt, value)
tk = scanner.MakeValueToken(symbol, txt, value)
// if err == nil {
// tk = scanner.MakeValueToken(sym, txt, value)
// } else {
@@ -546,17 +573,17 @@ func (scanner *Scanner) fetchIdentifier(firstCh byte) (tk *Token) {
} else {
txt := sb.String()
uptxt := strings.ToUpper(txt)
if sym, ok := keywords[uptxt]; ok {
tk = scanner.makeKeywordToken(sym, uptxt)
if symbol, ok := sym.Keywords[uptxt]; ok {
tk = scanner.makeKeywordToken(symbol, uptxt)
} else if uptxt == `TRUE` {
tk = scanner.MakeValueToken(SymBool, txt, true)
tk = scanner.MakeValueToken(sym.SymBool, txt, true)
} else if uptxt == `FALSE` {
tk = scanner.MakeValueToken(SymBool, txt, false)
tk = scanner.MakeValueToken(sym.SymBool, txt, false)
} else if ch, _ := scanner.peek(); ch == '(' {
scanner.readChar()
tk = scanner.MakeValueToken(SymFuncCall, txt+"(", txt)
tk = scanner.MakeValueToken(sym.SymFuncCall, txt+"(", txt)
} else {
tk = scanner.MakeValueToken(SymIdentifier, txt, txt)
tk = scanner.MakeValueToken(sym.SymIdentifier, txt, txt)
}
}
@@ -577,14 +604,14 @@ func (scanner *Scanner) fetchIdentifier(firstCh byte) (tk *Token) {
}
func (scanner *Scanner) fetchBlockComment() *Token {
return scanner.fetchUntil(SymComment, false, '*', '/')
return scanner.fetchUntil(sym.SymComment, false, '*', '/')
}
func (scanner *Scanner) fetchOnLineComment() *Token {
return scanner.fetchUntil(SymComment, true, '\n')
return scanner.fetchUntil(sym.SymComment, true, '\n')
}
func (scanner *Scanner) fetchUntil(sym Symbol, allowEos bool, endings ...byte) (tk *Token) {
func (scanner *Scanner) fetchUntil(sym sym.Symbol, allowEos bool, endings ...byte) (tk *Token) {
var err error
var ch byte
var sb strings.Builder
@@ -653,9 +680,9 @@ func (scanner *Scanner) fetchString(termCh byte, addQuote bool) (tk *Token) {
} else {
txt := sb.String()
if addQuote {
tk = scanner.MakeValueToken(SymString, `"`+txt+`"`, txt)
tk = scanner.MakeValueToken(sym.SymString, `"`+txt+`"`, txt)
} else {
tk = scanner.MakeValueToken(SymString, txt, txt)
tk = scanner.MakeValueToken(sym.SymString, txt, txt)
}
}
return
@@ -677,7 +704,7 @@ func (scanner *Scanner) skipBlanks() (err error) {
return
}
func (scanner *Scanner) translate(sym Symbol) Symbol {
func (scanner *Scanner) translate(sym sym.Symbol) sym.Symbol {
if scanner.translations != nil {
if translatedSym, ok := scanner.translations[sym]; ok {
return translatedSym
@@ -686,7 +713,7 @@ func (scanner *Scanner) translate(sym Symbol) Symbol {
return sym
}
func (scanner *Scanner) moveOn(sym Symbol, chars ...byte) (tk *Token) {
func (scanner *Scanner) moveOn(sym sym.Symbol, chars ...byte) (tk *Token) {
tk = NewToken(scanner.row, scanner.column, scanner.translate(sym), string(chars))
// for i := 1; i < len(chars); i++ {
if len(chars) > 1 {
@@ -696,22 +723,22 @@ func (scanner *Scanner) moveOn(sym Symbol, chars ...byte) (tk *Token) {
return
}
func (scanner *Scanner) accept(sym Symbol, chars ...byte) (tk *Token) {
func (scanner *Scanner) accept(sym sym.Symbol, chars ...byte) (tk *Token) {
tk = NewToken(scanner.row, scanner.column, scanner.translate(sym), string(chars))
return
}
func (scanner *Scanner) MakeToken(sym Symbol, chars ...byte) (tk *Token) {
func (scanner *Scanner) MakeToken(sym sym.Symbol, chars ...byte) (tk *Token) {
tk = NewToken(scanner.row, scanner.column, scanner.translate(sym), string(chars))
return
}
func (scanner *Scanner) makeKeywordToken(sym Symbol, upperCaseKeyword string) (tk *Token) {
func (scanner *Scanner) makeKeywordToken(sym sym.Symbol, upperCaseKeyword string) (tk *Token) {
tk = NewToken(scanner.row, scanner.column, scanner.translate(sym), upperCaseKeyword)
return
}
func (scanner *Scanner) MakeValueToken(sym Symbol, source string, value any) (tk *Token) {
func (scanner *Scanner) MakeValueToken(sym sym.Symbol, source string, value any) (tk *Token) {
tk = NewValueToken(scanner.row, scanner.column, scanner.translate(sym), source, value)
return
}
scan/symbol-map.go
+132 -130
View File
@@ -1,14 +1,16 @@
// Copyright (c) 2024-2026 Celestino Amoroso (celestino.amoroso@gmail.com).
// All rights reserved.
// Symbol.go
// symbol-map.go
package scan
import (
"strings"
"git.portale-stac.it/go-pkg/expr/sym"
)
var symbolMap map[Symbol]symbolSpec
var symbolMap map[sym.Symbol]symbolSpec
type SymbolClass int16
@@ -31,139 +33,139 @@ type symbolSpec struct {
func init() {
symbolMap = map[Symbol]symbolSpec{
SymUnknown: {"<unknown>", SymClassOther, PosLeaf}, // -1: Unknown symbol
SymNone: {"<null>", SymClassOther, PosLeaf}, // 0: Null value for variable of type symbol
SymError: {"<error>", SymClassOther, PosLeaf}, // 1: Error reading from stream
SymEos: {"<eos>", SymClassOther, PosLeaf}, // 2: End of stream
SymMinus: {"-", SymClassOperator, PosInfix}, // 3: '-'
SymMinusEqual: {"-=", SymClassOperator, PosInfix}, // 4: '-='
SymDoubleMinus: {"--", SymClassOperator, PosPostfix}, // 5: '--'
SymPlus: {"+", SymClassOperator, PosInfix}, // 6: '+'
SymPlusEqual: {"+=", SymClassOperator, PosInfix}, // 7: '+='
SymDoublePlus: {"++", SymClassOperator, PosPostfix}, // 8: '++'
SymStar: {"*", SymClassOperator, PosInfix}, // 9: '*'
SymDoubleStar: {"**", SymClassOperator, PosInfix}, // 10: '**'
SymSlash: {"/", SymClassOperator, PosInfix}, // 11: '/'
SymBackSlash: {"\\", SymClassOperator, PosLeaf}, // 12: '\'
SymVertBar: {"|", SymClassOperator, PosInfix}, // 13: '|'
SymDoubleVertBar: {"||", SymClassOperator, PosInfix}, // 14: '||'
SymComma: {",", SymClassOperator, PosInfix}, // 15: ','
SymColon: {":", SymClassOperator, PosInfix}, // 16: ':'
SymSemiColon: {";", SymClassOperator, PosInfix}, // 17: ';'
SymDot: {".", SymClassOperator, PosInfix}, // 18: '.'
SymDotSlash: {"./", SymClassOperator, PosInfix}, // 19: './'
SymQuote: {"'", SymClassDelimiter, PosLeaf}, // 20: '\''
SymDoubleQuote: {"\"", SymClassDelimiter, PosLeaf}, // 21: '"'
SymBackTick: {"`", SymClassDelimiter, PosLeaf}, // 22: '`'
SymExclamation: {"!", SymClassOperator, PosPostfix}, // 23: '!'
SymQuestion: {"?", SymClassOperator, PosInfix}, // 24: '?'
SymAmpersand: {"&", SymClassOperator, PosInfix}, // 25: '&'
SymDoubleAmpersand: {"&&", SymClassOperator, PosInfix}, // 26: '&&'
SymPercent: {"%", SymClassOperator, PosInfix}, // 27: '%'
SymAt: {"@", SymClassOperator, PosPrefix}, // 28: '@'
SymUndescore: {"_", SymClassIdentifier, PosLeaf}, // 29: '_'
SymEqual: {"=", SymClassOperator, PosInfix}, // 30: '='
SymColonEqual: {":=", SymClassOperator, PosInfix}, // 31: ':='
SymDoubleEqual: {"==", SymClassOperator, PosInfix}, // 32: '=='
SymLess: {"<", SymClassOperator, PosInfix}, // 33: '<'
SymLessOrEqual: {"<=", SymClassOperator, PosInfix}, // 34: '<='
SymGreater: {">", SymClassOperator, PosInfix}, // 35: '>'
SymGreaterOrEqual: {">=", SymClassOperator, PosInfix}, // 36: '>='
SymLessGreater: {"<>", SymClassOperator, PosInfix}, // 37: '<>'
SymNotEqual: {"!=", SymClassOperator, PosInfix}, // 38: '!='
SymDollar: {"$", SymClassOperator, PosPrefix}, // 39: '$'
SymHash: {"#", SymClassOperator, PosPrefix}, // 40: '#'
SymOpenRound: {"(", SymClassParenthesis, PosPrefix}, // 41: '('
SymClosedRound: {")", SymClassParenthesis, PosPostfix}, // 42: ')'
SymOpenSquare: {"[", SymClassParenthesis, PosPrefix}, // 43: '['
SymClosedSquare: {"]", SymClassParenthesis, PosPostfix}, // 44: ']'
SymOpenBrace: {"{", SymClassParenthesis, PosPrefix}, // 45: '{'
SymClosedBrace: {"}", SymClassParenthesis, PosPostfix}, // 46: '}'
SymTilde: {"~", SymClassOperator, PosPrefix}, // 47: '~'
SymDoubleQuestion: {"??", SymClassOperator, PosInfix}, // 48: '??'
SymQuestionEqual: {"?=", SymClassOperator, PosInfix}, // 49: '?='
SymQuestionExclam: {"?!", SymClassOperator, PosInfix}, // 50: '?!'
SymDoubleAt: {"@@", SymClassCommand, PosLeaf}, // 51: '@@'
SymDoubleColon: {"::", SymClassOperator, PosInfix}, // 52: '::'
SymDoubleGreater: {">>", SymClassOperator, PosInfix}, // 53: '>>'
SymDoubleLess: {"<<", SymClassOperator, PosInfix}, // 54: '<<'
SymCaret: {"^", SymClassOperator, PosInfix}, // 55: '^'
SymDollarRound: {"$(", SymClassOperator, PosPrefix}, // 56: '$('
SymOpenClosedRound: {"()", SymClassOperator, PosPostfix}, // 57: '()'
SymDoubleDollar: {"$$", SymClassCommand, PosLeaf}, // 58: '$$'
SymDoubleDot: {"..", SymClassOperator, PosInfix}, // 59: '..'
SymTripleDot: {"...", SymClassOperator, PosPostfix}, // 60: '...'
SymStarEqual: {"*=", SymClassOperator, PosInfix}, // 61: '*='
SymSlashEqual: {"/=", SymClassOperator, PosInfix}, // 62: '/='
SymPercEqual: {"%=", SymClassOperator, PosInfix}, // 63: '%='
SymDoubleLessEqual: {"<<=", SymClassOperator, PosInfix}, // 64: '<<='
SymDoubleGreaterEqual: {">>=", SymClassOperator, PosInfix}, // 65: '>>='
SymAmpersandEqual: {"&=", SymClassOperator, PosInfix}, // 66: '&='
SymVertBarEqual: {"|=", SymClassOperator, PosInfix}, // 67: '|='
SymCaretEqual: {"^=", SymClassOperator, PosInfix}, // 68: '^='
SymPlusGreater: {"+>", SymClassOperator, PosInfix}, // 69: '+>'
SymLessPlus: {"<+", SymClassOperator, PosInfix}, // 70: '<+'
SymPreInc: {"++", SymClassOperator, PosPrefix}, // 71: '++'
SymPreDec: {"--", SymClassOperator, PosPrefix}, // 72: '--'
SymOpenSquareLess: {"[<", SymClassOperator, PosPrefix}, // 97: '[<'
SymGreaterClosedSquare: {">]", SymClassOperator, PosPostfix}, // 98: '>]'
// SymChangeSign
// SymUnchangeSign
// SymIdentifier
// SymBool
// SymInteger
// SymVariable
// SymFloat
// SymFraction
// SymString
// SymIterator
// SymOr: "or",
// SymAnd: "and",
// SymNot: "not",
// SymComment
// SymFuncCall
// SymFuncDef
// SymList
// SymLinkedList
// SymDict
// SymIndex
// SymExpression
// SymSelector // <selector> ::= <expr> "?" <selector-case> {":" <selector-case>} ["::" <default-selector-case>]
// SymSelectorCase // <selector-case> ::= [<list>] "{" <multi-expr> "}"
// // SymOpenComment // 0: '/*'
// // SymClosedComment // 0: '*/'
// // SymOneLineComment // 0: '//'
symbolMap = map[sym.Symbol]symbolSpec{
sym.SymUnknown: {"<unknown>", SymClassOther, PosLeaf}, // -1: Unknown symbol
sym.SymNone: {"<null>", SymClassOther, PosLeaf}, // 0: Null value for variable of type symbol
sym.SymError: {"<error>", SymClassOther, PosLeaf}, // 1: Error reading from stream
sym.SymEos: {"<eos>", SymClassOther, PosLeaf}, // 2: End of stream
sym.SymMinus: {"-", SymClassOperator, PosInfix}, // 3: '-'
sym.SymMinusEqual: {"-=", SymClassOperator, PosInfix}, // 4: '-='
sym.SymDoubleMinus: {"--", SymClassOperator, PosPostfix}, // 5: '--'
sym.SymPlus: {"+", SymClassOperator, PosInfix}, // 6: '+'
sym.SymPlusEqual: {"+=", SymClassOperator, PosInfix}, // 7: '+='
sym.SymDoublePlus: {"++", SymClassOperator, PosPostfix}, // 8: '++'
sym.SymStar: {"*", SymClassOperator, PosInfix}, // 9: '*'
sym.SymDoubleStar: {"**", SymClassOperator, PosInfix}, // 10: '**'
sym.SymSlash: {"/", SymClassOperator, PosInfix}, // 11: '/'
sym.SymBackSlash: {"\\", SymClassOperator, PosLeaf}, // 12: '\'
sym.SymVertBar: {"|", SymClassOperator, PosInfix}, // 13: '|'
sym.SymDoubleVertBar: {"||", SymClassOperator, PosInfix}, // 14: '||'
sym.SymComma: {",", SymClassOperator, PosInfix}, // 15: ','
sym.SymColon: {":", SymClassOperator, PosInfix}, // 16: ':'
sym.SymSemiColon: {";", SymClassOperator, PosInfix}, // 17: ';'
sym.SymDot: {".", SymClassOperator, PosInfix}, // 18: '.'
sym.SymDotSlash: {"./", SymClassOperator, PosInfix}, // 19: './'
sym.SymQuote: {"'", SymClassDelimiter, PosLeaf}, // 20: '\''
sym.SymDoubleQuote: {"\"", SymClassDelimiter, PosLeaf}, // 21: '"'
sym.SymBackTick: {"`", SymClassDelimiter, PosLeaf}, // 22: '`'
sym.SymExclamation: {"!", SymClassOperator, PosPostfix}, // 23: '!'
sym.SymQuestion: {"?", SymClassOperator, PosInfix}, // 24: '?'
sym.SymAmpersand: {"&", SymClassOperator, PosInfix}, // 25: '&'
sym.SymDoubleAmpersand: {"&&", SymClassOperator, PosInfix}, // 26: '&&'
sym.SymPercent: {"%", SymClassOperator, PosInfix}, // 27: '%'
sym.SymAt: {"@", SymClassOperator, PosPrefix}, // 28: '@'
sym.SymUndescore: {"_", SymClassIdentifier, PosLeaf}, // 29: '_'
sym.SymEqual: {"=", SymClassOperator, PosInfix}, // 30: '='
sym.SymColonEqual: {":=", SymClassOperator, PosInfix}, // 31: ':='
sym.SymDoubleEqual: {"==", SymClassOperator, PosInfix}, // 32: '=='
sym.SymLess: {"<", SymClassOperator, PosInfix}, // 33: '<'
sym.SymLessOrEqual: {"<=", SymClassOperator, PosInfix}, // 34: '<='
sym.SymGreater: {">", SymClassOperator, PosInfix}, // 35: '>'
sym.SymGreaterOrEqual: {">=", SymClassOperator, PosInfix}, // 36: '>='
sym.SymLessGreater: {"<>", SymClassOperator, PosInfix}, // 37: '<>'
sym.SymNotEqual: {"!=", SymClassOperator, PosInfix}, // 38: '!='
sym.SymDollar: {"$", SymClassOperator, PosPrefix}, // 39: '$'
sym.SymHash: {"#", SymClassOperator, PosPrefix}, // 40: '#'
sym.SymOpenRound: {"(", SymClassParenthesis, PosPrefix}, // 41: '('
sym.SymClosedRound: {")", SymClassParenthesis, PosPostfix}, // 42: ')'
sym.SymOpenSquare: {"[", SymClassParenthesis, PosPrefix}, // 43: '['
sym.SymClosedSquare: {"]", SymClassParenthesis, PosPostfix}, // 44: ']'
sym.SymOpenBrace: {"{", SymClassParenthesis, PosPrefix}, // 45: '{'
sym.SymClosedBrace: {"}", SymClassParenthesis, PosPostfix}, // 46: '}'
sym.SymTilde: {"~", SymClassOperator, PosPrefix}, // 47: '~'
sym.SymDoubleQuestion: {"??", SymClassOperator, PosInfix}, // 48: '??'
sym.SymQuestionEqual: {"?=", SymClassOperator, PosInfix}, // 49: '?='
sym.SymQuestionExclam: {"?!", SymClassOperator, PosInfix}, // 50: '?!'
sym.SymDoubleAt: {"@@", SymClassCommand, PosLeaf}, // 51: '@@'
sym.SymDoubleColon: {"::", SymClassOperator, PosInfix}, // 52: '::'
sym.SymDoubleGreater: {">>", SymClassOperator, PosInfix}, // 53: '>>'
sym.SymDoubleLess: {"<<", SymClassOperator, PosInfix}, // 54: '<<'
sym.SymCaret: {"^", SymClassOperator, PosInfix}, // 55: '^'
sym.SymDollarRound: {"$(", SymClassOperator, PosPrefix}, // 56: '$('
sym.SymOpenClosedRound: {"()", SymClassOperator, PosPostfix}, // 57: '()'
sym.SymDoubleDollar: {"$$", SymClassCommand, PosLeaf}, // 58: '$$'
sym.SymDoubleDot: {"..", SymClassOperator, PosInfix}, // 59: '..'
sym.SymTripleDot: {"...", SymClassOperator, PosPostfix}, // 60: '...'
sym.SymStarEqual: {"*=", SymClassOperator, PosInfix}, // 61: '*='
sym.SymSlashEqual: {"/=", SymClassOperator, PosInfix}, // 62: '/='
sym.SymPercEqual: {"%=", SymClassOperator, PosInfix}, // 63: '%='
sym.SymDoubleLessEqual: {"<<=", SymClassOperator, PosInfix}, // 64: '<<='
sym.SymDoubleGreaterEqual: {">>=", SymClassOperator, PosInfix}, // 65: '>>='
sym.SymAmpersandEqual: {"&=", SymClassOperator, PosInfix}, // 66: '&='
sym.SymVertBarEqual: {"|=", SymClassOperator, PosInfix}, // 67: '|='
sym.SymCaretEqual: {"^=", SymClassOperator, PosInfix}, // 68: '^='
sym.SymPlusGreater: {"+>", SymClassOperator, PosInfix}, // 69: '+>'
sym.SymLessPlus: {"<+", SymClassOperator, PosInfix}, // 70: '<+'
sym.SymPreInc: {"++", SymClassOperator, PosPrefix}, // 71: '++'
sym.SymPreDec: {"--", SymClassOperator, PosPrefix}, // 72: '--'
sym.SymOpenSquareLess: {"[<", SymClassOperator, PosPrefix}, // 97: '[<'
sym.SymGreaterClosedSquare: {">]", SymClassOperator, PosPostfix}, // 98: '>]'
// sym.SymChangeSign
// sym.SymUnchangeSign
// sym.SymIdentifier
// sym.SymBool
// sym.SymInteger
// sym.SymVariable
// sym.SymFloat
// sym.SymFraction
// sym.SymString
// sym.SymIterator
// sym.SymOr: "or",
// sym.SymAnd: "and",
// sym.SymNot: "not",
// sym.SymComment
// sym.SymFuncCall
// sym.SymFuncDef
// sym.SymList
// sym.SymLinkedList
// sym.SymDict
// sym.SymIndex
// sym.SymExpression
// sym.SymSelector // <selector> ::= <expr> "?" <selector-case> {":" <selector-case>} ["::" <default-selector-case>]
// sym.SymSelectorCase // <selector-case> ::= [<list>] "{" <multi-expr> "}"
// // sym.SymOpenComment // 0: '/*'
// // sym.SymClosedComment // 0: '*/'
// // sym.SymOneLineComment // 0: '//'
// keywordBase
SymKwAnd: {"and", SymClassOperator, PosInfix},
SymKwNot: {"not", SymClassOperator, PosInfix},
SymKwOr: {"or", SymClassOperator, PosInfix},
SymKwBut: {"but", SymClassOperator, PosInfix},
SymKwMap: {"map", SymClassOperator, PosInfix},
SymKwFilter: {"filter", SymClassOperator, PosInfix},
SymKwDigest: {"digest", SymClassOperator, PosInfix},
SymKwCat: {"cat", SymClassOperator, PosInfix},
SymKwJoin: {"join", SymClassOperator, PosInfix},
SymKwGroupBy: {"groupby", SymClassOperator, PosInfix},
SymKwFunc: {"func(", SymClassDeclaration, PosPrefix},
SymKwBuiltin: {"builtin", SymClassOperator, PosPrefix},
SymKwPlugin: {"plugin", SymClassOperator, PosPrefix},
SymKwIn: {"in", SymClassOperator, PosInfix},
SymKwAt: {"at", SymClassOperator, PosInfix},
SymKwInclude: {"include", SymClassOperator, PosPrefix},
SymKwNil: {"nil", SymClassValue, PosLeaf},
SymKwUnset: {"unset", SymClassOperator, PosPrefix},
sym.SymKwAnd: {"and", SymClassOperator, PosInfix},
sym.SymKwNot: {"not", SymClassOperator, PosInfix},
sym.SymKwOr: {"or", SymClassOperator, PosInfix},
sym.SymKwBut: {"but", SymClassOperator, PosInfix},
sym.SymKwMap: {"map", SymClassOperator, PosInfix},
sym.SymKwFilter: {"filter", SymClassOperator, PosInfix},
sym.SymKwDigest: {"digest", SymClassOperator, PosInfix},
sym.SymKwCat: {"cat", SymClassOperator, PosInfix},
sym.SymKwJoin: {"join", SymClassOperator, PosInfix},
sym.SymKwGroupBy: {"groupby", SymClassOperator, PosInfix},
sym.SymKwFunc: {"func(", SymClassDeclaration, PosPrefix},
sym.SymKwBuiltin: {"builtin", SymClassOperator, PosPrefix},
sym.SymKwPlugin: {"plugin", SymClassOperator, PosPrefix},
sym.SymKwIn: {"in", SymClassOperator, PosInfix},
sym.SymKwAt: {"at", SymClassOperator, PosInfix},
sym.SymKwInclude: {"include", SymClassOperator, PosPrefix},
sym.SymKwNil: {"nil", SymClassValue, PosLeaf},
sym.SymKwUnset: {"unset", SymClassOperator, PosPrefix},
}
}
func SymToString(sym Symbol) string {
func SymToString(sym sym.Symbol) string {
if s, ok := symbolMap[sym]; ok {
return s.repr
}
return ""
}
func SymListToString(symList []Symbol, quote bool) string {
func SymListToString(symList []sym.Symbol, quote bool) string {
var sb strings.Builder
if len(symList) == 0 {
sb.WriteString("<nothing>")
@@ -186,13 +188,13 @@ func SymListToString(symList []Symbol, quote bool) string {
}
func StringEndsWithOperator(s string) bool {
return endingOperator(s) != SymNone
return endingOperator(s) != sym.SymNone
}
func endingOperator(s string) (sym Symbol) {
func endingOperator(s string) (resultSym sym.Symbol) {
var matchLength = 0
var repr string
sym = SymNone
resultSym = sym.SymNone
lower := strings.TrimRight(strings.ToLower(s), " \t")
for symbol, spec := range symbolMap {
if len(spec.repr) > matchLength || repr == spec.repr {
@@ -200,9 +202,9 @@ func endingOperator(s string) (sym Symbol) {
if isNotEndingSymbol(spec) && repr != spec.repr {
repr = spec.repr
matchLength = len(spec.repr)
sym = symbol
resultSym = symbol
} else {
sym = SymNone
resultSym = sym.SymNone
break
// matchLength = 0
}
scan/symbol.go
-161
View File
@@ -1,161 +0,0 @@
// Copyright (c) 2024-2026 Celestino Amoroso (celestino.amoroso@gmail.com).
// All rights reserved.
// symbol.go
package scan
type Symbol int16
const (
SymUnknown Symbol = iota - 1 // -1: Unknown symbol
SymNone // 0: Null value for variable of type symbol
SymError // 1: Error reading from stream
SymEos // 2: End of stream
SymMinus // 3: '-'
SymMinusEqual // 4: '-='
SymDoubleMinus // 5: '--'
SymPlus // 6: '+'
SymPlusEqual // 7: '+='
SymDoublePlus // 8: '++'
SymStar // 9: '*'
SymDoubleStar // 10: '**'
SymSlash // 11: '/'
SymBackSlash // 12: '\'
SymVertBar // 13: '|'
SymDoubleVertBar // 14: '||'
SymComma // 15: ','
SymColon // 16: ':'
SymSemiColon // 17: ';'
SymDot // 18: '.'
SymDotSlash // 19: './'
SymQuote // 20: '\''
SymDoubleQuote // 21: '"'
SymBackTick // 22: '`'
SymExclamation // 23: '!'
SymQuestion // 24: '?'
SymAmpersand // 25: '&'
SymDoubleAmpersand // 26: '&&'
SymPercent // 27: '%'
SymAt // 28: '@'
SymUndescore // 29: '_'
SymEqual // 30: '='
SymColonEqual // 31: ':='
SymDoubleEqual // 32: '=='
SymLess // 33: '<'
SymLessOrEqual // 34: '<='
SymGreater // 35: '>'
SymGreaterOrEqual // 36: '>='
SymLessGreater // 37: '<>'
SymNotEqual // 38: '!='
SymDollar // 39: '$'
SymHash // 40: '#'
SymOpenRound // 41: '('
SymClosedRound // 42: ')'
SymOpenSquare // 43: '['
SymClosedSquare // 44: ']'
SymOpenBrace // 45: '{'
SymClosedBrace // 46: '}'
SymTilde // 47: '~'
SymDoubleQuestion // 48: '??'
SymQuestionEqual // 49: '?='
SymQuestionExclam // 50: '?!'
SymDoubleAt // 51: '@@'
SymDoubleColon // 52: '::'
SymDoubleGreater // 53: '>>'
SymDoubleLess // 54: '<<'
SymCaret // 55: '^'
SymDollarRound // 56: '$('
SymOpenClosedRound // 57: '()'
SymDoubleDollar // 58: '$$'
SymDoubleDot // 59: '..'
SymTripleDot // 60: '...'
SymStarEqual // 61: '*='
SymSlashEqual // 62: '/='
SymPercEqual // 63: '%='
SymDoubleLessEqual // 64: '<<='
SymDoubleGreaterEqual // 65: '>>='
SymAmpersandEqual // 66: '&='
SymVertBarEqual // 67: '|='
SymCaretEqual // 68: '^='
SymPlusGreater // 69: '+>'
SymLessPlus // 70: '<+'
SymChangeSign // 71: '-'
SymUnchangeSign // 72: '+''
SymDereference // 73: '*'
SymPreInc // 74: '++'
SymPreDec // 75: '--'
SymIdentifier // 76: identifier
SymBool // 77: boolean
SymInteger // 78: integer
SymVariable // 79: variable
SymFloat // 80: float
SymFraction // 81: fraction
SymString // 82: string
SymIterator // 83: iterator
SymOr // 84: 'or'
SymAnd // 85: 'and'
SymNot // 86: 'not'
SymComment // 87: comment
SymFuncCall // 88: function call
SymFuncDef // 89: function definition
SymList // 90: list
SymDict // 91: dict
SymIndex // 92: index
SymRange // 93: range [index : index]
SymExpression // 94: expression
SymSelector // 95: selector <selector> ::= <expr> "?" <selector-case> {":" <selector-case>} ["::" <default-selector-case>]
SymSelectorCase // 96: <selector-case> ::= [<list>] "{" <multi-expr> "}"
SymOpenSquareLess // 97: '[<'
SymGreaterClosedSquare // 98: '>]'
SymLinkedList // 99: linked-list
// SymOpenComment // 0: '/*'
// SymClosedComment // 0: '*/'
// SymOneLineComment // 0: '//'
keywordBase
)
const (
SymKwAnd = keywordBase + iota
SymKwNot
SymKwOr
SymKwBut
SymKwFunc
SymKwBuiltin
SymKwPlugin
SymKwIn
SymKwAt
SymKwInclude
SymKwMap
SymKwFilter
SymKwDigest
SymKwCat
SymKwGroupBy
SymKwJoin
SymKwNil
SymKwUnset
)
var keywords map[string]Symbol
func init() {
//keywords = make(map[string]Symbol)
keywords = map[string]Symbol{
"AND": SymKwAnd,
"BUILTIN": SymKwBuiltin,
"PLUGIN": SymKwPlugin,
"BUT": SymKwBut,
"FUNC": SymKwFunc,
"IN": SymKwIn,
"AT": SymKwAt,
"INCLUDE": SymKwInclude,
"MAP": SymKwMap,
"CAT": SymKwCat,
"FILTER": SymKwFilter,
"NOT": SymKwNot,
"OR": SymKwOr,
"NIL": SymKwNil,
"UNSET": SymKwUnset,
"DIGEST": SymKwDigest,
"JOIN": SymKwJoin,
"GROUPBY": SymKwGroupBy,
}
}
scan/t_scanner_test.go
+44 -42
View File
@@ -9,57 +9,59 @@ import (
"reflect"
"strings"
"testing"
"git.portale-stac.it/go-pkg/expr/sym"
)
func TestScanner(t *testing.T) {
type inputType struct {
source string
wantSym Symbol
wantSym sym.Symbol
wantValue any
wantErr error
}
inputs := []inputType{
/* 1 */ {`123`, SymInteger, int64(123), nil},
/* 2 */ {"=", SymEqual, nil, nil},
/* 3 */ {`--`, SymDoubleMinus, nil, nil},
/* 4 */ {`++`, SymDoublePlus, nil, nil},
/* 5 */ {`**`, SymDoubleStar, nil, nil},
/* 6 */ {`&&`, SymDoubleAmpersand, nil, nil},
/* 7 */ {`||`, SymDoubleVertBar, nil, nil},
/* 8 */ {`NOT`, SymKwNot, nil, nil},
/* 9 */ {`AND`, SymKwAnd, nil, nil},
/* 10 */ {`or`, SymKwOr, nil, nil},
/* 11 */ {`+=`, SymPlusEqual, nil, nil},
/* 12 */ {`-=`, SymMinusEqual, nil, nil},
/* 13 */ {`|`, SymVertBar, nil, nil},
/* 14 */ {`:`, SymColon, nil, nil},
/* 15 */ {`;`, SymSemiColon, nil, nil},
/* 16 */ {`.`, SymDot, nil, nil},
/* 17 */ {`0.5`, SymFloat, float64(0.5), nil},
/* 18 */ {`\\`, SymBackSlash, nil, nil},
/* 19 */ {"`", SymBackTick, nil, nil},
/* 20 */ {"?", SymQuestion, nil, nil},
/* 21 */ {"&", SymAmpersand, nil, nil},
/* 22 */ {"@", SymAt, nil, nil},
/* 23 */ {`#`, SymHash, nil, nil},
/* 24 */ {`%`, SymPercent, nil, nil},
/* 25 */ {`\'`, SymQuote, nil, nil},
/* 26 */ {`\"`, SymDoubleQuote, nil, nil},
/* 27 */ {`_`, SymUndescore, nil, nil},
/* 28 */ {`<>`, SymLessGreater, nil, nil},
/* 29 */ {`[`, SymOpenSquare, nil, nil},
/* 30 */ {`]`, SymClosedSquare, nil, nil},
/* 31 */ {`{`, SymOpenBrace, nil, nil},
/* 32 */ {`}`, SymClosedBrace, nil, nil},
/* 33 */ {`(`, SymOpenRound, nil, nil},
/* 34 */ {`)`, SymClosedRound, nil, nil},
/* 35 */ {`1E+2`, SymFloat, float64(100), nil},
/* 36 */ {`1E+x`, SymError, errors.New("[1:5] expected integer exponent, got x"), nil},
/* 37 */ {`$`, SymDollar, nil, nil},
/* 38 */ {`\`, SymError, errors.New("incomplete escape sequence"), nil},
/* 39 */ {`"string"`, SymString, "string", nil},
/* 40 */ {`identifier`, SymIdentifier, "identifier", nil},
/* 1 */ {`123`, sym.SymInteger, int64(123), nil},
/* 2 */ {"=", sym.SymEqual, nil, nil},
/* 3 */ {`--`, sym.SymDoubleMinus, nil, nil},
/* 4 */ {`++`, sym.SymDoublePlus, nil, nil},
/* 5 */ {`**`, sym.SymDoubleStar, nil, nil},
/* 6 */ {`&&`, sym.SymDoubleAmpersand, nil, nil},
/* 7 */ {`||`, sym.SymDoubleVertBar, nil, nil},
/* 8 */ {`NOT`, sym.SymKwNot, nil, nil},
/* 9 */ {`AND`, sym.SymKwAnd, nil, nil},
/* 10 */ {`or`, sym.SymKwOr, nil, nil},
/* 11 */ {`+=`, sym.SymPlusEqual, nil, nil},
/* 12 */ {`-=`, sym.SymMinusEqual, nil, nil},
/* 13 */ {`|`, sym.SymVertBar, nil, nil},
/* 14 */ {`:`, sym.SymColon, nil, nil},
/* 15 */ {`;`, sym.SymSemiColon, nil, nil},
/* 16 */ {`.`, sym.SymDot, nil, nil},
/* 17 */ {`0.5`, sym.SymFloat, float64(0.5), nil},
/* 18 */ {`\\`, sym.SymBackSlash, nil, nil},
/* 19 */ {"`", sym.SymBackTick, nil, nil},
/* 20 */ {"?", sym.SymQuestion, nil, nil},
/* 21 */ {"&", sym.SymAmpersand, nil, nil},
/* 22 */ {"@", sym.SymAt, nil, nil},
/* 23 */ {`#`, sym.SymHash, nil, nil},
/* 24 */ {`%`, sym.SymPercent, nil, nil},
/* 25 */ {`\'`, sym.SymQuote, nil, nil},
/* 26 */ {`\"`, sym.SymDoubleQuote, nil, nil},
/* 27 */ {`_`, sym.SymUndescore, nil, nil},
/* 28 */ {`<>`, sym.SymLessGreater, nil, nil},
/* 29 */ {`[`, sym.SymOpenSquare, nil, nil},
/* 30 */ {`]`, sym.SymClosedSquare, nil, nil},
/* 31 */ {`{`, sym.SymOpenBrace, nil, nil},
/* 32 */ {`}`, sym.SymClosedBrace, nil, nil},
/* 33 */ {`(`, sym.SymOpenRound, nil, nil},
/* 34 */ {`)`, sym.SymClosedRound, nil, nil},
/* 35 */ {`1E+2`, sym.SymFloat, float64(100), nil},
/* 36 */ {`1E+x`, sym.SymError, errors.New("[1:5] expected integer exponent, got x"), nil},
/* 37 */ {`$`, sym.SymDollar, nil, nil},
/* 38 */ {`\`, sym.SymError, errors.New("incomplete escape sequence"), nil},
/* 39 */ {`"string"`, sym.SymString, "string", nil},
/* 40 */ {`identifier`, sym.SymIdentifier, "identifier", nil},
}
for i, input := range inputs {
@@ -79,7 +81,7 @@ func TestScanner(t *testing.T) {
t.Errorf("%d: %q -> got = (nil), want %v (value %v [%T])", i+1, input.source, input.wantSym, input.wantValue, input.wantValue)
// } else if tk.Sym != input.wantSym || tk.Value != input.wantValue {
} else if tk.Sym != input.wantSym || !reflect.DeepEqual(tk.Value, input.wantValue) {
if tk.Sym == SymError && input.wantSym == tk.Sym {
if tk.Sym == sym.SymError && input.wantSym == tk.Sym {
if tkErr, tkOk := tk.Value.(error); tkOk {
if inputErr, inputOk := input.wantValue.(error); inputOk {
if tkErr.Error() != inputErr.Error() {
scan/term-constructor-registry.go
+6 -4
View File
@@ -4,17 +4,19 @@
// term-constructor-registry.go
package scan
import "git.portale-stac.it/go-pkg/expr/sym"
const initialRegistryCapacity = 10
type termContructor func(tk *Token) *Term
var constructorRegistry map[Symbol]termContructor = nil
var constructorRegistry map[sym.Symbol]termContructor = nil
func RegisterTermConstructor(sym Symbol, constructor termContructor) {
func RegisterTermConstructor(symmbol sym.Symbol, constructor termContructor) {
if constructorRegistry == nil {
constructorRegistry = make(map[Symbol]termContructor, initialRegistryCapacity)
constructorRegistry = make(map[sym.Symbol]termContructor, initialRegistryCapacity)
}
constructorRegistry[sym] = constructor
constructorRegistry[symmbol] = constructor
}
func NewTerm(tk *Token) (inst *Term) {
scan/term.go
+17 -4
View File
@@ -8,13 +8,14 @@ import (
"strings"
"git.portale-stac.it/go-pkg/expr/kern"
"git.portale-stac.it/go-pkg/expr/sym"
)
type TermPriority uint32
const (
PriNone TermPriority = iota
PriRange
PriInterval
PriBut
PriAssign
PriIterOp // map, filter, digest, etc
@@ -160,7 +161,7 @@ func (t *Term) SetParent(parent *Term) {
}
}
func (t *Term) Symbol() Symbol {
func (t *Term) Symbol() sym.Symbol {
return t.Tk.Sym
}
@@ -200,12 +201,24 @@ func (t *Term) GetRightChild() (c kern.Term) {
return
}
func (t *Term) IsSymbol(symbol sym.Symbol) bool {
return t.Symbol() == symbol
}
func (t *Term) IsOneOf(symbols ...sym.Symbol) bool {
return t.Tk.IsOneOf(symbols)
}
func (t *Term) IsAssign() bool {
return t.Symbol() == SymEqual
return t.Symbol() == sym.SymEqual
}
func (t *Term) IsVar() bool {
return t.Symbol() == SymVariable
return t.Symbol() == sym.SymVariable
}
func (t *Term) SetSymbol(symbol sym.Symbol) {
t.Tk.SetSymbol(symbol)
}
// func (t *Term) IsFunc() bool {
scan/token.go
+24 -22
View File
@@ -8,12 +8,14 @@ import (
"fmt"
"io"
"slices"
"git.portale-stac.it/go-pkg/expr/sym"
)
type Token struct {
row int
col int
Sym Symbol
Sym sym.Symbol
source string
Value any
}
@@ -36,19 +38,19 @@ func (tk *Token) String() string {
return tk.source
}
func NewToken(row, col int, sym Symbol, source string) *Token {
return &Token{row: row, col: col, Sym: sym, source: source}
func NewToken(row, col int, symbol sym.Symbol, source string) *Token {
return &Token{row: row, col: col, Sym: symbol, source: source}
}
func NewValueToken(row, col int, sym Symbol, source string, value any) *Token {
return &Token{row: row, col: col, Sym: sym, source: source, Value: value}
func NewValueToken(row, col int, symbol sym.Symbol, source string, value any) *Token {
return &Token{row: row, col: col, Sym: symbol, source: source, Value: value}
}
func NewErrorToken(row, col int, err error) *Token {
if err == io.EOF {
return NewToken(row, col, SymEos, "<EOF>")
return NewToken(row, col, sym.SymEos, "<EOF>")
}
return NewValueToken(row, col, SymError, fmt.Sprintf("[%d:%d]", row, col), err)
return NewValueToken(row, col, sym.SymError, fmt.Sprintf("[%d:%d]", row, col), err)
}
func (tk *Token) Source() string {
@@ -68,31 +70,31 @@ func (tk *Token) Clone() (c *Token) {
}
func (tk *Token) IsEos() bool {
return tk.Sym == SymEos
return tk.Sym == sym.SymEos
}
func (tk *Token) IsError() bool {
return tk.Sym == SymError
return tk.Sym == sym.SymError
}
func (tk *Token) IsTerm(termSymbols []Symbol) bool {
return tk.IsEos() || tk.IsError() || tk.IsOneOf(termSymbols)
func (tk *Token) IsTerm(symbols []sym.Symbol) bool {
return tk.IsEos() || tk.IsError() || tk.IsOneOf(symbols)
}
func (tk *Token) IsOneOf(termSymbols []Symbol) bool {
return termSymbols != nil && slices.Index(termSymbols, tk.Sym) >= 0
func (tk *Token) IsOneOf(symbols []sym.Symbol) bool {
return symbols != nil && slices.Index(symbols, tk.Sym) >= 0
}
func (tk *Token) IsOneOfA(termSymbols ...Symbol) bool {
return slices.Index(termSymbols, tk.Sym) >= 0
func (tk *Token) IsOneOfA(symbols ...sym.Symbol) bool {
return slices.Index(symbols, tk.Sym) >= 0
}
func (tk *Token) IsSymbol(sym Symbol) bool {
return tk.Sym == sym
func (tk *Token) IsSymbol(symbol sym.Symbol) bool {
return tk.Sym == symbol
}
func (tk *Token) SetSymbol(sym Symbol) {
tk.Sym = sym
func (tk *Token) SetSymbol(symbol sym.Symbol) {
tk.Sym = symbol
}
func (tk *Token) Errorf(template string, args ...any) (err error) {
@@ -101,7 +103,7 @@ func (tk *Token) Errorf(template string, args ...any) (err error) {
}
func (tk *Token) Error() (err error) {
if tk.Sym == SymError {
if tk.Sym == sym.SymError {
if msg, ok := tk.Value.(error); ok {
err = fmt.Errorf("[%d:%d] %v", tk.row, tk.col, msg)
}
@@ -110,7 +112,7 @@ func (tk *Token) Error() (err error) {
}
func (tk *Token) ErrorText() (err string) {
if tk.Sym == SymError {
if tk.Sym == sym.SymError {
if msg, ok := tk.Value.(error); ok {
err = msg.Error()
}
@@ -137,6 +139,6 @@ func (tk *Token) ErrorExpectedGotStringWithPrefix(prefix, symbol, got string) (e
return
}
func (tk *Token) ErrorExpectedOneOfGot(expected ...Symbol) (err error) {
func (tk *Token) ErrorExpectedOneOfGot(expected ...sym.Symbol) (err error) {
return tk.ErrorExpectedGotStringWithPrefix("expected one of ", SymListToString(expected, true), SymToString(tk.Sym))
}