several changes:

1. moved scan/symbol.go to new package sym and adapted all files that reference Symbol type and its values.
2. new type interval defined by begin, end and step values.
3. replaced operator-range.go with operator-interval.go.
4. replaced range operator begin:end with begin..end..step.
5. new implementation of sub-collection extraction based on new interval literal.
This commit is contained in:
2026-07-27 16:01:52 +02:00
parent 07aecfc4e7
commit 8e596d5979
63 changed files with 1125 additions and 755 deletions
+154 -127
View File
@@ -12,20 +12,23 @@ import (
"strconv"
"strings"
"git.portale-stac.it/go-pkg/expr/sym"
"git.portale-stac.it/go-pkg/expr/util"
)
type Scanner struct {
current *Token
prev *Token
stage *Token
stream *bufio.Reader
row int
column int
translations map[Symbol]Symbol
current *Token
prev *Token
stage *Token
stream *bufio.Reader
row int
column int
translations map[sym.Symbol]sym.Symbol
unreadByte byte
prevRowLength int
}
func NewScanner(s io.Reader, translations map[Symbol]Symbol) (inst *Scanner) {
func NewScanner(s io.Reader, translations map[sym.Symbol]sym.Symbol) (inst *Scanner) {
inst = &Scanner{
stream: bufio.NewReader(s),
row: 1,
@@ -36,15 +39,15 @@ func NewScanner(s io.Reader, translations map[Symbol]Symbol) (inst *Scanner) {
return inst
}
func DefaultTranslations() map[Symbol]Symbol {
return map[Symbol]Symbol{
SymDoubleAmpersand: SymAnd,
SymKwAnd: SymAnd,
SymDoubleVertBar: SymOr,
SymKwOr: SymOr,
// SymTilde: SymNot,
SymKwNot: SymNot,
SymLessGreater: SymNotEqual,
func DefaultTranslations() map[sym.Symbol]sym.Symbol {
return map[sym.Symbol]sym.Symbol{
sym.SymDoubleAmpersand: sym.SymAnd,
sym.SymKwAnd: sym.SymAnd,
sym.SymDoubleVertBar: sym.SymOr,
sym.SymKwOr: sym.SymOr,
sym.SymKwNot: sym.SymNot,
sym.SymLessGreater: sym.SymNotEqual,
// SymTilde: sym.SymNot,
}
}
@@ -56,9 +59,25 @@ func (scanner *Scanner) Current() *Token {
return scanner.current
}
func (scanner *Scanner) setUnreadChar(ch byte) {
scanner.column--
if scanner.column == 0 {
scanner.column = scanner.prevRowLength
scanner.row--
}
scanner.unreadByte = ch
}
func (scanner *Scanner) readChar() (ch byte, err error) {
if ch, err = scanner.stream.ReadByte(); err == nil {
if scanner.unreadByte != 0 {
ch = scanner.unreadByte
scanner.unreadByte = 0
} else {
ch, err = scanner.stream.ReadByte()
}
if err == nil {
if ch == '\n' {
scanner.prevRowLength = scanner.column
scanner.row++
scanner.column = 0
} else {
@@ -127,237 +146,237 @@ func (scanner *Scanner) fetchNextToken() (tk *Token) {
switch ch {
case '+':
if next, _ := scanner.peek(); next == '+' {
tk = scanner.moveOn(SymDoublePlus, ch, next)
tk = scanner.moveOn(sym.SymDoublePlus, ch, next)
} else if next == '=' {
tk = scanner.moveOn(SymPlusEqual, ch, next)
tk = scanner.moveOn(sym.SymPlusEqual, ch, next)
} else if next == '>' {
tk = scanner.moveOn(SymPlusGreater, ch, next)
tk = scanner.moveOn(sym.SymPlusGreater, ch, next)
} else {
tk = scanner.MakeToken(SymPlus, ch)
tk = scanner.MakeToken(sym.SymPlus, ch)
}
case '-':
if next, _ := scanner.peek(); next == '-' {
tk = scanner.moveOn(SymDoubleMinus, ch, next)
tk = scanner.moveOn(sym.SymDoubleMinus, ch, next)
} else if next == '=' {
tk = scanner.moveOn(SymMinusEqual, ch, next)
tk = scanner.moveOn(sym.SymMinusEqual, ch, next)
} else {
tk = scanner.MakeToken(SymMinus, ch)
tk = scanner.MakeToken(sym.SymMinus, ch)
}
case '*':
if next, _ := scanner.peek(); next == '*' {
tk = scanner.moveOn(SymDoubleStar, ch, next)
tk = scanner.moveOn(sym.SymDoubleStar, ch, next)
// } else if next == '/' {
// tk = self.moveOn(SymClosedComment, ch, next)
} else if next, _ = scanner.peek(); next == '=' {
tk = scanner.moveOn(SymStarEqual, ch, next)
tk = scanner.moveOn(sym.SymStarEqual, ch, next)
} else {
tk = scanner.MakeToken(SymStar, ch)
tk = scanner.MakeToken(sym.SymStar, ch)
}
case '/':
if next, _ := scanner.peek(); next == '*' {
scanner.readChar()
tk = scanner.fetchBlockComment()
} else if next, _ = scanner.peek(); next == '=' {
tk = scanner.moveOn(SymSlashEqual, ch, next)
tk = scanner.moveOn(sym.SymSlashEqual, ch, next)
} else if next == '/' {
scanner.readChar()
tk = scanner.fetchOnLineComment()
} else {
tk = scanner.MakeToken(SymSlash, ch)
tk = scanner.MakeToken(sym.SymSlash, ch)
}
case '\\':
if escape {
tk = scanner.MakeToken(SymBackSlash, ch)
tk = scanner.MakeToken(sym.SymBackSlash, ch)
escape = false
} else {
escape = true
}
case '|':
if next, _ := scanner.peek(); next == '|' {
tk = scanner.moveOn(SymDoubleVertBar, ch, next)
tk = scanner.moveOn(sym.SymDoubleVertBar, ch, next)
} else if next, _ = scanner.peek(); next == '=' {
tk = scanner.moveOn(SymVertBarEqual, ch, next)
tk = scanner.moveOn(sym.SymVertBarEqual, ch, next)
} else {
tk = scanner.MakeToken(SymVertBar, ch)
tk = scanner.MakeToken(sym.SymVertBar, ch)
}
case ',':
tk = scanner.MakeToken(SymComma, ch)
tk = scanner.MakeToken(sym.SymComma, ch)
case '^':
if next, _ := scanner.peek(); next == '=' {
tk = scanner.moveOn(SymCaretEqual, ch, next)
tk = scanner.moveOn(sym.SymCaretEqual, ch, next)
} else {
tk = scanner.MakeToken(SymCaret, ch)
tk = scanner.MakeToken(sym.SymCaret, ch)
}
case ':':
if next, _ := scanner.peek(); next == ':' {
tk = scanner.moveOn(SymDoubleColon, ch, next)
tk = scanner.moveOn(sym.SymDoubleColon, ch, next)
} else if next == '=' {
tk = scanner.moveOn(SymColonEqual, ch, next)
tk = scanner.moveOn(sym.SymColonEqual, ch, next)
} else {
tk = scanner.MakeToken(SymColon, ch)
tk = scanner.MakeToken(sym.SymColon, ch)
}
case ';':
tk = scanner.MakeToken(SymSemiColon, ch)
tk = scanner.MakeToken(sym.SymSemiColon, ch)
case '.':
//if next, _ := self.peek(); next >= '0' && next <= '9' {
// tk = self.parseNumber(ch)
//} else if next == '/' {
if next, _ := scanner.peek(); next == '/' {
tk = scanner.moveOn(SymDotSlash, ch, next)
tk = scanner.moveOn(sym.SymDotSlash, ch, next)
} else if next == '.' {
tk = scanner.moveOn(sym.SymDoubleDot, ch, next)
// TODO: this section needs verification
if next1, _ := scanner.peek(); next1 == '.' {
tk = scanner.moveOn(SymTripleDot, ch, next, next1)
} else {
tk = scanner.moveOn(SymDoubleDot, ch, next)
tk = scanner.moveOn(sym.SymTripleDot, next, next1)
}
} else {
tk = scanner.MakeToken(SymDot, ch)
tk = scanner.MakeToken(sym.SymDot, ch)
}
case '\'':
if escape {
tk = scanner.MakeToken(SymQuote, ch)
tk = scanner.MakeToken(sym.SymQuote, ch)
escape = false
} else {
tk = scanner.fetchString(ch, true)
}
case '"':
if escape {
tk = scanner.MakeToken(SymDoubleQuote, ch)
tk = scanner.MakeToken(sym.SymDoubleQuote, ch)
escape = false
} else {
tk = scanner.fetchString(ch, true)
}
case '`':
tk = scanner.MakeToken(SymBackTick, ch)
tk = scanner.MakeToken(sym.SymBackTick, ch)
case '!':
if next, _ := scanner.peek(); next == '=' {
tk = scanner.moveOn(SymNotEqual, ch, next)
tk = scanner.moveOn(sym.SymNotEqual, ch, next)
} else {
tk = scanner.MakeToken(SymExclamation, ch)
tk = scanner.MakeToken(sym.SymExclamation, ch)
}
case '?':
if next, _ := scanner.peek(); next == '?' {
tk = scanner.moveOn(SymDoubleQuestion, ch, next)
tk = scanner.moveOn(sym.SymDoubleQuestion, ch, next)
} else if next == '=' {
tk = scanner.moveOn(SymQuestionEqual, ch, next)
tk = scanner.moveOn(sym.SymQuestionEqual, ch, next)
} else if next == '!' {
tk = scanner.moveOn(SymQuestionExclam, ch, next)
tk = scanner.moveOn(sym.SymQuestionExclam, ch, next)
} else {
tk = scanner.MakeToken(SymQuestion, ch)
tk = scanner.MakeToken(sym.SymQuestion, ch)
}
case '&':
if next, _ := scanner.peek(); next == '&' {
tk = scanner.moveOn(SymDoubleAmpersand, ch, next)
tk = scanner.moveOn(sym.SymDoubleAmpersand, ch, next)
} else if next, _ = scanner.peek(); next == '=' {
tk = scanner.moveOn(SymAmpersandEqual, ch, next)
tk = scanner.moveOn(sym.SymAmpersandEqual, ch, next)
} else {
tk = scanner.MakeToken(SymAmpersand, ch)
tk = scanner.MakeToken(sym.SymAmpersand, ch)
}
case '%':
if next, _ := scanner.peek(); next == '=' {
tk = scanner.moveOn(SymPercEqual, ch, next)
tk = scanner.moveOn(sym.SymPercEqual, ch, next)
} else {
tk = scanner.MakeToken(SymPercent, ch)
tk = scanner.MakeToken(sym.SymPercent, ch)
}
case '#':
tk = scanner.MakeToken(SymHash, ch)
tk = scanner.MakeToken(sym.SymHash, ch)
case '@':
if next, _ := scanner.peek(); (next >= 'a' && next <= 'z') || (next >= 'A' && next <= 'Z') {
scanner.readChar()
if tk = scanner.fetchIdentifier(next); tk.Sym == SymIdentifier {
if tk = scanner.fetchIdentifier(next); tk.Sym == sym.SymIdentifier {
//tk.Sym = SymIdRef
tk.source = "@" + tk.source
} else {
tk = scanner.makeErrorToken(fmt.Errorf("invalid variable reference %q", tk.source))
}
} else if next == '@' {
tk = scanner.moveOn(SymDoubleAt, ch, next)
tk = scanner.moveOn(sym.SymDoubleAt, ch, next)
} else {
tk = scanner.MakeToken(SymAt, ch)
tk = scanner.MakeToken(sym.SymAt, ch)
}
case '_':
tk = scanner.MakeToken(SymUndescore, ch)
tk = scanner.MakeToken(sym.SymUndescore, ch)
case '=':
if next, _ := scanner.peek(); next == '=' {
tk = scanner.moveOn(SymDoubleEqual, ch, next)
tk = scanner.moveOn(sym.SymDoubleEqual, ch, next)
} else {
tk = scanner.MakeToken(SymEqual, ch)
tk = scanner.MakeToken(sym.SymEqual, ch)
}
case '<':
if next, _ := scanner.peek(); next == '=' {
tk = scanner.moveOn(SymLessOrEqual, ch, next)
tk = scanner.moveOn(sym.SymLessOrEqual, ch, next)
} else if next == '<' {
scanner.readChar()
next2, _ := scanner.readChar()
scanner.unreadChar()
if next2 == '=' {
tk = scanner.moveOn(SymDoubleLessEqual, ch, next, next2)
tk = scanner.moveOn(sym.SymDoubleLessEqual, ch, next, next2)
} else {
tk = scanner.accept(SymDoubleLess, ch, next)
tk = scanner.accept(sym.SymDoubleLess, ch, next)
}
} else if next == '>' {
tk = scanner.moveOn(SymLessGreater, ch, next)
tk = scanner.moveOn(sym.SymLessGreater, ch, next)
} else if next == '+' {
tk = scanner.moveOn(SymLessPlus, ch, next)
tk = scanner.moveOn(sym.SymLessPlus, ch, next)
} else {
tk = scanner.MakeToken(SymLess, ch)
tk = scanner.MakeToken(sym.SymLess, ch)
}
case '>':
if next, _ := scanner.peek(); next == '=' {
tk = scanner.moveOn(SymGreaterOrEqual, ch, next)
tk = scanner.moveOn(sym.SymGreaterOrEqual, ch, next)
} else if next == '>' {
scanner.readChar()
next2, _ := scanner.readChar()
scanner.unreadChar()
if next2 == '=' {
tk = scanner.moveOn(SymDoubleGreaterEqual, ch, next, next2)
tk = scanner.moveOn(sym.SymDoubleGreaterEqual, ch, next, next2)
} else {
tk = scanner.accept(SymDoubleGreater, ch, next)
tk = scanner.accept(sym.SymDoubleGreater, ch, next)
}
} else if next == ']' {
tk = scanner.moveOn(SymGreaterClosedSquare, ch, next)
tk = scanner.moveOn(sym.SymGreaterClosedSquare, ch, next)
} else {
tk = scanner.MakeToken(SymGreater, ch)
tk = scanner.MakeToken(sym.SymGreater, ch)
}
case '$':
if next, _ := scanner.peek(); next == '(' {
tk = scanner.moveOn(SymDollarRound, ch, next)
tk = scanner.moveOn(sym.SymDollarRound, ch, next)
tk.source += ")"
} else if next == '$' {
tk = scanner.moveOn(SymDoubleDollar, ch, next)
tk = scanner.moveOn(sym.SymDoubleDollar, ch, next)
} else if next == '{' {
scanner.readChar()
if tk = scanner.fetchString('}', false); tk != nil {
tk.Sym = SymIdentifier
tk.Sym = sym.SymIdentifier
}
} else if next == '_' || (next >= 'a' && next <= 'z') || (next >= 'A' && next <= 'Z') {
scanner.readChar()
tk = scanner.fetchIdentifier(next)
} else {
tk = scanner.MakeToken(SymDollar, ch)
tk = scanner.MakeToken(sym.SymDollar, ch)
}
case '(':
// if next, _ := scanner.peek(); next == ')' {
// tk = scanner.moveOn(SymOpenClosedRound, ch, next)
// } else {
tk = scanner.MakeToken(SymOpenRound, ch)
tk = scanner.MakeToken(sym.SymOpenRound, ch)
// }
case ')':
tk = scanner.MakeToken(SymClosedRound, ch)
tk = scanner.MakeToken(sym.SymClosedRound, ch)
case '[':
if next, _ := scanner.peek(); next == '<' {
tk = scanner.moveOn(SymOpenSquareLess, ch, next)
tk = scanner.moveOn(sym.SymOpenSquareLess, ch, next)
} else {
tk = scanner.MakeToken(SymOpenSquare, ch)
tk = scanner.MakeToken(sym.SymOpenSquare, ch)
}
case ']':
tk = scanner.MakeToken(SymClosedSquare, ch)
tk = scanner.MakeToken(sym.SymClosedSquare, ch)
case '{':
tk = scanner.MakeToken(SymOpenBrace, ch)
tk = scanner.MakeToken(sym.SymOpenBrace, ch)
case '}':
tk = scanner.MakeToken(SymClosedBrace, ch)
tk = scanner.MakeToken(sym.SymClosedBrace, ch)
case '~':
tk = scanner.MakeToken(SymTilde, ch)
tk = scanner.MakeToken(sym.SymTilde, ch)
case 0:
if escape {
tk = scanner.makeErrorToken(errors.New("incomplete escape sequence"))
@@ -365,9 +384,9 @@ func (scanner *Scanner) fetchNextToken() (tk *Token) {
escape = false
default:
if /*ch == '_' ||*/ (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') {
if tk = scanner.fetchIdentifier(ch); tk.Sym == SymKwFunc {
if tk = scanner.fetchIdentifier(ch); tk.Sym == sym.SymKwFunc {
if next, _ := scanner.peek(); next == '(' {
tk = scanner.moveOn(SymFuncDef, ch, next)
tk = scanner.moveOn(sym.SymFuncDef, ch, next)
}
}
} else if ch >= '0' && ch <= '9' {
@@ -415,19 +434,20 @@ func (scanner *Scanner) initBase(currentFirstCh byte) (firstCh byte, numBase int
numBase = 10
if ch, err = scanner.peek(); err == nil {
if ch == 'b' || ch == 'B' {
switch ch {
case 'b', 'B':
numBase = 2
digitType = "binary"
scanner.readChar()
digitFunc = isBinaryDigit
firstCh, err = scanner.readChar()
} else if ch == 'o' || ch == 'O' {
case 'o', 'O':
numBase = 8
digitType = "octal"
scanner.readChar()
digitFunc = isOctalDigit
firstCh, err = scanner.readChar()
} else if ch == 'x' || ch == 'X' {
case 'x', 'X':
numBase = 16
digitType = "hex"
scanner.readChar()
@@ -449,7 +469,7 @@ func (scanner *Scanner) initBase(currentFirstCh byte) (firstCh byte, numBase int
func (scanner *Scanner) parseNumber(firstCh byte) (tk *Token) {
var err error
var ch byte
var sym Symbol = SymInteger
var symbol sym.Symbol = sym.SymInteger
var sb strings.Builder
var isDigit func(byte) bool = isDecimalDigit
var numBase = 10
@@ -463,18 +483,24 @@ func (scanner *Scanner) parseNumber(firstCh byte) (tk *Token) {
if numBase == 10 {
if err == nil && ch == '.' {
sym = SymFloat
sb.WriteByte(ch)
ch, err = scanner.readChar()
if ch >= '0' && ch <= '9' {
for ; err == nil && (ch >= '0' && ch <= '9'); ch, err = scanner.readChar() {
sb.WriteByte(ch)
if next, _ := scanner.peek(); next == '.' {
// do nothing, this is an interval operator
scanner.setUnreadChar('.')
} else {
symbol = sym.SymFloat
sb.WriteByte(ch)
ch, err = scanner.readChar()
if ch >= '0' && ch <= '9' {
for ; err == nil && (ch >= '0' && ch <= '9'); ch, err = scanner.readChar() {
sb.WriteByte(ch)
}
}
}
}
if err == nil {
if ch == 'e' || ch == 'E' {
sym = SymFloat
switch ch {
case 'e', 'E':
symbol = sym.SymFloat
sb.WriteByte(ch)
if ch, err = scanner.readChar(); err == nil {
if ch == '+' || ch == '-' {
@@ -489,8 +515,8 @@ func (scanner *Scanner) parseNumber(firstCh byte) (tk *Token) {
err = fmt.Errorf("[%d:%d] expected integer exponent, got %c", scanner.row, scanner.column, ch)
}
}
} else if ch == '(' {
sym = SymFraction
case '(':
symbol = sym.SymFraction
sb.WriteByte(ch)
ch, err = scanner.readChar()
for ; err == nil && (ch >= '0' && ch <= '9'); ch, err = scanner.readChar() {
@@ -514,15 +540,16 @@ func (scanner *Scanner) parseNumber(firstCh byte) (tk *Token) {
var value any
_ = scanner.sync(err) // TODO: Check this function
txt := sb.String()
if sym == SymFloat {
switch symbol {
case sym.SymFloat:
value, err = strconv.ParseFloat(txt, 64)
} else if sym == SymFraction {
case sym.SymFraction:
// value, err = kern.MakeGeneratingFraction(txt)
value = txt
} else {
default:
value, err = strconv.ParseInt(txt, numBase, 64)
}
tk = scanner.MakeValueToken(sym, txt, value)
tk = scanner.MakeValueToken(symbol, txt, value)
// if err == nil {
// tk = scanner.MakeValueToken(sym, txt, value)
// } else {
@@ -546,17 +573,17 @@ func (scanner *Scanner) fetchIdentifier(firstCh byte) (tk *Token) {
} else {
txt := sb.String()
uptxt := strings.ToUpper(txt)
if sym, ok := keywords[uptxt]; ok {
tk = scanner.makeKeywordToken(sym, uptxt)
if symbol, ok := sym.Keywords[uptxt]; ok {
tk = scanner.makeKeywordToken(symbol, uptxt)
} else if uptxt == `TRUE` {
tk = scanner.MakeValueToken(SymBool, txt, true)
tk = scanner.MakeValueToken(sym.SymBool, txt, true)
} else if uptxt == `FALSE` {
tk = scanner.MakeValueToken(SymBool, txt, false)
tk = scanner.MakeValueToken(sym.SymBool, txt, false)
} else if ch, _ := scanner.peek(); ch == '(' {
scanner.readChar()
tk = scanner.MakeValueToken(SymFuncCall, txt+"(", txt)
tk = scanner.MakeValueToken(sym.SymFuncCall, txt+"(", txt)
} else {
tk = scanner.MakeValueToken(SymIdentifier, txt, txt)
tk = scanner.MakeValueToken(sym.SymIdentifier, txt, txt)
}
}
@@ -577,14 +604,14 @@ func (scanner *Scanner) fetchIdentifier(firstCh byte) (tk *Token) {
}
func (scanner *Scanner) fetchBlockComment() *Token {
return scanner.fetchUntil(SymComment, false, '*', '/')
return scanner.fetchUntil(sym.SymComment, false, '*', '/')
}
func (scanner *Scanner) fetchOnLineComment() *Token {
return scanner.fetchUntil(SymComment, true, '\n')
return scanner.fetchUntil(sym.SymComment, true, '\n')
}
func (scanner *Scanner) fetchUntil(sym Symbol, allowEos bool, endings ...byte) (tk *Token) {
func (scanner *Scanner) fetchUntil(sym sym.Symbol, allowEos bool, endings ...byte) (tk *Token) {
var err error
var ch byte
var sb strings.Builder
@@ -653,9 +680,9 @@ func (scanner *Scanner) fetchString(termCh byte, addQuote bool) (tk *Token) {
} else {
txt := sb.String()
if addQuote {
tk = scanner.MakeValueToken(SymString, `"`+txt+`"`, txt)
tk = scanner.MakeValueToken(sym.SymString, `"`+txt+`"`, txt)
} else {
tk = scanner.MakeValueToken(SymString, txt, txt)
tk = scanner.MakeValueToken(sym.SymString, txt, txt)
}
}
return
@@ -677,7 +704,7 @@ func (scanner *Scanner) skipBlanks() (err error) {
return
}
func (scanner *Scanner) translate(sym Symbol) Symbol {
func (scanner *Scanner) translate(sym sym.Symbol) sym.Symbol {
if scanner.translations != nil {
if translatedSym, ok := scanner.translations[sym]; ok {
return translatedSym
@@ -686,7 +713,7 @@ func (scanner *Scanner) translate(sym Symbol) Symbol {
return sym
}
func (scanner *Scanner) moveOn(sym Symbol, chars ...byte) (tk *Token) {
func (scanner *Scanner) moveOn(sym sym.Symbol, chars ...byte) (tk *Token) {
tk = NewToken(scanner.row, scanner.column, scanner.translate(sym), string(chars))
// for i := 1; i < len(chars); i++ {
if len(chars) > 1 {
@@ -696,22 +723,22 @@ func (scanner *Scanner) moveOn(sym Symbol, chars ...byte) (tk *Token) {
return
}
func (scanner *Scanner) accept(sym Symbol, chars ...byte) (tk *Token) {
func (scanner *Scanner) accept(sym sym.Symbol, chars ...byte) (tk *Token) {
tk = NewToken(scanner.row, scanner.column, scanner.translate(sym), string(chars))
return
}
func (scanner *Scanner) MakeToken(sym Symbol, chars ...byte) (tk *Token) {
func (scanner *Scanner) MakeToken(sym sym.Symbol, chars ...byte) (tk *Token) {
tk = NewToken(scanner.row, scanner.column, scanner.translate(sym), string(chars))
return
}
func (scanner *Scanner) makeKeywordToken(sym Symbol, upperCaseKeyword string) (tk *Token) {
func (scanner *Scanner) makeKeywordToken(sym sym.Symbol, upperCaseKeyword string) (tk *Token) {
tk = NewToken(scanner.row, scanner.column, scanner.translate(sym), upperCaseKeyword)
return
}
func (scanner *Scanner) MakeValueToken(sym Symbol, source string, value any) (tk *Token) {
func (scanner *Scanner) MakeValueToken(sym sym.Symbol, source string, value any) (tk *Token) {
tk = NewValueToken(scanner.row, scanner.column, scanner.translate(sym), source, value)
return
}