|
@@ -10,8 +10,8 @@ import (
|
|
|
"os"
|
|
|
_ "reflect"
|
|
|
_ "runtime"
|
|
|
+ "strconv"
|
|
|
"strings"
|
|
|
- "strconv"
|
|
|
"unicode"
|
|
|
_ "unicode"
|
|
|
// "gitlab.com/beoran/woe/graphviz"
|
|
@@ -27,6 +27,11 @@ type Lexer struct {
|
|
|
io.RuneScanner
|
|
|
buffer []rune
|
|
|
Current rune
|
|
|
+ LoggerWrapper
|
|
|
+}
|
|
|
+
|
|
|
+func (lexer *Lexer) SetLogger(logger Logger) {
|
|
|
+ lexer.LoggerWrapper = LoggerWrapper{logger}
|
|
|
}
|
|
|
|
|
|
func (lexer *Lexer) ClearBuffer() {
|
|
@@ -34,70 +39,86 @@ func (lexer *Lexer) ClearBuffer() {
|
|
|
}
|
|
|
|
|
|
func (lexer *Lexer) MakeIntegerToken() Token {
|
|
|
- var sbuffer = string(lexer.buffer)
|
|
|
- i, err := strconv.ParseInt(sbuffer, 0, 64)
|
|
|
- if err == nil {
|
|
|
- lexer.ClearBuffer()
|
|
|
- return NewToken(TokenKindInteger, IntValue(i), lexer.Position)
|
|
|
- } else {
|
|
|
- lexer.ClearBuffer()
|
|
|
- return lexer.MakeErrorToken(err);
|
|
|
- }
|
|
|
+ var sbuffer = string(lexer.buffer)
|
|
|
+ i, err := strconv.ParseInt(sbuffer, 0, 64)
|
|
|
+ if err == nil {
|
|
|
+ lexer.ClearBuffer()
|
|
|
+ return NewToken(TokenKindInteger, IntValue(i), lexer.Position)
|
|
|
+ } else {
|
|
|
+ lexer.ClearBuffer()
|
|
|
+ return lexer.MakeErrorToken(err)
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
func (lexer *Lexer) MakeFloatToken() Token {
|
|
|
- var sbuffer = string(lexer.buffer)
|
|
|
- f, err := strconv.ParseFloat(sbuffer, 64)
|
|
|
- if err == nil {
|
|
|
- lexer.ClearBuffer()
|
|
|
- return NewToken(TokenKindFloat, FloatValue(f), lexer.Position)
|
|
|
- } else {
|
|
|
- lexer.ClearBuffer()
|
|
|
- return lexer.MakeErrorToken(err);
|
|
|
- }
|
|
|
+ var sbuffer = string(lexer.buffer)
|
|
|
+ f, err := strconv.ParseFloat(sbuffer, 64)
|
|
|
+ if err == nil {
|
|
|
+ lexer.ClearBuffer()
|
|
|
+ return NewToken(TokenKindFloat, FloatValue(f), lexer.Position)
|
|
|
+ } else {
|
|
|
+ lexer.ClearBuffer()
|
|
|
+ return lexer.MakeErrorToken(err)
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
func (lexer *Lexer) MakeBooleanToken(b bool) Token {
|
|
|
- lexer.ClearBuffer()
|
|
|
- return NewToken(TokenKindBoolean, BoolValue(b), lexer.Position)
|
|
|
+ lexer.ClearBuffer()
|
|
|
+ return NewToken(TokenKindBoolean, BoolValue(b), lexer.Position)
|
|
|
}
|
|
|
|
|
|
-
|
|
|
func (lexer *Lexer) MakeStringValueToken(kind TokenKind) Token {
|
|
|
- var sbuffer = string(lexer.buffer)
|
|
|
- return NewToken(kind, StringValue(sbuffer), lexer.Position)
|
|
|
-}
|
|
|
-
|
|
|
-func (lexer *Lexer) MakeToken(kind TokenKind) Token {
|
|
|
- switch (kind) {
|
|
|
- case TokenKindInteger : return lexer.MakeIntegerToken()
|
|
|
- case TokenKindFloat : return lexer.MakeFloatToken()
|
|
|
- case TokenKindString : fallthrough
|
|
|
- case TokenKindSymbol : fallthrough
|
|
|
- case TokenKindType : fallthrough
|
|
|
- case TokenKindError : fallthrough
|
|
|
- case TokenKindWord : return lexer.MakeStringValueToken(kind)
|
|
|
- case TokenKindBoolean : fallthrough
|
|
|
- case TokenKindGet : fallthrough
|
|
|
- case TokenKindSet : fallthrough
|
|
|
- case TokenKindOpenBlock : fallthrough
|
|
|
- case TokenKindCloseBlock: fallthrough
|
|
|
- case TokenKindOpenList : fallthrough
|
|
|
- case TokenKindCloseList : fallthrough
|
|
|
- case TokenKindOpenParen : fallthrough
|
|
|
- case TokenKindCloseParen: fallthrough
|
|
|
- case TokenKindEOX : fallthrough
|
|
|
- case TokenKindEOF :
|
|
|
- val := StringValue(string(lexer.buffer))
|
|
|
- lexer.ClearBuffer()
|
|
|
- return NewToken(kind, val, lexer.Position)
|
|
|
- default :
|
|
|
- return lexer.MakeErrorfToken("Internal error on token type %s", kind)
|
|
|
- }
|
|
|
+ var sbuffer = string(lexer.buffer)
|
|
|
+ return NewToken(kind, StringValue(sbuffer), lexer.Position)
|
|
|
+}
|
|
|
+
|
|
|
+func (lexer *Lexer) MakeToken(kind TokenKind) Token {
|
|
|
+ switch kind {
|
|
|
+ case TokenKindInteger:
|
|
|
+ return lexer.MakeIntegerToken()
|
|
|
+ case TokenKindFloat:
|
|
|
+ return lexer.MakeFloatToken()
|
|
|
+ case TokenKindString:
|
|
|
+ fallthrough
|
|
|
+ case TokenKindSymbol:
|
|
|
+ fallthrough
|
|
|
+ case TokenKindType:
|
|
|
+ fallthrough
|
|
|
+ case TokenKindError:
|
|
|
+ fallthrough
|
|
|
+ case TokenKindWord:
|
|
|
+ return lexer.MakeStringValueToken(kind)
|
|
|
+ case TokenKindBoolean:
|
|
|
+ fallthrough
|
|
|
+ case TokenKindGet:
|
|
|
+ fallthrough
|
|
|
+ case TokenKindSet:
|
|
|
+ fallthrough
|
|
|
+ case TokenKindOpenBlock:
|
|
|
+ fallthrough
|
|
|
+ case TokenKindCloseBlock:
|
|
|
+ fallthrough
|
|
|
+ case TokenKindOpenList:
|
|
|
+ fallthrough
|
|
|
+ case TokenKindCloseList:
|
|
|
+ fallthrough
|
|
|
+ case TokenKindOpenParen:
|
|
|
+ fallthrough
|
|
|
+ case TokenKindCloseParen:
|
|
|
+ fallthrough
|
|
|
+ case TokenKindEOX:
|
|
|
+ fallthrough
|
|
|
+ case TokenKindEOF:
|
|
|
+ val := StringValue(string(lexer.buffer))
|
|
|
+ lexer.ClearBuffer()
|
|
|
+ return NewToken(kind, val, lexer.Position)
|
|
|
+ default:
|
|
|
+ return lexer.MakeErrorfToken("Internal error on token type %s", kind)
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
func (lexer Lexer) MakeErrorToken(err error) Token {
|
|
|
- return NewToken(TokenKindError, err.Error(), lexer.Position)
|
|
|
+ return NewToken(TokenKindError, ErrorValue{err}, lexer.Position)
|
|
|
}
|
|
|
|
|
|
func (lexer Lexer) MakeErrorfToken(format string, va ...interface{}) Token {
|
|
@@ -106,7 +127,7 @@ func (lexer Lexer) MakeErrorfToken(format string, va ...interface{}) Token {
|
|
|
}
|
|
|
|
|
|
func (lexer Lexer) MakeEOFToken() Token {
|
|
|
- return NewToken(TokenKindEOF, "", lexer.Position)
|
|
|
+ return NewToken(TokenKindEOF, &EmptyValue{}, lexer.Position)
|
|
|
}
|
|
|
|
|
|
func (lexer *Lexer) Peek() (rune, error) {
|
|
@@ -208,11 +229,60 @@ func (lexer *Lexer) SkipWhile(predicate func(rune) bool) (bool, error) {
|
|
|
}
|
|
|
|
|
|
func isSpace(r rune) bool {
|
|
|
- return r == ' ' || r == '\t'
|
|
|
+ return r == ' ' || r == '\t' || r == '\v' || r == '\r'
|
|
|
+}
|
|
|
+
|
|
|
+func isComment(r rune) bool {
|
|
|
+ return r == '#'
|
|
|
}
|
|
|
|
|
|
func (lexer *Lexer) SkipSpace() error {
|
|
|
- _, err := lexer.SkipWhile(isSpace)
|
|
|
+ r, err := lexer.Skip()
|
|
|
+ lexer.LogDebug("Skipping %c.", r)
|
|
|
+ if err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ for ; isSpace(r) && err == nil; r, err = lexer.Skip() {
|
|
|
+ }
|
|
|
+ return err
|
|
|
+}
|
|
|
+
|
|
|
+func (lexer *Lexer) SkipBlockComment() error {
|
|
|
+ var err error
|
|
|
+ var r rune
|
|
|
+ lexer.LogDebug("Skipping block comment.")
|
|
|
+ for block := 1; block > 0; {
|
|
|
+ r, err = lexer.Skip()
|
|
|
+ if err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ if r == '{' {
|
|
|
+ block++
|
|
|
+ } else if r == '}' {
|
|
|
+ block--
|
|
|
+ }
|
|
|
+ lexer.LogDebug("Skipping block comment: %d", block)
|
|
|
+ }
|
|
|
+ return err
|
|
|
+}
|
|
|
+
|
|
|
+func (lexer *Lexer) SkipComment() error {
|
|
|
+ r, err := lexer.Skip()
|
|
|
+ lexer.LogDebug("Skipping %c.", r)
|
|
|
+ if err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ first := true
|
|
|
+ for r, err = lexer.Skip(); r != '\n' && err == nil; r, err = lexer.Skip() {
|
|
|
+ lexer.LogDebug("Skipping loop %c.", r)
|
|
|
+ if first && r == '{' {
|
|
|
+ first = false
|
|
|
+ return lexer.SkipBlockComment()
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
return err
|
|
|
}
|
|
|
|
|
@@ -265,51 +335,62 @@ func isDoubleQuote(r rune) bool {
|
|
|
}
|
|
|
|
|
|
func (lexer *Lexer) handleEscapeHexChars(amount int) error {
|
|
|
- buffer := make([]byte, 0)
|
|
|
- r, err := lexer.Skip()
|
|
|
- for index := 0 ; err == nil && index < amount ; {
|
|
|
- if unicode.Is(unicode.ASCII_Hex_Digit, r) {
|
|
|
- buffer = append(buffer, byte(r))
|
|
|
- } else {
|
|
|
- return fmt.Errorf("Not a hexadecimal digit: %c", r)
|
|
|
- }
|
|
|
- index++
|
|
|
- if (index < amount) {
|
|
|
- r, err = lexer.Skip()
|
|
|
- }
|
|
|
- }
|
|
|
- if err != nil {
|
|
|
- return err
|
|
|
- }
|
|
|
- i, err := strconv.ParseInt(string(buffer), 16, 32)
|
|
|
- if err != nil {
|
|
|
- return err
|
|
|
- }
|
|
|
- lexer.appendRune(rune(i))
|
|
|
- _, err = lexer.Peek()
|
|
|
- return err
|
|
|
+ buffer := make([]byte, 0)
|
|
|
+ r, err := lexer.Skip()
|
|
|
+ for index := 0; err == nil && index < amount; {
|
|
|
+ if unicode.Is(unicode.ASCII_Hex_Digit, r) {
|
|
|
+ buffer = append(buffer, byte(r))
|
|
|
+ } else {
|
|
|
+ return fmt.Errorf("Not a hexadecimal digit: %c", r)
|
|
|
+ }
|
|
|
+ index++
|
|
|
+ if index < amount {
|
|
|
+ r, err = lexer.Skip()
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ i, err := strconv.ParseInt(string(buffer), 16, 32)
|
|
|
+ if err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ lexer.appendRune(rune(i))
|
|
|
+ _, err = lexer.Peek()
|
|
|
+ return err
|
|
|
}
|
|
|
|
|
|
-
|
|
|
func (lexer *Lexer) handleEscape() error {
|
|
|
r, err := lexer.Skip()
|
|
|
- if err != nil {
|
|
|
+ if err != nil {
|
|
|
return err
|
|
|
}
|
|
|
switch r {
|
|
|
- case 'a': lexer.appendRune('\a')
|
|
|
- case 'b': lexer.appendRune('\b')
|
|
|
- case 'e': lexer.appendRune('\033')
|
|
|
- case 'f': lexer.appendRune('\f')
|
|
|
- case 'n': lexer.appendRune('\n')
|
|
|
- case 'r': lexer.appendRune('\r')
|
|
|
- case 't': lexer.appendRune('\t')
|
|
|
- case '\\': lexer.appendRune('\\')
|
|
|
- case '"': lexer.appendRune('"')
|
|
|
- // case 'o': fallthrough // No octals, for now.
|
|
|
- case 'x': err = lexer.handleEscapeHexChars(2)
|
|
|
- case 'u': err = lexer.handleEscapeHexChars(4)
|
|
|
- case 'U': err = lexer.handleEscapeHexChars(6)
|
|
|
+ case 'a':
|
|
|
+ lexer.appendRune('\a')
|
|
|
+ case 'b':
|
|
|
+ lexer.appendRune('\b')
|
|
|
+ case 'e':
|
|
|
+ lexer.appendRune('\033')
|
|
|
+ case 'f':
|
|
|
+ lexer.appendRune('\f')
|
|
|
+ case 'n':
|
|
|
+ lexer.appendRune('\n')
|
|
|
+ case 'r':
|
|
|
+ lexer.appendRune('\r')
|
|
|
+ case 't':
|
|
|
+ lexer.appendRune('\t')
|
|
|
+ case '\\':
|
|
|
+ lexer.appendRune('\\')
|
|
|
+ case '"':
|
|
|
+ lexer.appendRune('"')
|
|
|
+ // case 'o': fallthrough // No octals, for now.
|
|
|
+ case 'x':
|
|
|
+ err = lexer.handleEscapeHexChars(2)
|
|
|
+ case 'u':
|
|
|
+ err = lexer.handleEscapeHexChars(4)
|
|
|
+ case 'U':
|
|
|
+ err = lexer.handleEscapeHexChars(6)
|
|
|
default:
|
|
|
return fmt.Errorf("Unknown escape sequence character %c: %d", r, r)
|
|
|
}
|
|
@@ -319,25 +400,25 @@ func (lexer *Lexer) handleEscape() error {
|
|
|
|
|
|
func (lexer *Lexer) LexString() Token {
|
|
|
var err error
|
|
|
- var r rune
|
|
|
+ var r rune
|
|
|
|
|
|
_, err = lexer.Skip() // Skip first "
|
|
|
if err != nil {
|
|
|
return lexer.handleError(err)
|
|
|
}
|
|
|
-
|
|
|
- r, err = lexer.Skip()
|
|
|
- for ; r != '"' && err == nil ; {
|
|
|
- if r == '\\' {
|
|
|
+
|
|
|
+ r, err = lexer.Skip()
|
|
|
+ for r != '"' && err == nil {
|
|
|
+ if r == '\\' {
|
|
|
err = lexer.handleEscape()
|
|
|
- if err != nil {
|
|
|
- return lexer.handleError(err)
|
|
|
- }
|
|
|
+ if err != nil {
|
|
|
+ return lexer.handleError(err)
|
|
|
+ }
|
|
|
} else {
|
|
|
- lexer.appendRune(r)
|
|
|
- // still inside the string
|
|
|
+ lexer.appendRune(r)
|
|
|
+ // still inside the string
|
|
|
}
|
|
|
- r, err = lexer.Skip()
|
|
|
+ r, err = lexer.Skip()
|
|
|
}
|
|
|
if err != nil {
|
|
|
return lexer.MakeErrorfToken("when parsing string: %s", err)
|
|
@@ -377,7 +458,7 @@ func (lexer *Lexer) LexLongString() Token {
|
|
|
|
|
|
func (lexer *Lexer) LexWord() Token {
|
|
|
var err error
|
|
|
- first := true
|
|
|
+ first := true
|
|
|
|
|
|
_, err = lexer.Next()
|
|
|
if err != nil {
|
|
@@ -385,26 +466,29 @@ func (lexer *Lexer) LexWord() Token {
|
|
|
}
|
|
|
|
|
|
_, err = lexer.NextWhile(func(r rune) bool {
|
|
|
- if first {
|
|
|
- first = false
|
|
|
- return unicode.IsLetter(r) || r == '_'
|
|
|
- } else {
|
|
|
- return unicode.IsLetter(r) || unicode.IsNumber(r) || r == '_'
|
|
|
- }
|
|
|
+ if first {
|
|
|
+ first = false
|
|
|
+ return unicode.IsLetter(r) || r == '_'
|
|
|
+ } else {
|
|
|
+ return unicode.IsLetter(r) || unicode.IsNumber(r) || r == '_'
|
|
|
+ }
|
|
|
})
|
|
|
-
|
|
|
- if err != nil {
|
|
|
+
|
|
|
+ if err != nil {
|
|
|
return lexer.handleError(err)
|
|
|
}
|
|
|
-
|
|
|
- sbuffer := string(lexer.buffer)
|
|
|
-
|
|
|
- // handle key words
|
|
|
- switch sbuffer {
|
|
|
- case "true" : return lexer.MakeBooleanToken(true)
|
|
|
- case "false": return lexer.MakeBooleanToken(false)
|
|
|
- default: return lexer.MakeToken(TokenKindWord)
|
|
|
- }
|
|
|
+
|
|
|
+ sbuffer := string(lexer.buffer)
|
|
|
+
|
|
|
+ // handle key words
|
|
|
+ switch sbuffer {
|
|
|
+ case "true":
|
|
|
+ return lexer.MakeBooleanToken(true)
|
|
|
+ case "false":
|
|
|
+ return lexer.MakeBooleanToken(false)
|
|
|
+ default:
|
|
|
+ return lexer.MakeToken(TokenKindWord)
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
func (lexer *Lexer) LexSymbol() Token {
|
|
@@ -416,34 +500,46 @@ func (lexer *Lexer) LexSymbol() Token {
|
|
|
}
|
|
|
|
|
|
_, err = lexer.NextWhile(func(r rune) bool {
|
|
|
- return !unicode.IsSpace(r)
|
|
|
+ return !unicode.IsSpace(r)
|
|
|
})
|
|
|
-
|
|
|
- if err != nil {
|
|
|
+
|
|
|
+ if err != nil {
|
|
|
return lexer.handleError(err)
|
|
|
}
|
|
|
|
|
|
return lexer.MakeToken(TokenKindSymbol)
|
|
|
}
|
|
|
|
|
|
-
|
|
|
-func (lexer *Lexer) lex() Token {
|
|
|
+func (lexer *Lexer) skipSpaceAndCommentAndPeek() (rune, error) {
|
|
|
r, err := lexer.Peek()
|
|
|
-
|
|
|
if err != nil {
|
|
|
- return lexer.handleError(err)
|
|
|
+ return r, err
|
|
|
}
|
|
|
-
|
|
|
- if isSpace(r) {
|
|
|
- err = lexer.SkipSpace()
|
|
|
+ for isSpace(r) || isComment(r) {
|
|
|
+ if isSpace(r) {
|
|
|
+ err = lexer.SkipSpace()
|
|
|
+ } else if isComment(r) {
|
|
|
+ err = lexer.SkipComment()
|
|
|
+ }
|
|
|
if err != nil {
|
|
|
- return lexer.handleError(err)
|
|
|
+ return r, err
|
|
|
}
|
|
|
- r, err = lexer.Peek()
|
|
|
+ lexer.LogDebug("Peeked again: >%c<", r)
|
|
|
+ r, err := lexer.Peek()
|
|
|
if err != nil {
|
|
|
- return lexer.handleError(err)
|
|
|
+ return r, err
|
|
|
}
|
|
|
}
|
|
|
+ return r, err
|
|
|
+}
|
|
|
+
|
|
|
+func (lexer *Lexer) lex() Token {
|
|
|
+ r, err := lexer.skipSpaceAndCommentAndPeek()
|
|
|
+ lexer.LogDebug(" After skip: >%c< >%v<\n", r, err)
|
|
|
+
|
|
|
+ if err != nil {
|
|
|
+ return lexer.handleError(err)
|
|
|
+ }
|
|
|
|
|
|
if unicode.IsDigit(r) || r == '-' {
|
|
|
return lexer.LexNumber()
|
|
@@ -461,8 +557,8 @@ func (lexer *Lexer) lex() Token {
|
|
|
if r == '`' {
|
|
|
return lexer.LexLongString()
|
|
|
}
|
|
|
-
|
|
|
- if r == ':' {
|
|
|
+
|
|
|
+ if r == ':' {
|
|
|
return lexer.LexSymbol()
|
|
|
}
|
|
|
|
|
@@ -514,16 +610,17 @@ func (lexer *Lexer) LexAll() []Token {
|
|
|
return res
|
|
|
}
|
|
|
|
|
|
-func NewLexer(scanner io.RuneScanner, filename string) Lexer {
|
|
|
- lexer := Lexer{}
|
|
|
+func NewLexer(scanner io.RuneScanner, filename string) *Lexer {
|
|
|
+ lexer := &Lexer{}
|
|
|
lexer.RuneScanner = scanner
|
|
|
lexer.Position.FileName = filename
|
|
|
lexer.Position.Column = 1
|
|
|
lexer.Position.Line = 1
|
|
|
+ lexer.LoggerWrapper = LoggerWrapper{nil}
|
|
|
return lexer
|
|
|
}
|
|
|
|
|
|
-func NewLexerFromInputString(input string) Lexer {
|
|
|
+func NewLexerFromString(input string) *Lexer {
|
|
|
reader := strings.NewReader(input)
|
|
|
return NewLexer(reader, "<input>")
|
|
|
}
|
|
@@ -533,7 +630,7 @@ func NewLexerFromFileName(filename string) (*Lexer, error) {
|
|
|
if err != nil {
|
|
|
bread := bufio.NewReader(read)
|
|
|
lex := NewLexer(bread, filename)
|
|
|
- return &lex, nil
|
|
|
+ return lex, nil
|
|
|
}
|
|
|
return nil, err
|
|
|
}
|