package muesli import ( _ "bytes" _ "errors" "fmt" _ "io" _ "reflect" _ "runtime" "strings" _ "unicode" "io" "os" "bufio" "unicode" // "gitlab.com/beoran/woe/graphviz" // _ "gitlab.com/beoran/woe/monolog" ) type Position struct { FileName string Line int Column int } type Lexer struct { Position Index int Start int io.RuneScanner buffer []rune Current rune } /** Token Kind. Uses a rune to easily handle single character tokens. */ type TokenKind rune const ( TokenKindInteger = TokenKind('i') TokenKindFloat = TokenKind('f') TokenKindString = TokenKind('s') TokenKindBoolean = TokenKind('b') TokenKindWord = TokenKind('w') TokenKindType = TokenKind('t') TokenKindGet = TokenKind('$') TokenKindSet = TokenKind('=') TokenKindOpenBlock = TokenKind('{') TokenKindCloseBlock = TokenKind('}') TokenKindOpenList = TokenKind('[') TokenKindCloseList = TokenKind(']') TokenKindOpenParen = TokenKind('(') TokenKindCloseParen = TokenKind(')') TokenKindError = TokenKind('!') TokenKindEOX = TokenKind('\n') ) func NewToken(kind TokenKind, val Value, pos Position) Token { return Token{kind, val, pos} } func (lexer Lexer) MakeToken(kind TokenKind) Token { val := StringValue(string(lexer.buffer)) return NewToken(kind, val, lexer.Position) } func (lexer * Lexer) Next() (rune, error) { r, _, err := lexer.RuneScanner.ReadRune() if err != nil { return 0, err } lexer.Current = r lexer.buffer = append(lexer.buffer, r) lexer.Index++ lexer.Position.Column++ if r == '\n' { lexer.Position.Column = 1 lexer.Position.Line++ } return lexer.buffer[len(lexer.buffer) - 1], nil } func (lexer * Lexer) Previous() error { err := lexer.RuneScanner.UnreadRune() if err != nil { return err } lexer.Index-- lexer.Position.Column-- if (len(lexer.buffer) > 0) { r := lexer.buffer[len(lexer.buffer) - 1]; lexer.buffer = lexer.buffer[0: len(lexer.buffer) - 1]; if r == '\n' { lexer.Position.Column = 1 lexer.Position.Line++ } lexer.Current = r } return nil } func (lexer * Lexer) SkipSpace() (error) { var r rune var err error r = lexer.Current for unicode.IsSpace(r) { r, err = lexer.Next() if err != nil { return err } } lexer.Previous() return nil } func (lexer * Lexer) LexNumber() (Token, error) { isFloat := false var r rune var err error r = lexer.Current for unicode.IsDigit(r) || r == '.' { if r == '.' { if isFloat { // double . in floating point is an error tok := lexer.MakeToken(TokenKindError) err = fmt.Errorf("Double period . in floating point constant.") return tok, err } else { isFloat = true } } r, err = lexer.Next() if err != nil { return lexer.MakeToken(TokenKindError), err } } lexer.Previous() if isFloat { return lexer.MakeToken(TokenKindFloat), nil } else { return lexer.MakeToken(TokenKindInteger), nil } } func (lexer * Lexer) LexString() (Token, error) { inEscape := false var r rune var err error r, err = lexer.Next() if err != nil { return lexer.MakeToken(TokenKindError), err } for r != '"' || inEscape { if r == '\\' { // TODO escape parsing, now just a single character after it if inEscape { // double backslash } else { inEscape = true } } else { inEscape = false } r, err = lexer.Next() if err != nil { return lexer.MakeToken(TokenKindError), err } } return lexer.MakeToken(TokenKindString), nil } func (lexer * Lexer) LexLongString() (Token, error) { var r rune var err error r, err = lexer.Next() if err != nil { return lexer.MakeToken(TokenKindError), err } for r != '`' { r, err = lexer.Next() if err != nil { return lexer.MakeToken(TokenKindError), err } } return lexer.MakeToken(TokenKindString), nil } func (lexer * Lexer) LexWord() (Token, error) { var r rune var err error r, err = lexer.Next() if err != nil { return lexer.MakeToken(TokenKindError), err } for r != '`' { r, err = lexer.Next() if err != nil { return lexer.MakeToken(TokenKindError), err } } return lexer.MakeToken(TokenKindString), nil } func (lexer * Lexer) Lex() (Token, error) { r, err := lexer.Next() if err != nil { return lexer.MakeToken(TokenKindError), err } if unicode.IsSpace(r) { lexer.SkipSpace() } if unicode.IsDigit(r) { return lexer.LexNumber() } if r == '\n' || r == '.' { return lexer.MakeToken(TokenKindEOX), nil } if r == '"' { return lexer.LexString() } if r == '`' { return lexer.LexLongString() } switch (TokenKind(r)) { case TokenKindGet : fallthrough case TokenKindSet : fallthrough case TokenKindOpenBlock : fallthrough case TokenKindCloseBlock: fallthrough case TokenKindOpenList : fallthrough case TokenKindCloseList : fallthrough case TokenKindOpenParen : fallthrough case TokenKindCloseParen: return lexer.MakeToken(TokenKind(r)), nil default: } if unicode.IsLetter(r) { return lexer.LexWord() } return lexer.MakeToken(TokenKindError), fmt.Errorf("Unknown character") } func NewLexer(scanner io.RuneScanner, filename string) Lexer { lexer := Lexer{} lexer.RuneScanner = scanner lexer.Position.FileName = filename lexer.Position.Column = 1 lexer.Position.Line = 1 return lexer } func NewLexerFromInputString(input string) Lexer { reader := strings.NewReader(input) return NewLexer(reader, "") } func NewLexerFromFileName(filename string) (*Lexer, error) { read, err := os.Open(filename) if err != nil { bread := bufio.NewReader(read) lex := NewLexer(bread, filename) return &lex, nil } return nil , err }