muesli/lexer.go

package muesli


import (
    _ "bytes"
    _ "errors"
    "fmt"
    _ "io"
    _ "reflect"
    _ "runtime"
    "strings"
    _ "unicode"
    "io"
    "os"
    "bufio"
    "unicode"
    // "gitlab.com/beoran/woe/graphviz"
    // _ "gitlab.com/beoran/woe/monolog"
)


type Position struct {
    FileName    string
    Line        int
    Column      int
}

type Lexer struct {
    Position
    Index       int
    Start       int
    io.RuneScanner
    buffer      []rune
    Current     rune
}



/** Token Kind. Uses a rune to easily handle single character tokens. */
type TokenKind rune

const (    
    TokenKindInteger    = TokenKind('i') 
    TokenKindFloat      = TokenKind('f')
    TokenKindString     = TokenKind('s')
    TokenKindBoolean    = TokenKind('b')
    TokenKindWord       = TokenKind('w')
    TokenKindType       = TokenKind('t')
    TokenKindGet        = TokenKind('$')
    TokenKindSet        = TokenKind('=')
    TokenKindOpenBlock  = TokenKind('{')
    TokenKindCloseBlock = TokenKind('}')
    TokenKindOpenList   = TokenKind('[')
    TokenKindCloseList  = TokenKind(']')
    TokenKindOpenParen  = TokenKind('(')
    TokenKindCloseParen = TokenKind(')')
    TokenKindError      = TokenKind('!')
    TokenKindEOX        = TokenKind('\n')
)



func NewToken(kind TokenKind, val Value, pos Position) Token {
    return Token{kind, val, pos}
}

func (lexer Lexer) MakeToken(kind TokenKind) Token {
    val := StringValue(string(lexer.buffer))
    return NewToken(kind, val, lexer.Position)
}

func (lexer * Lexer) Next() (rune, error) {
    r, _, err := lexer.RuneScanner.ReadRune()
    if err != nil {
        return 0, err
    }
    lexer.Current = r
    lexer.buffer = append(lexer.buffer, r)
    lexer.Index++
    lexer.Position.Column++
    if r == '\n' {
        lexer.Position.Column = 1
        lexer.Position.Line++
    }
    return lexer.buffer[len(lexer.buffer) - 1], nil
}

func (lexer * Lexer) Previous() error {
    err := lexer.RuneScanner.UnreadRune()
    if err != nil {
        return err
    }    
    
    lexer.Index--
    lexer.Position.Column--

    if (len(lexer.buffer) > 0)  { 
        r := lexer.buffer[len(lexer.buffer) - 1];
        lexer.buffer = lexer.buffer[0: len(lexer.buffer) - 1];
    
        if r == '\n' {
            lexer.Position.Column = 1
            lexer.Position.Line++
        }
        
        lexer.Current = r
    }
    return nil
}



    
func (lexer * Lexer) SkipSpace() (error) {
    var r rune
    var err error
    r = lexer.Current
    
    for unicode.IsSpace(r) { 
      r, err = lexer.Next()
      if err != nil { 
        return err
      }
    }
    lexer.Previous()
    return nil 
}


func (lexer * Lexer) LexNumber() (Token, error) { 
  isFloat := false
  var r rune
  var err error
  
  r = lexer.Current  
    
  for unicode.IsDigit(r) || r == '.'  {
    if r == '.' {
      if isFloat { // double . in floating point is an error
        tok := lexer.MakeToken(TokenKindError) 
        err = fmt.Errorf("Double period . in floating point constant.")
        return tok, err
      } else {
        isFloat = true
      }
    }
    r, err = lexer.Next()
    if err != nil {
      return lexer.MakeToken(TokenKindError), err
    }
  }
  
  lexer.Previous()
  if isFloat {
    return lexer.MakeToken(TokenKindFloat), nil
  } else {
    return lexer.MakeToken(TokenKindInteger), nil  
  }
  
}

func (lexer * Lexer) LexString() (Token, error) { 
  inEscape := false
  var r rune
  var err error
  
  r, err = lexer.Next()
  if err != nil {
    return lexer.MakeToken(TokenKindError), err
  }
 
  
  for  r != '"'  || inEscape {
    if r == '\\' {
      // TODO escape parsing, now just a single character after it
      if inEscape { // double backslash
      } else {
        inEscape = true
      }
    } else {
        inEscape = false
    }
    r, err = lexer.Next()
    if err != nil {
      return lexer.MakeToken(TokenKindError), err
    }
  }
  
  return lexer.MakeToken(TokenKindString), nil
}

func (lexer * Lexer) LexLongString() (Token, error) { 
  var r rune
  var err error
  
  r, err = lexer.Next()
  if err != nil {
    return lexer.MakeToken(TokenKindError), err
  }
  
  for  r != '`' {
    r, err = lexer.Next()
    if err != nil {
      return lexer.MakeToken(TokenKindError), err
    }
  }
  
  return lexer.MakeToken(TokenKindString), nil
}

func (lexer * Lexer) LexWord() (Token, error) { 
  var r rune
  var err error
  
  r, err = lexer.Next()
  if err != nil {
    return lexer.MakeToken(TokenKindError), err
  }
  
  for  r != '`' {
    r, err = lexer.Next()
    if err != nil {
      return lexer.MakeToken(TokenKindError), err
    }
  }
  
  return lexer.MakeToken(TokenKindString), nil
}

func (lexer * Lexer) Lex() (Token, error) {
    r, err := lexer.Next()
    if err != nil {
        return lexer.MakeToken(TokenKindError), err
    }
    
    if unicode.IsSpace(r) {
        lexer.SkipSpace()
    }

    
    if unicode.IsDigit(r) {
        return lexer.LexNumber()
    }
    
    if r == '\n' || r == '.' {
       return lexer.MakeToken(TokenKindEOX), nil 
    }
        
    if r == '"' {
        return lexer.LexString()
    }
    
    if r == '`' {
        return lexer.LexLongString()
    }
    
        
    switch (TokenKind(r)) {     
        case TokenKindGet       : fallthrough
        case TokenKindSet       : fallthrough
        case TokenKindOpenBlock : fallthrough
        case TokenKindCloseBlock: fallthrough
        case TokenKindOpenList  : fallthrough
        case TokenKindCloseList : fallthrough
        case TokenKindOpenParen : fallthrough
        case TokenKindCloseParen: 
            return lexer.MakeToken(TokenKind(r)), nil
        default:
    }
    
    if unicode.IsLetter(r) {
        return lexer.LexWord()
    }

    return lexer.MakeToken(TokenKindError), fmt.Errorf("Unknown character")
}

func NewLexer(scanner io.RuneScanner, filename string) Lexer {
    lexer := Lexer{}
    lexer.RuneScanner       = scanner
    lexer.Position.FileName = filename
    lexer.Position.Column   = 1
    lexer.Position.Line     = 1
    return lexer
}

func NewLexerFromInputString(input string) Lexer {
    reader := strings.NewReader(input)
    return NewLexer(reader, "<input>") 
}


func NewLexerFromFileName(filename string) (*Lexer, error) {
    read, err   := os.Open(filename)
    if err != nil {
        bread   := bufio.NewReader(read)
        lex     := NewLexer(bread, filename)
        return &lex, nil
    }
    return nil , err
}