Now working on parser.

ast.go

@@ -0,0 +1,140 @@
+// Abstract Syntax tree for the MUESLI interpreter
+package muesli
+
+import "fmt"
+
+
+/* AST node kind */
+type Ast struct {
+	Parent * Ast
+    Child  * Ast
+    Before * Ast
+    After  * Ast
+	AstKind
+	*Token
+}
+
+
+func NewAst(kind AstKind, parent * Ast, token * Token) *Ast {
+    child := &Ast{parent, nil, nil, nil,  kind, token}
+    return child
+}
+
+func (ast * Ast) LastSibling() * Ast {
+    res := ast
+    for res != nil && res.After != nil {
+        res = res.After
+    }
+    return res
+}
+
+func (ast * Ast) LastChild() * Ast {
+    return ast.Child.LastSibling()
+}
+
+/* Detaches, I.E removes this node and all it's children from the parent tree. */
+func (ast * Ast) Remove() * Ast {
+    parent := ast.Parent
+    before := ast.Before
+    after  := ast.After
+    if before != nil {
+        before.After = after
+    }
+    if after != nil {
+        after.Before = before
+    }
+    if parent != nil {
+        /* Special case if ast is the first child of it's parent. */
+        if ast == parent.Child {
+            parent.Child = after
+        }
+    }
+    ast.Parent = nil
+    return ast
+}
+
+func (ast * Ast) InsertSibling(sibling * Ast) * Ast {
+    after := ast.After
+    ast.After = sibling
+    sibling.Before = ast
+    sibling.After  = after
+    if after != nil {
+        after.Before = sibling
+    }
+    sibling.Parent = ast.Parent
+    return sibling
+}
+
+func (ast * Ast) AppendSibling(sibling * Ast) * Ast {
+    return ast.LastSibling().InsertSibling(sibling)
+}
+
+func (ast * Ast) AppendChild(child * Ast) * Ast {
+    child.Parent = ast
+    if ast.Child == nil {
+        ast.Child = child
+    } else {
+        ast.Child.AppendSibling(child)
+    }
+    return child
+}
+
+func (ast * Ast) NewSibling(kind AstKind, token * Token) * Ast {
+    sibling := NewAst(kind, ast.Parent, token)
+    return ast.AppendSibling(sibling)
+}
+
+func (ast * Ast) NewChild(kind AstKind, token * Token) * Ast {
+    sibling := NewAst(kind, ast.Parent, token)
+    return ast.AppendChild(sibling)
+}
+
+func (ast * Ast) Walk(walker func(node * Ast) * Ast) * Ast {
+    if found := walker(ast); found != nil {
+        return found
+    }
+    if ast.Child != nil {
+        if found := ast.Child.Walk(walker); found != nil {
+            return found
+        }
+    }
+    if ast.After != nil {
+        if found := ast.After.Walk(walker); found != nil {
+            return found
+        }
+    }
+    return nil
+}
+
+func (ast * Ast) String() string {
+    return fmt.Sprintf("Ast %d %v", ast.AstKind, ast.Token)
+}
+
+func (ast * Ast) Display() {
+    ast.Walk(func(node * Ast) * Ast {
+        fmt.Printf("Tree: %s\n", node)
+        return nil
+    })
+}
+
+func (ast * Ast) Depth() int {
+    var depth int = 0;
+    parent := ast.Parent
+    for parent != nil {
+        depth ++;
+        parent = parent.Parent
+    }
+    return depth
+}
+
+func (ast * Ast) CountChildren() int {
+    var count int = 0;
+    child := ast.Child
+    for child != nil {
+        count ++;
+        child = child.After
+    }
+    return count;
+}
+
+

lexer.go

@@ -1,401 +1,539 @@
 package muesli
 
-
 import (
-    _ "bytes"
-    _ "errors"
-    "fmt"
-    _ "io"
-    _ "reflect"
-    _ "runtime"
-    "strings"
-    _ "unicode"
-    "io"
-    "os"
-    "bufio"
-    "unicode"
-    // "gitlab.com/beoran/woe/graphviz"
-    // _ "gitlab.com/beoran/woe/monolog"
+	"bufio"
+	_ "bytes"
+	_ "errors"
+	"fmt"
+	"io"
+	_ "io"
+	"os"
+	_ "reflect"
+	_ "runtime"
+	"strings"
+    "strconv"
+	"unicode"
+	_ "unicode"
+	// "gitlab.com/beoran/woe/graphviz"
+	// _ "gitlab.com/beoran/woe/monolog"
 )
 
-
-
-
 /* A Lexer splits scanned input into tokens.
  */
 type Lexer struct {
-    Position
-    Index       int
-    Start       int
-    io.RuneScanner
-    buffer      []rune
-    Current     rune
+	Position
+	Index int
+	Start int
+	io.RuneScanner
+	buffer  []rune
+	Current rune
+}
+
+func (lexer *Lexer) ClearBuffer() {
+	lexer.buffer = make([]rune, 0)
 }
 
+func (lexer *Lexer) MakeIntegerToken() Token {
+    var sbuffer = string(lexer.buffer)
+    i, err := strconv.ParseInt(sbuffer, 0, 64)
+    if err == nil { 
+        lexer.ClearBuffer()
+        return NewToken(TokenKindInteger, IntValue(i), lexer.Position)
+    } else {
+        lexer.ClearBuffer()
+        return lexer.MakeErrorToken(err);
+    }
+}
 
-func (lexer * Lexer) ClearBuffer() {
-  lexer.buffer = make([]rune, 0)
+func (lexer *Lexer) MakeFloatToken() Token {
+    var sbuffer = string(lexer.buffer)
+    f, err := strconv.ParseFloat(sbuffer, 64)
+    if err == nil { 
+        lexer.ClearBuffer()
+        return NewToken(TokenKindFloat, FloatValue(f), lexer.Position)
+    } else {
+        lexer.ClearBuffer()
+        return lexer.MakeErrorToken(err);
+    }
 }
 
-func (lexer * Lexer) MakeToken(kind TokenKind) Token {
-    val := StringValue(string(lexer.buffer))
+func (lexer *Lexer) MakeBooleanToken(b bool) Token {
     lexer.ClearBuffer()
-    return NewToken(kind, val, lexer.Position)
+    return NewToken(TokenKindBoolean, BoolValue(b), lexer.Position)
 }
 
-func (lexer Lexer) MakeErrorToken(err error) Token {
-    return NewToken(TokenKindError, err.Error(), lexer.Position)
+
+func (lexer *Lexer) MakeStringValueToken(kind TokenKind) Token {
+    var sbuffer = string(lexer.buffer)
+    return NewToken(kind, StringValue(sbuffer), lexer.Position)
+}
+
+func (lexer *Lexer) MakeToken(kind TokenKind) Token {    
+    switch (kind)   { 
+        case TokenKindInteger   : return lexer.MakeIntegerToken() 
+        case TokenKindFloat     : return lexer.MakeFloatToken()
+        case TokenKindString    : fallthrough 
+        case TokenKindSymbol    : fallthrough 
+        case TokenKindType      : fallthrough
+        case TokenKindError     : fallthrough
+        case TokenKindWord      : return lexer.MakeStringValueToken(kind)
+        case TokenKindBoolean   : fallthrough
+        case TokenKindGet       : fallthrough 
+        case TokenKindSet       : fallthrough 
+        case TokenKindOpenBlock : fallthrough 
+        case TokenKindCloseBlock: fallthrough 
+        case TokenKindOpenList  : fallthrough 
+        case TokenKindCloseList : fallthrough 
+        case TokenKindOpenParen : fallthrough 
+        case TokenKindCloseParen: fallthrough 
+        case TokenKindEOX       : fallthrough
+        case TokenKindEOF       : 
+            val := StringValue(string(lexer.buffer))
+            lexer.ClearBuffer()
+            return NewToken(kind, val, lexer.Position)
+        default :
+            return lexer.MakeErrorfToken("Internal error on token type %s", kind)
+    }
 }
 
-func (lexer Lexer) MakeErrorfToken(format string, va ... interface{}) Token {
-    err := fmt.Errorf(format, va...)
-    return lexer.MakeErrorToken(err)
+func (lexer Lexer) MakeErrorToken(err error) Token {
+	return NewToken(TokenKindError, err.Error(), lexer.Position)
 }
 
+func (lexer Lexer) MakeErrorfToken(format string, va ...interface{}) Token {
+	err := fmt.Errorf(format, va...)
+	return lexer.MakeErrorToken(err)
+}
 
 func (lexer Lexer) MakeEOFToken() Token {
-    return NewToken(TokenKindEOF, "", lexer.Position)
+	return NewToken(TokenKindEOF, "", lexer.Position)
 }
 
+func (lexer *Lexer) Peek() (rune, error) {
+	r, _, err := lexer.RuneScanner.ReadRune()
+	err2 := lexer.RuneScanner.UnreadRune()
 
-func (lexer * Lexer) Peek() (rune, error) {
-    r, _, err := lexer.RuneScanner.ReadRune()
-    err2 := lexer.RuneScanner.UnreadRune()
-    
-    if err == nil {
-        err = err2
-    }
-    return r, err
+	if err == nil {
+		err = err2
+	}
+	return r, err
 }
 
 /* Advances the lexer's position based on the rune r read. */
-func (lexer * Lexer) advance(r rune) {   
-    lexer.Current = r
-    lexer.Index++
-    lexer.Position.Column++
-    if r == '\n' {
-        lexer.Position.Column = 1
-        lexer.Position.Line++
-    }
+func (lexer *Lexer) advance(r rune) {
+	lexer.Current = r
+	lexer.Index++
+	lexer.Position.Column++
+	if r == '\n' {
+		lexer.Position.Column = 1
+		lexer.Position.Line++
+	}
 }
 
-/* Actually reads the next rune from the lexer's input source and stores
- * them in the lexer's token buffer. */
-func (lexer * Lexer) Next()  (rune, error) {  
-    r, _, err := lexer.RuneScanner.ReadRune()
-    if err != nil {
-        return 0, err
-    }
-    lexer.advance(r)
-    lexer.buffer = append(lexer.buffer, r)
-    return r, nil
+/* Append a rune to the lexer's buffer. */
+func (lexer *Lexer) appendRune(r rune) {
+	lexer.buffer = append(lexer.buffer, r)
 }
 
-/* Advances the lexer's input buffer but does not store the read runes*/
-func (lexer * Lexer) Skip() (rune, error) {  
-    r, _, err := lexer.RuneScanner.ReadRune()
-    if err != nil {
-        return 0, err
-    }
-    lexer.advance(r)
-    return r, nil
+/* Advances the lexer's input buffer but does not store the rune read,
+ * but just returns it. */
+func (lexer *Lexer) Skip() (rune, error) {
+	r, _, err := lexer.RuneScanner.ReadRune()
+	if err != nil {
+		return 0, err
+	}
+	lexer.advance(r)
+	return r, nil
 }
 
-
-func (lexer * Lexer) NextIf(predicate func(rune) bool) (bool, error) {
-  r, err := lexer.Peek()
-  if err != nil {
-    return false, err
-  }
-  if (predicate(r)) {
-    r, err = lexer.Next()
-    if err != nil {
-      return true, err
-    }
-    return true, nil
-  }
-  return false, nil
-}
-
-func (lexer * Lexer) SkipIf(predicate func(rune) bool) (bool, error) {
-  r, err := lexer.Peek()
-  if err != nil {
-    return false, err
-  }
-  if (predicate(r)) {
-    r, err = lexer.Skip()
-    if err != nil {
-      return true, err
-    }
-    return true, nil
-  }
-  return false, nil
+/* Actually reads the next rune from the lexer's input source and stores
+ * them in the lexer's token buffer.
+ * Shorthand for r, err := lexer.Skip() ; lexer.appendRune(r)  */
+func (lexer *Lexer) Next() (rune, error) {
+	r, err := lexer.Skip()
+	if err == nil {
+		lexer.appendRune(r)
+	}
+	return r, nil
 }
 
-
-func (lexer * Lexer) NextWhile(predicate func(rune) bool) (bool, error) {
-    result := true
-    ok, err := lexer.NextIf(predicate)
-    result = result || ok
-    for ; ok && (err == nil) ; ok, err = lexer.NextIf(predicate) {
-        result = result || ok
-    }
-    return result, err
+func (lexer *Lexer) NextIf(predicate func(rune) bool) (bool, error) {
+	r, err := lexer.Peek()
+	if err != nil {
+		return false, err
+	}
+	if predicate(r) {
+		r, err = lexer.Next()
+		if err != nil {
+			return true, err
+		}
+		return true, nil
+	}
+	return false, nil
 }
 
+func (lexer *Lexer) SkipIf(predicate func(rune) bool) (bool, error) {
+	r, err := lexer.Peek()
+	if err != nil {
+		return false, err
+	}
+	if predicate(r) {
+		r, err = lexer.Skip()
+		if err != nil {
+			return true, err
+		}
+		return true, nil
+	}
+	return false, nil
+}
 
-func (lexer * Lexer) SkipWhile(predicate func(rune) bool) (bool, error) {
-    result := true
-    ok, err := lexer.SkipIf(predicate)
-    result = result || ok
-    for ; ok && (err == nil) ; ok, err = lexer.SkipIf(predicate) {
-        result = result || ok
-    }
-    return result, err
+func (lexer *Lexer) NextWhile(predicate func(rune) bool) (bool, error) {
+	result := true
+	ok, err := lexer.NextIf(predicate)
+	result = result || ok
+	for ; ok && (err == nil); ok, err = lexer.NextIf(predicate) {
+		result = result || ok
+	}
+	return result, err
 }
 
+func (lexer *Lexer) SkipWhile(predicate func(rune) bool) (bool, error) {
+	result := true
+	ok, err := lexer.SkipIf(predicate)
+	result = result || ok
+	for ; ok && (err == nil); ok, err = lexer.SkipIf(predicate) {
+		result = result || ok
+	}
+	return result, err
+}
 
 func isSpace(r rune) bool {
-  return r == ' ' || r == '\t'
+	return r == ' ' || r == '\t'
 }
 
+func (lexer *Lexer) SkipSpace() error {
+	_, err := lexer.SkipWhile(isSpace)
+	return err
+}
 
-func (lexer * Lexer) SkipSpace() (error) {
-    _, err := lexer.SkipWhile(isSpace)
-    return err
- }
-
-/* Handles errors including EOF by either returning an error token or an 
+/* Handles errors including EOF by either returning an error token or an
  * EOF token.
  */
-func (lexer * Lexer) handleError(err error) Token {
-    if err == io.EOF {
-        return lexer.MakeEOFToken()
-    } else {
-        return lexer.MakeErrorToken(err)
-    }
+func (lexer *Lexer) handleError(err error) Token {
+	if err == io.EOF {
+		return lexer.MakeEOFToken()
+	} else {
+		return lexer.MakeErrorToken(err)
+	}
 }
 
-
-func (lexer * Lexer) LexNumber() Token { 
-  isFloat := false
-  
-  // skip any first -
-  _, err := lexer.NextIf(func (r rune) bool {
-    return r == '-'
-  })
-   
-  _, err = lexer.NextWhile(func (r rune) bool {
-    if unicode.IsDigit(r) {
-        return true
-    } else if r == '.' {
-      if isFloat { 
-          return false // double point in floating point
-      } else {
-          isFloat = true
-          return true
-      }
-    } else {
-        return false
-    }
-  })
-  
-  
-  if err != nil {
-      return lexer.MakeErrorfToken("when parsing number: %s", err)
-  }
-  if isFloat {
-      return lexer.MakeToken(TokenKindFloat)
-  } else {
-      return lexer.MakeToken(TokenKindInteger)  
-  }  
+func (lexer *Lexer) LexNumber() Token {
+	isFloat := false
+
+	// skip any first -
+	_, err := lexer.NextIf(func(r rune) bool {
+		return r == '-'
+	})
+
+	_, err = lexer.NextWhile(func(r rune) bool {
+		if unicode.IsDigit(r) {
+			return true
+		} else if r == '.' {
+			if isFloat {
+				return false // double point in floating point
+			} else {
+				isFloat = true
+				return true
+			}
+		} else {
+			return false
+		}
+	})
+
+	if err != nil {
+		return lexer.MakeErrorfToken("when parsing number: %s", err)
+	}
+	if isFloat {
+		return lexer.MakeToken(TokenKindFloat)
+	} else {
+		return lexer.MakeToken(TokenKindInteger)
+	}
 }
 
 func isDoubleQuote(r rune) bool {
-  return r == '"'
+	return r == '"'
 }
 
-func (lexer * Lexer) LexString() Token { 
-  inEscape := false
-  var err error
-  
-  _, err = lexer.Skip() // Skip first " 
-  if err != nil {
-      return lexer.handleError(err)
-  }
-    
-  _, err = lexer.NextWhile(func (r rune) bool {
-      if r == '"' && !inEscape {
-        return false
-      }
-      if r == '\\' {
-        // TODO escape parsing, now just a single character after it      
-          if inEscape { // double backslash
-            inEscape = false
-          } else {
-            inEscape = true
-          }
-      } else {
-            inEscape = false
-      }
-      return true // still inside the string
-  })
-  if err != nil {
-        return lexer.MakeErrorfToken("when parsing string: %s", err)
-  }
-  
-  _, err = lexer.Skip() // skip last "
-  if err != nil {
-      return lexer.handleError(err)
-  }
-  
-  return lexer.MakeToken(TokenKindString)
-}
-
-func (lexer * Lexer) LexLongString() Token { 
-  var err error
-  
-  _, err = lexer.Skip()
-  if err != nil {
-      return lexer.handleError(err)
-  }
-  
-  _, err = lexer.NextWhile(func (r rune) bool {
-      return r != '`'
-  })
-
-  if err != nil {
-        return lexer.MakeErrorfToken("when parsing long string: %s", err)
-  }
-  
-  _, err = lexer.Skip()
-  if err != nil {
-      return lexer.handleError(err)
-  }
-  
-  return lexer.MakeToken(TokenKindString)
-}
-
-func (lexer * Lexer) LexWord() Token { 
-  var err error
-  
-  _, err = lexer.Next()
-  if err != nil {
-    return lexer.handleError(err)
-  }
-  
-  lexer.NextWhile(func(r rune) bool {
-      return unicode.IsLetter(r)
-  })
-  
-  return lexer.MakeToken(TokenKindWord)
-}
-
-func (lexer * Lexer) lex() Token {
-    r, err := lexer.Peek()
- 
+func (lexer *Lexer) handleEscapeHexChars(amount int) error {
+    buffer := make([]byte, 0)
+    r, err := lexer.Skip()
+    for index  := 0 ; err == nil && index < amount ;  {
+        if unicode.Is(unicode.ASCII_Hex_Digit, r) {
+            buffer = append(buffer, byte(r))
+        } else {
+            return fmt.Errorf("Not a hexadecimal digit: %c", r)
+        }
+        index++
+        if (index < amount) { 
+            r, err = lexer.Skip()
+        }
+    }
     if err != nil {
-      return lexer.handleError(err)
+        return err
     }
-    
-    if isSpace(r) { 
-        err = lexer.SkipSpace()
-        if err != nil {
-          return lexer.handleError(err)
-        } 
-        r, err = lexer.Peek()
-        if err != nil {
-          return lexer.handleError(err)
-        }
+    i, err := strconv.ParseInt(string(buffer), 16, 32)
+    if err != nil {
+        return err
     }
+    lexer.appendRune(rune(i))
+    _, err = lexer.Peek()
+    return err
+}
+
+
+func (lexer *Lexer) handleEscape() error {
+	r, err := lexer.Skip()
+    if err != nil {
+		return err
+	}
+	switch r {
+        case 'a':   lexer.appendRune('\a')
+        case 'b':   lexer.appendRune('\b')
+        case 'e':   lexer.appendRune('\033')
+        case 'f':   lexer.appendRune('\f')
+        case 'n':   lexer.appendRune('\n')
+        case 'r':   lexer.appendRune('\r')
+        case 't':   lexer.appendRune('\t')
+        case '\\':  lexer.appendRune('\\')
+        case '"':   lexer.appendRune('"')
+        // case 'o':   fallthrough   // No octals, for now.
+        case 'x':   err = lexer.handleEscapeHexChars(2) 
+        case 'u':   err = lexer.handleEscapeHexChars(4)
+        case 'U':   err = lexer.handleEscapeHexChars(6)
+	default:
+		return fmt.Errorf("Unknown escape sequence character %c: %d", r, r)
+	}
+
+	return err
+}
+
+func (lexer *Lexer) LexString() Token {
+	var err error
+    var r rune
+
+	_, err = lexer.Skip() // Skip first "
+	if err != nil {
+		return lexer.handleError(err)
+	}
     
+    r, err = lexer.Skip() 
+	for  ; r != '"' && err == nil ; { 
+        if r == '\\' {
+			err = lexer.handleEscape()
+            if err != nil {
+                return lexer.handleError(err)
+            }
+		} else {
+            lexer.appendRune(r)
+            // still inside the string
+		}
+        r, err = lexer.Skip()
+	}
+	if err != nil {
+		return lexer.MakeErrorfToken("when parsing string: %s", err)
+	}
+
+	_, err = lexer.Skip() // skip last "
+	if err != nil {
+		return lexer.handleError(err)
+	}
+
+	return lexer.MakeToken(TokenKindString)
+}
 
-    if unicode.IsDigit(r) || r == '-' {
-        return lexer.LexNumber()
-    }
+func (lexer *Lexer) LexLongString() Token {
+	var err error
+
+	_, err = lexer.Skip()
+	if err != nil {
+		return lexer.handleError(err)
+	}
+
+	_, err = lexer.NextWhile(func(r rune) bool {
+		return r != '`'
+	})
+
+	if err != nil {
+		return lexer.MakeErrorfToken("when parsing long string: %s", err)
+	}
+
+	_, err = lexer.Skip()
+	if err != nil {
+		return lexer.handleError(err)
+	}
+
+	return lexer.MakeToken(TokenKindString)
+}
+
+func (lexer *Lexer) LexWord() Token {
+	var err error
+    first := true
+
+	_, err = lexer.Next()
+	if err != nil {
+		return lexer.handleError(err)
+	}
+
+	_, err = lexer.NextWhile(func(r rune) bool {
+        if first {
+            first = false
+            return unicode.IsLetter(r) || r == '_'
+        } else {
+            return unicode.IsLetter(r) || unicode.IsNumber(r) || r == '_'
+        }
+	})
     
-    if r == '\n' || r == '.' {
-       lexer.Next()
-       return lexer.MakeToken(TokenKindEOX)
-    }
-        
-    if r == '"' {
-        return lexer.LexString()
-    }
+    if err != nil {
+		return lexer.handleError(err)
+	}
     
-    if r == '`' {
-        return lexer.LexLongString()
-    }
+    sbuffer := string(lexer.buffer)
     
-        
-    switch (TokenKind(r)) {     
-        case TokenKindGet       : fallthrough
-        case TokenKindSet       : fallthrough
-        case TokenKindOpenBlock : fallthrough
-        case TokenKindCloseBlock: fallthrough
-        case TokenKindOpenList  : fallthrough
-        case TokenKindCloseList : fallthrough
-        case TokenKindOpenParen : fallthrough
-        case TokenKindCloseParen: 
-            lexer.Next()
-            return lexer.MakeToken(TokenKind(r))
-        default:
+    // handle key words    
+    switch sbuffer {
+        case "true" :  return lexer.MakeBooleanToken(true)
+        case "false":  return lexer.MakeBooleanToken(false)
+        default: 	   return lexer.MakeToken(TokenKindWord)
     }
+}
+
+func (lexer *Lexer) LexSymbol() Token {
+	var err error
+
+	_, err = lexer.Skip()
+	if err != nil {
+		return lexer.handleError(err)
+	}
+
+	_, err = lexer.NextWhile(func(r rune) bool {
+        return !unicode.IsSpace(r)
+	})
     
-    if unicode.IsLetter(r) {
-        return lexer.LexWord()
-    }
+    if err != nil {
+		return lexer.handleError(err)
+	}
 
-    return lexer.MakeErrorfToken("Unknown character: %c", r)
+	return lexer.MakeToken(TokenKindSymbol)
 }
 
-func (lexer * Lexer) Lex() Token {
-  res := lexer.lex()
-  lexer.ClearBuffer() // ensure buffer is cleared after lexing, always.
-  return res
+
+func (lexer *Lexer) lex() Token {
+	r, err := lexer.Peek()
+
+	if err != nil {
+		return lexer.handleError(err)
+	}
+
+	if isSpace(r) {
+		err = lexer.SkipSpace()
+		if err != nil {
+			return lexer.handleError(err)
+		}
+		r, err = lexer.Peek()
+		if err != nil {
+			return lexer.handleError(err)
+		}
+	}
+
+	if unicode.IsDigit(r) || r == '-' {
+		return lexer.LexNumber()
+	}
+
+	if r == '\n' || r == '.' {
+		lexer.Next()
+		return lexer.MakeToken(TokenKindEOX)
+	}
+
+	if r == '"' {
+		return lexer.LexString()
+	}
+
+	if r == '`' {
+		return lexer.LexLongString()
+	}
+    
+    if r == ':' {
+		return lexer.LexSymbol()
+	}
+
+	switch TokenKind(r) {
+	case TokenKindGet:
+		fallthrough
+	case TokenKindSet:
+		fallthrough
+	case TokenKindOpenBlock:
+		fallthrough
+	case TokenKindCloseBlock:
+		fallthrough
+	case TokenKindOpenList:
+		fallthrough
+	case TokenKindCloseList:
+		fallthrough
+	case TokenKindOpenParen:
+		fallthrough
+	case TokenKindCloseParen:
+		lexer.Next()
+		return lexer.MakeToken(TokenKind(r))
+	default:
+	}
+
+	if unicode.IsLetter(r) {
+		return lexer.LexWord()
+	}
+
+	return lexer.MakeErrorfToken("Unknown character: %c", r)
 }
 
-func (lexer * Lexer) LexAll() []Token {
-    var token Token
+func (lexer *Lexer) Lex() Token {
+	res := lexer.lex()
+	lexer.ClearBuffer() // ensure buffer is cleared after lexing, always.
+	return res
+}
 
-    res := make([]Token, 0)
+func (lexer *Lexer) LexAll() []Token {
+	var token Token
 
-    for token = lexer.Lex() ; ! token.IsLast() ; token = lexer.Lex() {
-          fmt.Printf("token: %s %v\n", token.String(), token.IsLast())
-          res = append(res, token)
-    } 
-    
-    fmt.Printf("Last token: %s %v\n", token.String(), token.IsLast())
-    res = append(res, token)
- 
-    return res
-}
+	res := make([]Token, 0)
 
+	for token = lexer.Lex(); !token.IsLast(); token = lexer.Lex() {
+		res = append(res, token)
+	}
+
+	res = append(res, token)
+
+	return res
+}
 
 func NewLexer(scanner io.RuneScanner, filename string) Lexer {
-    lexer := Lexer{}
-    lexer.RuneScanner       = scanner
-    lexer.Position.FileName = filename
-    lexer.Position.Column   = 1
-    lexer.Position.Line     = 1
-    return lexer
+	lexer := Lexer{}
+	lexer.RuneScanner = scanner
+	lexer.Position.FileName = filename
+	lexer.Position.Column = 1
+	lexer.Position.Line = 1
+	return lexer
 }
 
 func NewLexerFromInputString(input string) Lexer {
-    reader := strings.NewReader(input)
-    return NewLexer(reader, "<input>") 
+	reader := strings.NewReader(input)
+	return NewLexer(reader, "<input>")
 }
 
-
 func NewLexerFromFileName(filename string) (*Lexer, error) {
-    read, err   := os.Open(filename)
-    if err != nil {
-        bread   := bufio.NewReader(read)
-        lex     := NewLexer(bread, filename)
-        return &lex, nil
-    }
-    return nil , err
+	read, err := os.Open(filename)
+	if err != nil {
+		bread := bufio.NewReader(read)
+		lex := NewLexer(bread, filename)
+		return &lex, nil
+	}
+	return nil, err
 }
- 
-
-

lexer_test.go

@@ -1,39 +1,65 @@
 package muesli
 
 import (
-    _ "strings"
-    "testing"
+	_ "strings"
+	"testing"
 )
 
-
 func LexText(input string) []Token {
-    lexer  := NewLexerFromInputString(input)
-    tokens := lexer.LexAll()
-    return tokens
+	lexer := NewLexerFromInputString(input)
+	tokens := lexer.LexAll()
+	return tokens
 }
 
 func Assert(test *testing.T, ok bool, text string) bool {
-    if !ok {
-        test.Errorf(text)
+	if !ok {
+		test.Errorf(text)
+	}
+	return ok
+}
+
+func HelperTryLexText(input string, test *testing.T) {
+	tokens := LexText(input)
+	for i := 0; i < len(tokens); i++ {
+		// test.Logf("%d: %s", i, tokens[i].String())
+	}
+}
+
+func HelperLexExpect(input string, wantKind TokenKind, wantValue Value, test *testing.T) {
+	lexer := NewLexerFromInputString(input)
+	token := lexer.Lex()
+    if (token.TokenKind == wantKind) && (token.Value == wantValue) {
+        test.Logf("Token as expected %v %v", token.TokenKind, token.Value)
+    } else {
+        test.Errorf("Unexpected token kind or value: %v %v >%v< >%v<", 
+            token.TokenKind, wantKind, token.Value, wantValue)
     }
-    return ok
 }
 
-func HelperTryLexText(input string, test * testing.T) {
-  tokens := LexText(input)
-  for i:= 0; i < len(tokens) ; i++ {
-    test.Logf("%d: %s", i, tokens[i].String())
-  }
+func TestLex(test *testing.T) {
+    HelperLexExpect("word\n", TokenKindWord, StringValue("word"), test)
+    HelperLexExpect(":symbol\n", TokenKindSymbol, StringValue("symbol"), test)
+    HelperLexExpect("1234\n", TokenKindInteger, IntValue(1234), test)
+    HelperLexExpect("-3.14\n", TokenKindFloat, FloatValue(-3.14), test)
+    HelperLexExpect(`"Hello \"world" \n`, TokenKindString, StringValue(`Hello "world`), test)
+    HelperLexExpect("true\n", TokenKindBoolean, TrueValue, test)
+    HelperLexExpect("false\n", TokenKindBoolean, FalseValue, test)
 }
 
+
 func TestLexing(test *testing.T) {
-    const input = `
+	const input = `
     greet "hi there"
     
     add 5 10 
     mulf -2.0 3.1415
-    
-    say "hello \"world\\"
+    say1 "unicode « ❤ 🂱"
+    say2 "hello world"
+    say3 "quote \" endquote"
+    say4 "escape \a\b\e\f\n\r\t\"\\"
+    say5 "numescape \xab \u2764 \U01f0b1"
+    say_6 "hello \"world\\"
+    :❤a_symbol❤
 
 define open a door {
     set (door open) true
@@ -43,7 +69,7 @@ def increment variable by value {
     =variable (add variable $value)
 }
 `
-    test.Log("Hi test!")
+	test.Log("Hi test!")
 
-    HelperTryLexText(input, test)
-} 
+	HelperTryLexText(input, test)
+}

parser.go

@@ -1,86 +1,131 @@
-package muesli 
+package muesli
 
 import (
-    _ "bytes"
-    _ "errors"
-    _ "fmt"
-    _ "io"
-    _ "reflect"
-    _ "runtime"
-    _ "strings"
-    _ "unicode"
-    _ "io"
-    _ "os"
-    _ "bufio"
-    _ "unicode"
-    // "gitlab.com/beoran/woe/graphviz"
-    // _ "gitlab.com/beoran/woe/monolog"
-    "gitlab.com/beoran/gdast/tree"
+	_ "bufio"
+	_ "bytes"
+	_ "errors"
+	_ "fmt"
+	_ "io"
+	_ "os"
+	_ "reflect"
+	_ "runtime"
+	_ "strings"
+	_ "unicode"
+	// "gitlab.com/beoran/woe/graphviz"
+	// _ "gitlab.com/beoran/woe/monolog"
 )
 
-
-
 type AstKind int
 
 const (
-    AstKindProgram      = AstKind(iota)
-    AstKindStatements
-    AstKindStatement
-    AstKindCommand
-    AstKindArguments
-    AstKindBlock
-    AstKindList
-    AstKindCapture
-    AstKindWordValue
-    AstKindWord
-    AstKindType
-    AstKindValue
-    AstKindEox
-    AstKindError
+	AstKindProgram = AstKind(iota)
+	AstKindStatements
+	AstKindStatement
+	AstKindSet
+	AstKindGet
+	AstKindCommand
+	AstKindArguments
+	AstKindBlock
+	AstKindList
+	AstKindCapture
+	AstKindWordValue
+	AstKindWord
+	AstKindType
+	AstKindValue
+	AstKindEox
+	AstKindError
 )
 
-
-
-
-
-/* AST node kind */
-type Ast struct {
-    tree.Node
-    AstKind
-    * Token
-}
-
-
 type Parser struct {
-    Lexer
-    Ast
+	Lexer Lexer
+	Ast
+	next    *Token
+	current *Token
 }
 
-
-
-
-
-func (parser * Parser) ParseProgram() error {
-    /*
-    for parser.Position < len(Input) {
-    }
-    */
-    return nil
+func (parser *Parser) Peek() *Token {
+	if parser.next == nil {
+		token := parser.Lexer.Lex()
+		parser.next = &token
+	}
+	return parser.next
 }
 
+func (parser *Parser) Next() *Token {
+	next := parser.Peek()
+	parser.current = next
+	parser.next = nil
+	parser.Peek()
+	return parser.current
+}
 
+func (parser *Parser) Require(wanted TokenKind, astkind AstKind, parent *Ast) *Ast {
+	token := parser.Next()
+	if token.TokenKind == wanted {
+		return parent.NewChild(astkind, token)
+	}
+	return parent.NewChild(AstKindError, token)
+}
 
-func (parser * Parser) Parse() (Ast, error) {
-    parser.Index     = 0
-    err             := parser.ParseProgram()
-    return parser.Ast, err
+func (parser *Parser) ParseMany(
+	astkind AstKind, parsefunc func(*Parser) *Ast) *Ast {
+	ast := NewAst(astkind, nil, nil)
+	for sub := parsefunc(parser); sub != nil && sub.AstKind != AstKindError; sub = parsefunc(parser) {
+		ast.AppendChild(sub)
+	}
+	return ast
 }
 
+func (parser *Parser) NewAstError(message string) *Ast {
+	sv := StringValue(message)
+	pos := parser.current.Position
+	tok := NewToken(TokenKindError, sv, pos)
+	return NewAst(AstKindError, nil, &tok)
+}
 
+func (parser *Parser) ParseAny(astkind AstKind, parsefuncs ...(func(*Parser) *Ast)) *Ast {
+	ast := NewAst(astkind, nil, nil)
+	for _, parsefunc := range parsefuncs {
+		sub := parsefunc(parser)
+		if sub != nil {
+			ast.AppendChild(sub)
+			return ast
+		}
+	}
+	err := parser.NewAstError("Unexpected token")
+	ast.AppendChild(err)
+	return ast
+}
 
+func (parser *Parser) ParseSet() *Ast {
+	return nil
+}
 
+func (parser *Parser) ParseGet() *Ast {
+	return nil
+}
 
+func (parser *Parser) ParseCommand() *Ast {
+	return nil
+}
 
+func (parser *Parser) ParseStatement() *Ast {
+	return parser.ParseAny(AstKindStatement,
+		(*Parser).ParseSet, (*Parser).ParseGet, (*Parser).ParseCommand)
+}
 
+func (parser *Parser) ParseStatements() *Ast {
+	return parser.ParseMany(AstKindStatements, (*Parser).ParseStatement)
+}
 
+func (parser *Parser) ParseProgram() *Ast {
+	ast := NewAst(AstKindProgram, nil, nil)
+	stats := parser.ParseStatements()
+	ast.AppendChild(stats)
+	return ast
+}
 
+func (parser *Parser) Parse() *Ast {
+	ast := parser.ParseProgram()
+	return ast
+}

token.go

@@ -1,106 +1,103 @@
 package muesli
 
-
 import (
-    "fmt"
+	"fmt"
 )
 
-
 /* Position of a token in an input stream */
 type Position struct {
-    FileName    string
-    Line        int
-    Column      int
+	FileName string
+	Line     int
+	Column   int
 }
 
-
 /* Token Kind. Uses a rune to easily handle single character tokens. */
 type TokenKind rune
 
-const (    
-    TokenKindInteger    = TokenKind('i') 
-    TokenKindFloat      = TokenKind('f')
-    TokenKindString     = TokenKind('s')
-    TokenKindBoolean    = TokenKind('b')
-    TokenKindWord       = TokenKind('w')
-    TokenKindType       = TokenKind('t')
-    TokenKindGet        = TokenKind('$')
-    TokenKindSet        = TokenKind('=')
-    TokenKindOpenBlock  = TokenKind('{')
-    TokenKindCloseBlock = TokenKind('}')
-    TokenKindOpenList   = TokenKind('[')
-    TokenKindCloseList  = TokenKind(']')
-    TokenKindOpenParen  = TokenKind('(')
-    TokenKindCloseParen = TokenKind(')')
-    TokenKindError      = TokenKind('!')
-    TokenKindEOX        = TokenKind('\n')
-    TokenKindEOF        = TokenKind(0x255)
+const (
+	TokenKindInteger    = TokenKind('i')
+	TokenKindFloat      = TokenKind('f')
+	TokenKindString     = TokenKind('s')
+    TokenKindSymbol     = TokenKind('S')
+	TokenKindBoolean    = TokenKind('b')
+	TokenKindWord       = TokenKind('w')
+	TokenKindType       = TokenKind('t')
+	TokenKindGet        = TokenKind('$')
+	TokenKindSet        = TokenKind('=')
+	TokenKindOpenBlock  = TokenKind('{')
+	TokenKindCloseBlock = TokenKind('}')
+	TokenKindOpenList   = TokenKind('[')
+	TokenKindCloseList  = TokenKind(']')
+	TokenKindOpenParen  = TokenKind('(')
+	TokenKindCloseParen = TokenKind(')')
+	TokenKindError      = TokenKind('!')
+	TokenKindEOX        = TokenKind('\n')
+	TokenKindEOF        = TokenKind(0x255)
 )
 
 /* Names of the different token types. */
-var TokenKindNames map[TokenKind] string = map[TokenKind]string{
-    TokenKindInteger   :  "Integer"   ,
-    TokenKindFloat     :  "Float"     ,
-    TokenKindString    :  "String"    ,
-    TokenKindBoolean   :  "Boolean"   ,
-    TokenKindWord      :  "Word"      ,
-    TokenKindType      :  "Type"      ,
-    TokenKindGet       :  "Get"       ,
-    TokenKindSet       :  "Set"       ,
-    TokenKindOpenBlock :  "OpenBlock" ,
-    TokenKindCloseBlock:  "CloseBlock",
-    TokenKindOpenList  :  "OpenList"  ,
-    TokenKindCloseList :  "CloseList" ,
-    TokenKindOpenParen :  "OpenParen" ,
-    TokenKindCloseParen:  "CloseParen",
-    TokenKindError     :  "Error"     ,
-    TokenKindEOX       :  "EOX"       ,
-    TokenKindEOF       :  "EOF"       , 
+var TokenKindNames map[TokenKind]string = map[TokenKind]string{
+	TokenKindInteger:    "Integer",
+	TokenKindFloat:      "Float",
+	TokenKindSymbol:     "Symbol",
+	TokenKindString:     "String",
+    TokenKindBoolean:    "Boolean",
+	TokenKindWord:       "Word",
+	TokenKindType:       "Type",
+	TokenKindGet:        "Get",
+	TokenKindSet:        "Set",
+	TokenKindOpenBlock:  "OpenBlock",
+	TokenKindCloseBlock: "CloseBlock",
+	TokenKindOpenList:   "OpenList",
+	TokenKindCloseList:  "CloseList",
+	TokenKindOpenParen:  "OpenParen",
+	TokenKindCloseParen: "CloseParen",
+	TokenKindError:      "Error",
+	TokenKindEOX:        "EOX",
+	TokenKindEOF:        "EOF",
 }
 
 /* Transforms a token kid to a String */
 func (kind TokenKind) String() string {
-  name, ok := TokenKindNames[kind]
-  if !ok {
-    return "Unknown TokenKind!"
-  }
-  return name
+	name, ok := TokenKindNames[kind]
+	if !ok {
+		return "Unknown TokenKind!"
+	}
+	return name
 }
 
-
 type Token struct {
-    TokenKind
-    Value
-    Position
+	TokenKind
+	Value
+	Position
 }
 
 func (token Token) String() string {
-  return fmt.Sprintf("<%s:%q:%v>", token.TokenKind.String(), 
-                      token.Value, token.Position)
+	return fmt.Sprintf("<%s:%q:%v>", token.TokenKind.String(),
+		token.Value, token.Position)
 }
 
 func (token Token) Error() string {
-    if token.TokenKind == TokenKindError {
-        return token.Value.(string)
-    }
-    return "No error"
+	if token.TokenKind == TokenKindError {
+		return token.Value.(string)
+	}
+	return "No error"
 }
 
-/* Returns whether or not the token is the last to be expected, 
+/* Returns whether or not the token is the last to be expected,
  * that is either an error or EOF. */
 func (token Token) IsLast() bool {
-    switch token.TokenKind {  
-        case TokenKindError:    return true
-        case TokenKindEOF:      return true
-        default:                return false;
-    }
+	switch token.TokenKind {
+	case TokenKindError:
+		return true
+	case TokenKindEOF:
+		return true
+	default:
+		return false
+	}
 }
 
-
 /* Creates a new token. */
 func NewToken(kind TokenKind, val Value, pos Position) Token {
-    return Token{kind, val, pos}
+	return Token{kind, val, pos}
 }
-
-
-

value.go

@@ -1,6 +1,5 @@
 package muesli
 
-
 /* Run time values */
 type Value interface {
 }
@@ -13,14 +12,22 @@ type StringValue string
 
 type BoolValue bool
 
+const (
+    TrueValue    = BoolValue(true)
+    FalseValue   = BoolValue(false)
+)
+
+var     NilValue = Value(nil)
+
 type WordValue string
 
 type TypeValue string
 
+type ErrorValue error
+
 type AnyValue struct {
 }
 
 type ListValue struct {
-    List []Value
+	List []Value
 }
-