Add skipping functions, skip quotes on string correctly, and support negative numeric constants.

lexer.go

@@ -68,44 +68,37 @@ func (lexer * Lexer) Peek() (rune, error) {
     return r, err
 }
 
-func (lexer * Lexer) Next()  (rune, error) {  
-    r, _, err := lexer.RuneScanner.ReadRune()
-    if err != nil {
-        return 0, err
-    }
+/* Advances the lexer's position based on the rune r read. */
+func (lexer * Lexer) advance(r rune) {   
     lexer.Current = r
-    lexer.buffer = append(lexer.buffer, r)
     lexer.Index++
     lexer.Position.Column++
     if r == '\n' {
         lexer.Position.Column = 1
         lexer.Position.Line++
     }
-    return lexer.buffer[len(lexer.buffer) - 1], nil
 }
 
-func (lexer * Lexer) oldPrevious() error {
-    fmt.Printf("Previous: now %c \n", lexer.Current)
-    err := lexer.RuneScanner.UnreadRune()
+/* Actually reads the next rune from the lexer's input source and stores
+ * them in the lexer's token buffer. */
+func (lexer * Lexer) Next()  (rune, error) {  
+    r, _, err := lexer.RuneScanner.ReadRune()
     if err != nil {
-        return err
-    }    
-    
-    lexer.Index--
-    lexer.Position.Column--
+        return 0, err
+    }
+    lexer.advance(r)
+    lexer.buffer = append(lexer.buffer, r)
+    return r, nil
+}
 
-    if (len(lexer.buffer) > 0)  { 
-        r := lexer.buffer[len(lexer.buffer) - 1];
-        lexer.buffer = lexer.buffer[0: len(lexer.buffer) - 1];
-    
-        if r == '\n' {
-            lexer.Position.Column = 1 // XXX wrong
-            lexer.Position.Line--
-        }
-        
-        lexer.Current = r
+/* Advances the lexer's input buffer but does not store the read runes*/
+func (lexer * Lexer) Skip() (rune, error) {  
+    r, _, err := lexer.RuneScanner.ReadRune()
+    if err != nil {
+        return 0, err
     }
-    return nil
+    lexer.advance(r)
+    return r, nil
 }
 
 
@@ -124,6 +117,22 @@ func (lexer * Lexer) NextIf(predicate func(rune) bool) (bool, error) {
   return false, nil
 }
 
+func (lexer * Lexer) SkipIf(predicate func(rune) bool) (bool, error) {
+  r, err := lexer.Peek()
+  if err != nil {
+    return false, err
+  }
+  if (predicate(r)) {
+    r, err = lexer.Skip()
+    if err != nil {
+      return true, err
+    }
+    return true, nil
+  }
+  return false, nil
+}
+
+
 func (lexer * Lexer) NextWhile(predicate func(rune) bool) (bool, error) {
     result := true
     ok, err := lexer.NextIf(predicate)
@@ -135,16 +144,24 @@ func (lexer * Lexer) NextWhile(predicate func(rune) bool) (bool, error) {
 }
 
 
+func (lexer * Lexer) SkipWhile(predicate func(rune) bool) (bool, error) {
+    result := true
+    ok, err := lexer.SkipIf(predicate)
+    result = result || ok
+    for ; ok && (err == nil) ; ok, err = lexer.SkipIf(predicate) {
+        result = result || ok
+    }
+    return result, err
+}
+
+
 func isSpace(r rune) bool {
   return r == ' ' || r == '\t'
 }
 
 
 func (lexer * Lexer) SkipSpace() (error) {
-    _, err := lexer.NextWhile(isSpace)
-    if err == nil { 
-      lexer.ClearBuffer()
-    }
+    _, err := lexer.SkipWhile(isSpace)
     return err
  }
 
@@ -163,7 +180,12 @@ func (lexer * Lexer) handleError(err error) Token {
 func (lexer * Lexer) LexNumber() Token { 
   isFloat := false
   
-  _, err := lexer.NextWhile(func (r rune) bool {
+  // skip any first -
+  _, err := lexer.NextIf(func (r rune) bool {
+    return r == '-'
+  })
+   
+  _, err = lexer.NextWhile(func (r rune) bool {
     if unicode.IsDigit(r) {
         return true
     } else if r == '.' {
@@ -197,7 +219,7 @@ func (lexer * Lexer) LexString() Token {
   inEscape := false
   var err error
   
-  _, err = lexer.Next() 
+  _, err = lexer.Skip() // Skip first " 
   if err != nil {
       return lexer.handleError(err)
   }
@@ -222,7 +244,7 @@ func (lexer * Lexer) LexString() Token {
         return lexer.MakeErrorfToken("when parsing string: %s", err)
   }
   
-  _, err = lexer.Next()
+  _, err = lexer.Skip() // skip last "
   if err != nil {
       return lexer.handleError(err)
   }
@@ -233,7 +255,7 @@ func (lexer * Lexer) LexString() Token {
 func (lexer * Lexer) LexLongString() Token { 
   var err error
   
-  _, err = lexer.Next()
+  _, err = lexer.Skip()
   if err != nil {
       return lexer.handleError(err)
   }
@@ -246,7 +268,7 @@ func (lexer * Lexer) LexLongString() Token {
         return lexer.MakeErrorfToken("when parsing long string: %s", err)
   }
   
-  _, err = lexer.Next()
+  _, err = lexer.Skip()
   if err != nil {
       return lexer.handleError(err)
   }
@@ -288,7 +310,7 @@ func (lexer * Lexer) lex() Token {
     }
     
 
-    if unicode.IsDigit(r) {
+    if unicode.IsDigit(r) || r == '-' {
         return lexer.LexNumber()
     }
     

lexer_test.go

@@ -30,6 +30,9 @@ func TestLexing(test *testing.T) {
     const input = `
     greet "hi there"
     
+    add 5 10 
+    mulf -2.0 3.1415
+    
     say "hello \"world\\"
 
 define open a door {