muesli/lexer_test.go

package muesli

import (
	_ "strings"
	"testing"
)

func LexText(input string) []Token {
	lexer := NewLexerFromInputString(input)
	tokens := lexer.LexAll()
	return tokens
}

func Assert(test *testing.T, ok bool, text string) bool {
	if !ok {
		test.Errorf(text)
	}
	return ok
}

func HelperTryLexText(input string, test *testing.T) {
	tokens := LexText(input)
	for i := 0; i < len(tokens); i++ {
		// test.Logf("%d: %s", i, tokens[i].String())
	}
}

func HelperLexExpect(input string, wantKind TokenKind, wantValue Value, test *testing.T) {
	lexer := NewLexerFromInputString(input)
	token := lexer.Lex()
    if (token.TokenKind == wantKind) && (token.Value == wantValue) {
        test.Logf("Token as expected %v %v", token.TokenKind, token.Value)
    } else {
        test.Errorf("Unexpected token kind or value: %v %v >%v< >%v<", 
            token.TokenKind, wantKind, token.Value, wantValue)
    }
}

func TestLex(test *testing.T) {
    HelperLexExpect("word\n", TokenKindWord, StringValue("word"), test)
    HelperLexExpect(":symbol\n", TokenKindSymbol, StringValue("symbol"), test)
    HelperLexExpect("1234\n", TokenKindInteger, IntValue(1234), test)
    HelperLexExpect("-3.14\n", TokenKindFloat, FloatValue(-3.14), test)
    HelperLexExpect(`"Hello \"world" \n`, TokenKindString, StringValue(`Hello "world`), test)
    HelperLexExpect("true\n", TokenKindBoolean, TrueValue, test)
    HelperLexExpect("false\n", TokenKindBoolean, FalseValue, test)
}


func TestLexing(test *testing.T) {
	const input = `
    greet "hi there"
    
    add 5 10 
    mulf -2.0 3.1415
    say1 "unicode « ❤ 🂱"
    say2 "hello world"
    say3 "quote \" endquote"
    say4 "escape \a\b\e\f\n\r\t\"\\"
    say5 "numescape \xab \u2764 \U01f0b1"
    say_6 "hello \"world\\"
    :❤a_symbol❤

define open a door {
    set (door open) true
}

def increment variable by value {
    =variable (add variable $value)
}
`
	test.Log("Hi test!")

	HelperTryLexText(input, test)
}