muesli/lexer_test.go

package muesli

import (
	"reflect"
	"runtime"
	_ "strings"
	"testing"
)

func LexText(input string) []Token {
	lexer := NewLexerFromString(input)
	tokens := lexer.LexAll()
	return tokens
}

func Assert(test *testing.T, ok bool, text string) bool {
	if !ok {
		test.Errorf(text)
	}
	return ok
}

func HelperTryLexText(input string, test *testing.T) {
	tokens := LexText(input)
	for i := 0; i < len(tokens); i++ {
		// test.Logf("%d: %s", i, tokens[i].String())
	}
}

func HelperLexExpect(input string, wantKind TokenKind, wantValue Value, test *testing.T) {
	lexer := NewLexerFromString(input)
	lexer.SetLogger(&testLogger{})
	token := lexer.Lex()
	if (token.TokenKind == wantKind) && (token.Value == wantValue) {
		/* test.Logf("Token as expected %v %v", token.TokenKind, token.Value) */
	} else {
		test.Errorf("Unexpected token kind or value: %v %v >%v< >%v<",
			token.TokenKind, wantKind, token.Value, wantValue)
	}
}

func HelperFunctionName(f interface{}) string {
	fp := reflect.ValueOf(f).Pointer()
	info := runtime.FuncForPC(fp)
	if info != nil {
		return info.Name()
	}
	return "unknown"
}

func HelperLexTestSkip(input string, want rune, call func(*Lexer) error, test *testing.T) {
	var r rune
	lexer := NewLexerFromString(input)
	lexer.SetLogger(&testLogger{})
	fn := HelperFunctionName(call)
	err := call(lexer)
	if err != nil {
		test.Errorf("Unexpected error result: %s: %v", fn, err)
	}
	r, err = lexer.Peek()
	if err != nil {
		test.Errorf("Unexpected error result on peek for %s: %v", fn, err)
	}
	if r != want {
		test.Errorf("Unexpected character peeked for %s: %c: %c", fn, r, want)
	}
}

func TestLexParts(test *testing.T) {
	HelperLexTestSkip(" abc", 'a', (*Lexer).SkipSpace, test)
	HelperLexTestSkip("     xyz", 'x', (*Lexer).SkipSpace, test)
	HelperLexTestSkip(" #  \nd", 'd', (*Lexer).SkipComment, test)
	HelperLexTestSkip("#{}e", 'e', (*Lexer).SkipComment, test)
	HelperLexTestSkip("#{{}{{}}}f", 'f', (*Lexer).SkipComment, test)
	HelperLexTestSkip("    \tword\n", 'w', (*Lexer).SkipSpace, test)
}

func TestLex(test *testing.T) {
	HelperLexExpect("word\n", TokenKindWord, StringValue("word"), test)
	HelperLexExpect(":symbol\n", TokenKindSymbol, StringValue("symbol"), test)
	HelperLexExpect("1234\n", TokenKindInteger, IntValue(1234), test)
	HelperLexExpect("-3.14\n", TokenKindFloat, FloatValue(-3.14), test)
	HelperLexExpect(`"Hello \"world" \n`, TokenKindString, StringValue(`Hello "world`), test)
	HelperLexExpect("true\n", TokenKindBoolean, TrueValue, test)
	HelperLexExpect("false\n", TokenKindBoolean, FalseValue, test)
	HelperLexExpect("    \tword\n", TokenKindWord, StringValue("word"), test)
	/*
		HelperLexExpect("# comment should be ignored\ntrue\n", TokenKindBoolean, TrueValue, test)
		HelperLexExpect("  # comment should be ignored\ntrue\n", TokenKindBoolean, TrueValue, test)
		HelperLexExpect("  # comment should be ignored\n  true\n", TokenKindBoolean, TrueValue, test)
		HelperLexExpect("#{ comment should be ignored\n this too } true\n", TokenKindBoolean, TrueValue, test)
	*/
}

func TestLexing(test *testing.T) {
	const input = `
    greet "hi there"
    
    add 5 10 
    mulf -2.0 3.1415
    say1 "unicode « ❤ 🂱"
    say2 "hello world"
    say3 "quote \" endquote"
    say4 "escape \a\b\e\f\n\r\t\"\\"
    say5 "numescape \xab \u2764 \U01f0b1"
    say_6 "hello \"world\\"
    :❤a_symbol❤

define open a door {
    set (door open) true
}

def increment variable by value {
    =variable (add variable $value)
}
`
	// HelperTryLexText(input, test)
}


func TestLexDesignFile(test *testing.T) {
	lexer, err := NewLexerFromFilename("design_muesli.muesli")
	if err != nil { 
        test.Errorf("Error parsing file : %s", err)
        return
    }
    if lexer == nil {
        test.Errorf("Error: lexer is nil.")
        return
    }
    defer lexer.Report()
    tokens := lexer.LexAll()
	for i := 0; i < len(tokens); i++ {
		test.Logf("%d: %s", i, tokens[i].String())
	}
}