ll1/flexer/flexer_test.go

package flexer

import "testing"

const (
	tWord = Kind(-1 - iota)
	tArrow
	tSpace
	tString
	tPlus = Kind('+')
	tEos  = Kind('.')
)

func TestFlexer(t *testing.T) {
	pos := Position{}
	expected := []Token{
		MakeToken(pos, tSpace, "\t "),
		MakeToken(pos, tWord, "PROGRAM"),
		MakeToken(pos, tSpace, "  "),
		MakeToken(pos, tArrow, "->"),
		MakeToken(pos, tSpace, "  "),
		MakeToken(pos, tWord, "STATEMENT"),
		MakeToken(pos, tPlus, "+"),
		MakeToken(pos, tSpace, " "),
		MakeToken(pos, tEos, ".\n"),
		MakeToken(pos, tWord, "say"),
		MakeToken(pos, tSpace, " "),
		MakeToken(pos, tString, "hello\nworld"),
		MakeToken(pos, tEos, "."),
	}
	f := NewFlexer(`test`, "\t PROGRAM  ->  STATEMENT+ .\nsay \"hello\\nworld\".")
	f.Rule(tSpace, `[ \t]+`, "", nil)
	f.Rule(tWord, `[A-Za-z_]+`, "", nil)
	f.Rule(tArrow, `\->`, "", nil)
	f.Rule(tPlus, `\+`, "", nil)
	f.Rule(tEos, `\.[\n\r]*`, "", nil)
	f.Rule(SkipKind, `"`, "", ContextAction("string"))
	f.Rule(tString, `"`, "string", PopAction(tString))
	f.Rule(SkipKind, `\\[etnru][0-9a-f]*`, "string", EscapeAction('"'))
	f.Rule(SkipKind, `.`, "string", StoreAction())

	toks := LexAll(f)

	for i, e := range expected {
		tok := toks[i]
		t.Logf("toks: %d, %v", i, tok)
		ko := tok.Kind()
		ke := e.Kind()
		if ko != ke {
			t.Errorf("error: kind:%d|%d|", ko, ke)
		}
		to := tok.Text()
		te := e.Text()
		if to != te {
			t.Errorf("error: text:%s|%s|", to, te)
		}
	}
	if !f.EOF() {
		t.Errorf("error: should be EOF")
	}
}