lexer_test.go 3.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140
  1. package muesli
  2. import (
  3. "reflect"
  4. "runtime"
  5. _ "strings"
  6. "testing"
  7. )
  8. func LexText(input string) []Token {
  9. lexer := NewLexerFromString(input)
  10. tokens := lexer.LexAll()
  11. return tokens
  12. }
  13. func Assert(test *testing.T, ok bool, text string) bool {
  14. if !ok {
  15. test.Errorf(text)
  16. }
  17. return ok
  18. }
  19. func HelperTryLexText(input string, test *testing.T) {
  20. tokens := LexText(input)
  21. for i := 0; i < len(tokens); i++ {
  22. // test.Logf("%d: %s", i, tokens[i].String())
  23. }
  24. }
  25. func HelperLexExpect(input string, wantKind TokenKind, wantValue Value, test *testing.T) {
  26. lexer := NewLexerFromString(input)
  27. lexer.SetLogger(&testLogger{})
  28. token := lexer.Lex()
  29. if (token.TokenKind == wantKind) && (token.Value == wantValue) {
  30. /* test.Logf("Token as expected %v %v", token.TokenKind, token.Value) */
  31. } else {
  32. test.Errorf("Unexpected token kind or value: %v %v >%v< >%v<",
  33. token.TokenKind, wantKind, token.Value, wantValue)
  34. }
  35. }
  36. func HelperFunctionName(f interface{}) string {
  37. fp := reflect.ValueOf(f).Pointer()
  38. info := runtime.FuncForPC(fp)
  39. if info != nil {
  40. return info.Name()
  41. }
  42. return "unknown"
  43. }
  44. func HelperLexTestSkip(input string, want rune, call func(*Lexer) error, test *testing.T) {
  45. var r rune
  46. lexer := NewLexerFromString(input)
  47. lexer.SetLogger(&testLogger{})
  48. fn := HelperFunctionName(call)
  49. err := call(lexer)
  50. if err != nil {
  51. test.Errorf("Unexpected error result: %s: %v", fn, err)
  52. }
  53. r, err = lexer.Peek()
  54. if err != nil {
  55. test.Errorf("Unexpected error result on peek for %s: %v", fn, err)
  56. }
  57. if r != want {
  58. test.Errorf("Unexpected character peeked for %s: %c: %c", fn, r, want)
  59. }
  60. }
  61. func TestLexParts(test *testing.T) {
  62. HelperLexTestSkip(" abc", 'a', (*Lexer).SkipSpace, test)
  63. HelperLexTestSkip(" xyz", 'x', (*Lexer).SkipSpace, test)
  64. HelperLexTestSkip(" # \nd", 'd', (*Lexer).SkipComment, test)
  65. HelperLexTestSkip("#{}e", 'e', (*Lexer).SkipComment, test)
  66. HelperLexTestSkip("#{{}{{}}}f", 'f', (*Lexer).SkipComment, test)
  67. HelperLexTestSkip(" \tword\n", 'w', (*Lexer).SkipSpace, test)
  68. }
  69. func TestLex(test *testing.T) {
  70. HelperLexExpect("word\n", TokenKindWord, StringValue("word"), test)
  71. HelperLexExpect(":symbol\n", TokenKindSymbol, StringValue("symbol"), test)
  72. HelperLexExpect("1234\n", TokenKindInteger, IntValue(1234), test)
  73. HelperLexExpect("-3.14\n", TokenKindFloat, FloatValue(-3.14), test)
  74. HelperLexExpect(`"Hello \"world" \n`, TokenKindString, StringValue(`Hello "world`), test)
  75. HelperLexExpect("true\n", TokenKindBoolean, TrueValue, test)
  76. HelperLexExpect("false\n", TokenKindBoolean, FalseValue, test)
  77. HelperLexExpect(" \tword\n", TokenKindWord, StringValue("word"), test)
  78. /*
  79. HelperLexExpect("# comment should be ignored\ntrue\n", TokenKindBoolean, TrueValue, test)
  80. HelperLexExpect(" # comment should be ignored\ntrue\n", TokenKindBoolean, TrueValue, test)
  81. HelperLexExpect(" # comment should be ignored\n true\n", TokenKindBoolean, TrueValue, test)
  82. HelperLexExpect("#{ comment should be ignored\n this too } true\n", TokenKindBoolean, TrueValue, test)
  83. */
  84. }
  85. func TestLexing(test *testing.T) {
  86. const input = `
  87. greet "hi there"
  88. add 5 10
  89. mulf -2.0 3.1415
  90. say1 "unicode « ❤ 🂱"
  91. say2 "hello world"
  92. say3 "quote \" endquote"
  93. say4 "escape \a\b\e\f\n\r\t\"\\"
  94. say5 "numescape \xab \u2764 \U01f0b1"
  95. say_6 "hello \"world\\"
  96. :❤a_symbol❤
  97. define open a door {
  98. set (door open) true
  99. }
  100. def increment variable by value {
  101. =variable (add variable $value)
  102. }
  103. `
  104. // HelperTryLexText(input, test)
  105. }
  106. func TestLexDesignFile(test *testing.T) {
  107. lexer, err := NewLexerFromFilename("design_muesli.muesli")
  108. if err != nil {
  109. test.Errorf("Error parsing file : %s", err)
  110. return
  111. }
  112. if lexer == nil {
  113. test.Errorf("Error: lexer is nil.")
  114. return
  115. }
  116. defer lexer.Report()
  117. tokens := lexer.LexAll()
  118. for i := 0; i < len(tokens); i++ {
  119. test.Logf("%d: %s", i, tokens[i].String())
  120. }
  121. }