flexer_lexer.go 2.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081
  1. package flexgen
  2. import "os"
  3. import "io"
  4. import "src.eruta.nl/beoran/ll1/common"
  5. import "src.eruta.nl/beoran/ll1/flexer"
  6. const (
  7. FlexgenKindDot = common.Kind(-1 - iota)
  8. FlexgenKindLiteralString
  9. FlexgenKindLiteralRaw
  10. FlexgenKindLiteralChar
  11. FlexgenKindTerminal
  12. FlexgenKindArrow
  13. FlexgenKindKeyword
  14. FlexgenKindWhitespace
  15. FlexgenKindLineComment
  16. FlexgenKindBlockComment
  17. FlexgenKindAction
  18. )
  19. /*
  20. // common generator's input own lexer specification
  21. dot -> '\.' .
  22. arrow -> "(?:->|→)" .
  23. terminal -> "[[:isLower:]][[:isAlNum]_-]+" .
  24. literal-string -> `"` flex-esc-string .
  25. literal-raw -> "`" flex-string .
  26. literal-char -> "'" flex-string .
  27. common-keyword -> "flex-skip|flex-string|flex-esc-string" .
  28. whitespace -> "[ \t\n\r]+" flex-skip .
  29. line-comment -> "//[^\n\r]+" flex-skip .
  30. block-comment -> "/\*(?ms:.)*?\* /" .
  31. common-action -> `%{(?ms:.)*?}%` .
  32. */
  33. type CheckedError struct {
  34. Error error
  35. }
  36. func Check(err error) {
  37. if err != nil {
  38. panic(CheckedError{err})
  39. }
  40. }
  41. func LexInputString(name, input string) []common.Token {
  42. f := flexer.NewFlexer(name, input)
  43. Check(f.Lexeme(FlexgenKindWhitespace, `[ \t\n\r]+`, "", nil))
  44. Check(f.Lexeme(FlexgenKindLineComment, `//[^\n\r]+[\n\r]+`, "", nil))
  45. Check(f.Lexeme(FlexgenKindBlockComment, `/\*(?ms:.)*?\*/`, "", nil))
  46. Check(f.Lexeme(FlexgenKindDot, `\.`, "", nil))
  47. Check(f.Lexeme(FlexgenKindArrow, `(?:->|→)`, "", nil))
  48. Check(f.Lexeme(FlexgenKindTerminal, `[[:lower:]][[:alnum:]_-]+`, "", nil))
  49. f.EscapedStringLexeme(FlexgenKindLiteralString, `"`, `"`, "literal-string")
  50. f.RawStringLexeme(FlexgenKindLiteralRaw, "`", "`", "literal-raw")
  51. f.RawStringLexeme(FlexgenKindLiteralChar, `''`, `''`, "literal-char")
  52. Check(f.Lexeme(FlexgenKindKeyword, `flex-skip|flex-string|flex-esc-string`, "", nil))
  53. Check(f.Lexeme(FlexgenKindAction, `@{(?ms:.)*?}@`, "", nil))
  54. skipKinds := []common.Kind{common.SkipKind, FlexgenKindWhitespace, FlexgenKindBlockComment, FlexgenKindLineComment}
  55. return flexer.LexAll(f, skipKinds...)
  56. }
  57. func LexFileName(name string) ([]common.Token, error) {
  58. fin, err := os.Open(name)
  59. if err != nil {
  60. return nil, err
  61. }
  62. defer fin.Close()
  63. buf, err := io.ReadAll(fin)
  64. if err != nil {
  65. return nil, err
  66. }
  67. input := string(buf)
  68. return LexInputString(name, input), nil
  69. }