ll1/flexgen/flexer_lexer.go

package flexgen

import "os"
import "io"

import "src.eruta.nl/beoran/ll1/common"
import "src.eruta.nl/beoran/ll1/flexer"

const (
	FlexgenKindDot = common.Kind(-1 - iota)
	FlexgenKindLiteralString
	FlexgenKindLiteralRaw
	FlexgenKindLiteralChar
	FlexgenKindTerminal
	FlexgenKindArrow
	FlexgenKindKeyword
	FlexgenKindWhitespace
	FlexgenKindLineComment
	FlexgenKindBlockComment
	FlexgenKindAction
)

/*

// common generator's input own lexer specification
dot          	-> '\.' .
arrow        	-> "(?:->|→)" .
terminal     	-> "[[:isLower:]][[:isAlNum]_-]+" .
literal-string  -> `"` flex-esc-string  .
literal-raw     -> "`" flex-string  .
literal-char    -> "'" flex-string  .
common-keyword	-> "flex-skip|flex-string|flex-esc-string" .
whitespace   	-> "[ \t\n\r]+" flex-skip .
line-comment 	-> "//[^\n\r]+" flex-skip .
block-comment	-> "/\*(?ms:.)*?\* /" .
common-action   -> `%{(?ms:.)*?}%` .

*/

type CheckedError struct {
	Error error
}

func Check(err error) {
	if err != nil {
		panic(CheckedError{err})
	}
}

func LexInputString(name, input string) []common.Token {
	f := flexer.NewFlexer(name, input)
	Check(f.Lexeme(FlexgenKindWhitespace, `[ \t\n\r]+`, "", nil))
	Check(f.Lexeme(FlexgenKindLineComment, `//[^\n\r]+[\n\r]+`, "", nil))
	Check(f.Lexeme(FlexgenKindBlockComment, `/\*(?ms:.)*?\*/`, "", nil))
	Check(f.Lexeme(FlexgenKindDot, `\.`, "", nil))
	Check(f.Lexeme(FlexgenKindArrow, `(?:->|→)`, "", nil))
	Check(f.Lexeme(FlexgenKindTerminal, `[[:lower:]][[:alnum:]_-]+`, "", nil))
	f.EscapedStringLexeme(FlexgenKindLiteralString, `"`, `"`, "literal-string")
	f.RawStringLexeme(FlexgenKindLiteralRaw, "`", "`", "literal-raw")
	f.RawStringLexeme(FlexgenKindLiteralChar, `''`, `''`, "literal-char")
	Check(f.Lexeme(FlexgenKindKeyword, `flex-skip|flex-string|flex-esc-string`, "", nil))
	Check(f.Lexeme(FlexgenKindAction, `@{(?ms:.)*?}@`, "", nil))

	skipKinds := []common.Kind{common.SkipKind, FlexgenKindWhitespace, FlexgenKindBlockComment, FlexgenKindLineComment}

	return flexer.LexAll(f, skipKinds...)
}

func LexFileName(name string) ([]common.Token, error) {
	fin, err := os.Open(name)
	if err != nil {
		return nil, err
	}
	defer fin.Close()
	buf, err := io.ReadAll(fin)
	if err != nil {
		return nil, err
	}
	input := string(buf)
	return LexInputString(name, input), nil
}