Add end token and rule to ensure proper ending of lexing and parsing.

common/common.go

@@ -23,6 +23,7 @@ func (p Location) String() string {
 type Kind int
 
 const (
+	EndKind   Kind = -29000
 	SkipKind  Kind = -30000
 	ErrorKind Kind = -31000
 )

flexer/flexer.go

@@ -303,7 +303,8 @@ func KeepToken(tok Token, skips ...Kind) bool {
 // Lexes all tokens from the lexer until it reaches
 // EOF, or until it cannot progress anymore.
 // All tokens in the skip array will be skipped
-// from the results.
+// from the results. If the lexer reachest he end of input,
+// a token with kind EndKind will be appended
 func LexAll(lex Lexer, skips ...Kind) []Token {
 	res := []Token{}
 	for !lex.EOF() {
@@ -319,5 +320,7 @@ func LexAll(lex Lexer, skips ...Kind) []Token {
 			}
 		}
 	}
+	// here we reached EOF
+	res = append(res, lex.MakeToken(EndKind, "<end>"))
 	return res
 }

flexgen/flexer_parser.go

@@ -20,7 +20,7 @@ func MakeFlexerGrammar() *Grammar {
 	lexeme := g.Seq("lexeme", "", terminal, arrow, pattern, keywords, optAction, dot)
 	lexemesRef := g.Ref("lexemesRef", "lexemes")
 	lexemes := g.Opt("lexemes", "", And(lexeme, lexemesRef))
-	top := g.Seq("top", "", lexemes, End{})
+	top := g.Seq("top", "", lexemes, End())
 	g.Top = top
 	return g
 }

grammar/grammar.go

@@ -88,14 +88,19 @@ func (e Epsilon) String() string {
 }
 
 // End corresponds to EOF or the end of the input.
-type End struct {
+type EndOfInput struct {
 	Terminal
 }
 
-func (e End) String() string {
+func (e EndOfInput) String() string {
 	return "$"
 }
 
+func End() EndOfInput {
+	terminal := Term("<end>", EndKind)
+	return EndOfInput{terminal}
+}
+
 type Nonterminal struct {
 	BasicRule
 	Define   Rule
@@ -390,7 +395,7 @@ func (g *Grammar) AddRule(r Rule) Rule {
 	return r
 }
 
-func Term(name string, kind Kind) Rule {
+func Term(name string, kind Kind) Terminal {
 	term := Terminal{}
 	term.name = name
 	term.Kind = kind

parser/parser.go

@@ -94,8 +94,9 @@ func (g *Parser) ParseEpsilon(term Epsilon) error {
 	return nil
 }
 
-func (p *Parser) ParseEnd(term End) error {
-	if p.Index >= len(p.Tokens) {
+func (p *Parser) ParseEnd(term EndOfInput) error {
+	if p.Index >= (len(p.Tokens) - 1) {
+		p.Index = len(p.Tokens) // reach the end.
 		return nil
 	} else {
 		return p.MakeError("Expected end of input.")
@@ -187,7 +188,7 @@ func (p *Parser) ParseRule(r Rule) error {
 	switch rv := r.(type) {
 	case Epsilon:
 		p.ParseEpsilon(rv)
-	case End:
+	case EndOfInput:
 		p.ParseEnd(rv)
 	case Terminal:
 		p.ParseTerminal(rv)