raku.go 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035
  1. // raku
  2. /* Raku is an easy to use scripting language that can also be used easily interactively
  3. Desrired syntax (verified LL(1) on smlweb.cpsc.ucalgary.ca)
  4. PROGRAM -> STATEMENTS.
  5. STATEMENTS -> STATEMENT STATEMENTS | .
  6. STATEMENT -> EXPRESSION EOX | DEFINITION | BLOCK .
  7. DEFINITION -> define WORDOP WORDOPS BLOCK.
  8. WORDOPS -> WORDOP WORDOPS | .
  9. EXPRESSION -> WORDVALUE PARAMETERSS.
  10. PARAMETERS -> PARAMETER PARAMETERS | .
  11. PARAMETER -> WORDVALUE | PARENTHESIS | BLOCK | operator.
  12. PARENTHESIS -> '(' EXPRESSION ')' | ot EXPRESSION ct.
  13. BLOCK -> oe STATEMENTS ce | do STATEMENTS end .
  14. WORDOP -> word | operator | a | the.
  15. WORDVALUE -> word | VALUE | a | the.
  16. VALUE -> string | number | symbol.
  17. EOX -> eol | period.
  18. Lexer:
  19. */
  20. package raku
  21. import (
  22. "bytes"
  23. "fmt"
  24. "io"
  25. "reflect"
  26. "runtime"
  27. "strings"
  28. "unicode"
  29. "gitlab.com/beoran/woe/graphviz"
  30. "gitlab.com/beoran/woe/monolog"
  31. "gitlab.com/beoran/woe/tree"
  32. )
  33. type Value string
  34. type TokenType int64
  35. type Position struct {
  36. Index int
  37. Row int
  38. Column int
  39. }
  40. const (
  41. TokenPeriod TokenType = TokenType('.')
  42. TokenComma TokenType = TokenType(',')
  43. TokenSemicolon TokenType = TokenType(';')
  44. TokenColon TokenType = TokenType(':')
  45. TokenOpenParen TokenType = TokenType('(')
  46. TokenCloseParen TokenType = TokenType(')')
  47. TokenOpenBrace TokenType = TokenType('{')
  48. TokenCloseBrace TokenType = TokenType('}')
  49. TokenOpenBracket TokenType = TokenType('[')
  50. TokenCloseBracket TokenType = TokenType(']')
  51. TokenNone TokenType = 0
  52. TokenError TokenType = -1
  53. TokenWord TokenType = -2
  54. TokenEOL TokenType = -3
  55. TokenEOF TokenType = -4
  56. TokenNumber TokenType = -5
  57. TokenOperator TokenType = -6
  58. TokenString TokenType = -7
  59. TokenSymbol TokenType = -8
  60. TokenFirstKeyword TokenType = -9
  61. TokenKeywordA TokenType = -10
  62. TokenKeywordDo TokenType = -11
  63. TokenKeywordEnd TokenType = -12
  64. TokenKeywordThe TokenType = -13
  65. TokenKeywordDef TokenType = -14
  66. TokenLastKeyword TokenType = -15
  67. TokenLast TokenType = -15
  68. )
  69. type Token struct {
  70. TokenType
  71. Value
  72. Position
  73. }
  74. var tokenTypeMap map[TokenType]string = map[TokenType]string{
  75. TokenNone: "TokenNone",
  76. TokenError: "TokenError",
  77. TokenWord: "TokenWord",
  78. TokenEOL: "TokenEOL",
  79. TokenEOF: "TokenEOF",
  80. TokenNumber: "TokenNumber",
  81. TokenOperator: "TokenOperator",
  82. TokenString: "TokenString",
  83. TokenSymbol: "TokenSymbol",
  84. TokenKeywordA: "TokenKeywordA",
  85. TokenKeywordDo: "TokenKeywordDo",
  86. TokenKeywordEnd: "TokenKeywordEnd",
  87. TokenKeywordThe: "TokenKeywordThe",
  88. TokenKeywordDef: "TokenKeywordDef",
  89. }
  90. var keywordMap map[string]TokenType = map[string]TokenType{
  91. "a": TokenKeywordA,
  92. "an": TokenKeywordA,
  93. "do": TokenKeywordDo,
  94. "def": TokenKeywordDef,
  95. "define": TokenKeywordDef,
  96. "end": TokenKeywordEnd,
  97. "the": TokenKeywordThe,
  98. }
  99. var sigilMap map[string]TokenType = map[string]TokenType{
  100. "[": TokenOpenBracket,
  101. "{": TokenOpenBrace,
  102. "(": TokenOpenParen,
  103. "]": TokenCloseBracket,
  104. "}": TokenCloseBrace,
  105. ")": TokenCloseParen,
  106. }
  107. const operator_chars = "&|@'^-*%/+=<>~\\"
  108. func (me TokenType) String() string {
  109. name, found := tokenTypeMap[me]
  110. if found {
  111. return name
  112. } else {
  113. if (me > 0) && (me < 256) {
  114. return fmt.Sprintf("TokenChar<%c>", byte(me))
  115. }
  116. return fmt.Sprintf("Unknown Token %d", int(me))
  117. }
  118. }
  119. func (me Token) String() string {
  120. return fmt.Sprintf("Token: %s >%s< %d %d %d.", me.TokenType, string(me.Value), me.Index, me.Row, me.Column)
  121. }
  122. type TokenChannel chan *Token
  123. type Lexer struct {
  124. Reader io.Reader
  125. Current Position
  126. Last Position
  127. Token Token
  128. rule LexerRule
  129. Output TokenChannel
  130. buffer []byte
  131. runes []rune
  132. }
  133. type LexerRule func(lexer *Lexer) LexerRule
  134. func (me *Lexer) Emit(t TokenType, v Value) {
  135. tok := &Token{t, v, me.Current}
  136. me.Output <- tok
  137. }
  138. func (me *Lexer) Error(message string, args ...interface{}) {
  139. value := fmt.Sprintf(message, args...)
  140. monolog.Error("Lex Error: %s", value)
  141. me.Emit(TokenError, Value(value))
  142. }
  143. func LexError(me *Lexer) LexerRule {
  144. me.Error("Error")
  145. return nil
  146. }
  147. func (me *Lexer) SkipComment() bool {
  148. if me.Peek() == '#' {
  149. if me.Next() == '(' {
  150. return me.SkipNotIn(")")
  151. } else {
  152. return me.SkipNotIn("\r\n")
  153. }
  154. }
  155. return true
  156. }
  157. /* Returns whether or not a keyword was found, and if so, the TokenType
  158. of the keyword.*/
  159. func LookupKeyword(word string) (bool, TokenType) {
  160. kind, found := keywordMap[word]
  161. return found, kind
  162. }
  163. /* Returns whether or not a special operator or sigil was found, and if so,
  164. returns the TokenTyp of the sigil.*/
  165. func LookupSigil(sigil string) (bool, TokenType) {
  166. fmt.Printf("LookupSigil: %s\n", sigil)
  167. kind, found := sigilMap[sigil]
  168. return found, kind
  169. }
  170. func LexSigil(me *Lexer) LexerRule {
  171. me.Found(TokenType(me.Peek()))
  172. _ = me.Next()
  173. me.Advance()
  174. return LexNormal
  175. }
  176. func LexWord(me *Lexer) LexerRule {
  177. me.SkipNotIn(" \t\r\n'({[]})")
  178. iskw, kind := LookupKeyword(me.CurrentStringValue())
  179. if iskw {
  180. me.Found(kind)
  181. } else {
  182. me.Found(TokenWord)
  183. }
  184. return LexNormal
  185. }
  186. func LexSymbol(me *Lexer) LexerRule {
  187. me.SkipNotIn(" \t\r\n'({[]})")
  188. me.Found(TokenSymbol)
  189. return LexNormal
  190. }
  191. func LexNumber(me *Lexer) LexerRule {
  192. me.SkipNotIn(" \t\r\n'({[]})")
  193. me.Found(TokenNumber)
  194. return LexNormal
  195. }
  196. func LexWhitespace(me *Lexer) LexerRule {
  197. me.SkipWhitespace()
  198. me.Advance()
  199. return LexNormal
  200. }
  201. func LexComment(me *Lexer) LexerRule {
  202. if !me.SkipComment() {
  203. me.Error("Unterminated comment")
  204. return LexError
  205. }
  206. me.Advance()
  207. return LexNormal
  208. }
  209. func LexPunctuator(me *Lexer) LexerRule {
  210. me.Found(TokenType(me.Peek()))
  211. me.Advance()
  212. return LexNormal
  213. }
  214. func LexEOL(me *Lexer) LexerRule {
  215. me.SkipIn("\r\n")
  216. me.Found(TokenEOL)
  217. return LexNormal
  218. }
  219. func LexOperator(me *Lexer) LexerRule {
  220. me.SkipIn(operator_chars)
  221. me.Found(TokenOperator)
  222. return LexNormal
  223. }
  224. func lexEscape(me *Lexer) error {
  225. _ = me.Next()
  226. return nil
  227. }
  228. func LexString(me *Lexer) LexerRule {
  229. open := me.Peek()
  230. do_escape := open == '"'
  231. peek := me.Next()
  232. me.Advance()
  233. for ; peek != '\000'; peek = me.Next() {
  234. if do_escape && peek == '\\' {
  235. if err := lexEscape(me); err != nil {
  236. return LexError
  237. }
  238. } else if peek == open {
  239. me.Found(TokenString)
  240. _ = me.Next()
  241. me.Advance()
  242. return LexNormal
  243. }
  244. }
  245. me.Error("Unexpected EOF in string.")
  246. return nil
  247. }
  248. func LexNumberOrOperator(me *Lexer) LexerRule {
  249. if unicode.IsDigit(me.Next()) {
  250. return LexNumber
  251. } else {
  252. _ = me.Previous()
  253. return LexOperator
  254. }
  255. }
  256. func LexNormal(me *Lexer) LexerRule {
  257. peek := me.Peek()
  258. if peek == '#' {
  259. return LexComment
  260. } else if strings.ContainsRune(" \t", peek) {
  261. return LexWhitespace
  262. } else if strings.ContainsRune(".,;:", peek) {
  263. return LexPunctuator
  264. } else if strings.ContainsRune("([{}])", peek) {
  265. return LexSigil
  266. } else if strings.ContainsRune("$", peek) {
  267. return LexSymbol
  268. } else if strings.ContainsRune("\r\n", peek) {
  269. return LexEOL
  270. } else if strings.ContainsRune("+-", peek) {
  271. return LexNumberOrOperator
  272. } else if strings.ContainsRune("\"`", peek) {
  273. return LexString
  274. } else if peek == '\000' {
  275. me.Emit(TokenEOF, "")
  276. return nil
  277. } else if unicode.IsLetter(peek) {
  278. return LexWord
  279. } else if unicode.IsDigit(peek) {
  280. return LexNumber
  281. } else if strings.ContainsRune(operator_chars, peek) {
  282. return LexOperator
  283. } else {
  284. return LexError
  285. }
  286. }
  287. func OpenLexer(reader io.Reader) *Lexer {
  288. lexer := &Lexer{}
  289. lexer.Reader = reader
  290. lexer.Output = make(TokenChannel)
  291. // lexer.buffer = new(byte[1024])
  292. return lexer
  293. }
  294. func (me *Lexer) ReadReaderOnce() (bool, error) {
  295. buffer := make([]byte, 1024)
  296. n, err := me.Reader.Read(buffer)
  297. monolog.Debug("read %v %d %v\n", buffer[:n], n, err)
  298. if n > 0 {
  299. me.buffer = append(me.buffer, buffer[:n]...)
  300. monolog.Debug("append %s", me.buffer)
  301. }
  302. if err == io.EOF {
  303. return true, nil
  304. } else if err != nil {
  305. me.Error("Error reading from reader: %s", err)
  306. return true, err
  307. }
  308. return false, nil
  309. }
  310. func (me *Lexer) ReadReader() error {
  311. me.buffer = make([]byte, 0)
  312. more, err := me.ReadReaderOnce()
  313. for err == nil && more {
  314. more, err = me.ReadReaderOnce()
  315. }
  316. me.runes = bytes.Runes(me.buffer)
  317. return err
  318. }
  319. func (me *Lexer) Peek() rune {
  320. if (me.Current.Index) >= len(me.runes) {
  321. return '\000'
  322. }
  323. return me.runes[me.Current.Index]
  324. }
  325. func (me *Lexer) PeekNext() rune {
  326. if (me.Current.Index + 1) >= len(me.runes) {
  327. return '\000'
  328. }
  329. return me.runes[me.Current.Index+1]
  330. }
  331. func (me *Lexer) Next() rune {
  332. if me.Peek() == '\n' {
  333. me.Current.Column = 0
  334. me.Current.Row++
  335. }
  336. me.Current.Index++
  337. if me.Current.Index >= len(me.runes) {
  338. //me.Emit(TokenEOF, "")
  339. }
  340. return me.Peek()
  341. }
  342. func (me *Lexer) Previous() rune {
  343. if me.Current.Index > 0 {
  344. me.Current.Index--
  345. if me.Peek() == '\n' {
  346. me.Current.Column = 0
  347. me.Current.Row++
  348. }
  349. }
  350. return me.Peek()
  351. }
  352. func (me *Lexer) SkipRune() {
  353. _ = me.Next()
  354. }
  355. func (me *Lexer) SkipIn(set string) bool {
  356. for strings.ContainsRune(set, me.Next()) {
  357. monolog.Debug("SkipIn: %s %c\n", set, me.Peek())
  358. if me.Peek() == '\000' {
  359. return false
  360. }
  361. }
  362. return true
  363. }
  364. func (me *Lexer) SkipNotIn(set string) bool {
  365. _ = me.Next()
  366. for !strings.ContainsRune(set, me.Peek()) {
  367. if me.Next() == '\000' {
  368. return false
  369. }
  370. }
  371. return true
  372. }
  373. func (me *Lexer) SkipWhile(should_skip func(r rune) bool) bool {
  374. for should_skip(me.Peek()) {
  375. if me.Next() == '\000' {
  376. return false
  377. }
  378. }
  379. return true
  380. }
  381. func (me *Lexer) SkipWhitespace() {
  382. me.SkipIn(" \t")
  383. }
  384. func (me *Lexer) Advance() {
  385. me.Last = me.Current
  386. }
  387. func (me *Lexer) Rewind() {
  388. me.Current = me.Last
  389. }
  390. func (me *Lexer) CurrentRuneValue() []rune {
  391. return me.runes[me.Last.Index:me.Current.Index]
  392. }
  393. func (me *Lexer) CurrentStringValue() string {
  394. return string(me.CurrentRuneValue())
  395. }
  396. func (me *Lexer) Found(kind TokenType) {
  397. me.Emit(kind, Value(me.CurrentStringValue()))
  398. me.Advance()
  399. }
  400. func GetFunctionName(fun interface{}) string {
  401. return runtime.FuncForPC(reflect.ValueOf(fun).Pointer()).Name()
  402. }
  403. func (me *Lexer) Start() {
  404. if err := me.ReadReader(); err == nil || err == io.EOF {
  405. rule := LexNormal
  406. for rule != nil {
  407. monolog.Debug("Lexer Rule: %s\n", GetFunctionName(rule))
  408. rule = rule(me)
  409. }
  410. } else {
  411. me.Error("Could not read in input buffer: %s", err)
  412. }
  413. close(me.Output)
  414. }
  415. func (me *Lexer) TryLexing() {
  416. go me.Start()
  417. for token := range me.Output {
  418. monolog.Info("Token %s", token)
  419. }
  420. }
  421. type AstType int
  422. const (
  423. AstTypeProgram = AstType(iota)
  424. AstTypeStatements
  425. AstTypeStatement
  426. AstTypeDefinition
  427. AstTypeWords
  428. AstTypeExpression
  429. AstTypeWordExpression
  430. AstTypeWordCallop
  431. AstTypeOperation
  432. AstTypeOperations
  433. AstTypeCallArgs
  434. AstTypeValueExpression
  435. AstTypeValueCallop
  436. AstTypeParametersNonempty
  437. AstTypeParameters
  438. AstTypeParameter
  439. AstTypeBlock
  440. AstTypeWordValue
  441. AstTypeWord
  442. AstTypeValue
  443. AstTypeEox
  444. AstTypeOperator
  445. AstTypeParenthesis
  446. AstTypeModifier
  447. AstTypeError
  448. )
  449. var astTypeMap map[AstType]string = map[AstType]string{
  450. AstTypeProgram: "AstTypeProgram",
  451. AstTypeStatements: "AstTypeStatements",
  452. AstTypeStatement: "AstTypeStatement:",
  453. AstTypeDefinition: "AstTypeDefinition",
  454. AstTypeWords: "AstTypeWords",
  455. AstTypeExpression: "AstTypeExpression",
  456. AstTypeWordExpression: "AstTypeWordExpression",
  457. AstTypeWordCallop: "AstTypeWordCallop",
  458. AstTypeOperation: "AstTypeOperation",
  459. AstTypeOperations: "AstTypeOperations",
  460. AstTypeCallArgs: "AstTypeCallArgs",
  461. AstTypeValueExpression: "AstTypeValueExpression",
  462. AstTypeValueCallop: "AstTypeValueCallop",
  463. AstTypeParametersNonempty: "AstTypeParametersNonempty",
  464. AstTypeParameters: "AstTypeParameters",
  465. AstTypeParameter: "AstTypeParameter",
  466. AstTypeBlock: "AstTypeBlock",
  467. AstTypeWordValue: "AstTypeWordValue",
  468. AstTypeWord: "AstTypeWord",
  469. AstTypeValue: "AstTypeValue",
  470. AstTypeEox: "AstTypeEox",
  471. AstTypeOperator: "AstTypeOperator",
  472. AstTypeParenthesis: "AstTypeParenthesis",
  473. AstTypeModifier: "AstTypeModifier",
  474. AstTypeError: "AstTypeError",
  475. }
  476. func (me AstType) String() string {
  477. name, found := astTypeMap[me]
  478. if found {
  479. return name
  480. } else {
  481. return fmt.Sprintf("Unknown AstType %d", int(me))
  482. }
  483. }
  484. type Ast struct {
  485. tree.Node
  486. AstType
  487. *Token
  488. }
  489. func (me *Ast) NewChild(kind AstType, token *Token) *Ast {
  490. child := &Ast{}
  491. child.AstType = kind
  492. child.Token = token
  493. tree.AppendChild(me, child)
  494. return child
  495. }
  496. func (me *Ast) Walk(walker func(ast *Ast) *Ast) *Ast {
  497. node_res := tree.Walk(me,
  498. func(node tree.Noder) tree.Noder {
  499. ast_res := walker(node.(*Ast))
  500. if ast_res == nil {
  501. return nil
  502. } else {
  503. return ast_res
  504. }
  505. })
  506. if node_res != nil {
  507. return node_res.(*Ast)
  508. } else {
  509. return nil
  510. }
  511. }
  512. func (me *Ast) Remove() {
  513. _ = tree.Remove(me)
  514. }
  515. func NewAst(kind AstType) *Ast {
  516. ast := &Ast{}
  517. ast.AstType = kind
  518. ast.Token = nil
  519. return ast
  520. }
  521. type ParseAction func(parser *Parser) bool
  522. type RuleType int
  523. const (
  524. RuleTypeNone = RuleType(iota)
  525. RuleTypeAlternate
  526. RuleTypeSequence
  527. )
  528. type Rule struct {
  529. tree.Node
  530. Name string
  531. RuleType
  532. ParseAction
  533. }
  534. func NewRule(name string, ruty RuleType) *Rule {
  535. res := &Rule{}
  536. res.RuleType = ruty
  537. res.Name = name
  538. return res
  539. }
  540. func (me *Rule) NewChild(action ParseAction) *Rule {
  541. child := NewRule("foo", RuleTypeNone)
  542. tree.AppendChild(me, child)
  543. return child
  544. }
  545. func (me *Rule) Walk(walker func(rule *Rule) *Rule) *Rule {
  546. node_res := tree.Walk(me,
  547. func(node tree.Noder) tree.Noder {
  548. rule_res := walker(node.(*Rule))
  549. if rule_res == nil {
  550. return nil
  551. } else {
  552. return rule_res
  553. }
  554. })
  555. return node_res.(*Rule)
  556. }
  557. type Parser struct {
  558. *Ast
  559. *Lexer
  560. now *Ast
  561. lookahead *Token
  562. }
  563. func (me *Parser) SetupRules() {
  564. }
  565. func (me *Parser) Expect(types ...TokenType) bool {
  566. monolog.Debug("Expecting: ", types, " from ", me.now.AstType, " have ", me.LookaheadType(), " \n")
  567. for _, t := range types {
  568. if me.LookaheadType() == t {
  569. monolog.Debug("Found: ", t, "\n")
  570. return true
  571. }
  572. }
  573. monolog.Debug("Not found.\n")
  574. return false
  575. }
  576. type Parsable interface {
  577. isParsable()
  578. }
  579. func (me TokenType) isParsable() {
  580. }
  581. func (me ParseAction) isParsable() {
  582. }
  583. /* Advance the lexer but only of there is no lookahead token already available in me.lookahead.
  584. */
  585. func (me *Parser) Advance() *Token {
  586. if me.lookahead == nil {
  587. me.lookahead = <-me.Lexer.Output
  588. }
  589. return me.lookahead
  590. }
  591. func (me *Parser) DropLookahead() {
  592. me.lookahead = nil
  593. }
  594. func (me *Parser) Lookahead() *Token {
  595. return me.lookahead
  596. }
  597. func (me *Parser) LookaheadType() TokenType {
  598. if me.lookahead == nil {
  599. return TokenError
  600. }
  601. return me.Lookahead().TokenType
  602. }
  603. func (me *Parser) Consume(atyp AstType, types ...TokenType) bool {
  604. me.Advance()
  605. res := me.Expect(types...)
  606. if res {
  607. me.NewAstChild(atyp)
  608. me.DropLookahead()
  609. }
  610. return res
  611. }
  612. func (me *Parser) ConsumeWithoutAst(types ...TokenType) bool {
  613. me.Advance()
  614. res := me.Expect(types...)
  615. if res {
  616. me.DropLookahead()
  617. }
  618. return res
  619. }
  620. /*
  621. func (me * Parser) OneOf(restype AstType, options ...Parsable) bool {
  622. res := false
  623. k, v := range options {
  624. switch option := v.Type {
  625. case TokenType: res := Consume(restype, option)
  626. case ParseAction: res := option(me)
  627. }
  628. }
  629. return res
  630. }
  631. */
  632. func (me *Parser) ParseEOX() bool {
  633. return me.ConsumeWithoutAst(TokenEOL, TokenPeriod)
  634. }
  635. func (me *Parser) ParseValue() bool {
  636. return me.Consume(AstTypeValue, TokenString, TokenNumber, TokenSymbol)
  637. }
  638. func (me *Parser) ParseWord() bool {
  639. return me.Consume(AstTypeWord, TokenWord, TokenKeywordA, TokenKeywordThe)
  640. }
  641. func (me *Parser) ParseWordValue() bool {
  642. me.NewAstChildDescend(AstTypeWordValue)
  643. res := me.ParseValue() || me.ParseWord()
  644. me.AstAscend(res)
  645. return res
  646. }
  647. func (me *Parser) ParseParametersNonempty() bool {
  648. res := false
  649. for me.ParseParameter() {
  650. res = true
  651. }
  652. return res
  653. }
  654. func (me *Parser) ParseCallArgs() bool {
  655. me.NewAstChildDescend(AstTypeCallArgs)
  656. res := me.ParseParameters() && me.ParseEOX()
  657. me.AstAscend(res)
  658. return res
  659. }
  660. func (me *Parser) ParseOperator() bool {
  661. return me.Consume(AstTypeOperator, TokenOperator)
  662. }
  663. func (me *Parser) NewAstChild(tyty AstType) *Ast {
  664. return me.now.NewChild(tyty, me.lookahead)
  665. }
  666. func (me *Parser) NewAstChildDescend(tyty AstType) {
  667. node := me.NewAstChild(tyty)
  668. me.now = node
  669. }
  670. func (me *Parser) AstAscend(keep bool) {
  671. if me.now.Parent() != nil {
  672. now := me.now
  673. me.now = now.Parent().(*Ast)
  674. if !keep {
  675. now.Remove()
  676. }
  677. }
  678. }
  679. func (me TokenType) BlockCloseForOpen() (TokenType, bool) {
  680. switch me {
  681. case TokenOpenBrace:
  682. return TokenCloseBrace, true
  683. case TokenKeywordDo:
  684. return TokenKeywordEnd, true
  685. default:
  686. return TokenError, false
  687. }
  688. }
  689. func (me TokenType) ParenthesisCloseForOpen() (TokenType, bool) {
  690. switch me {
  691. case TokenOpenBracket:
  692. return TokenCloseBracket, true
  693. case TokenOpenParen:
  694. return TokenCloseParen, true
  695. default:
  696. return TokenError, false
  697. }
  698. }
  699. func (me *Parser) ParseBlock() bool {
  700. me.Advance()
  701. open := me.LookaheadType()
  702. done, ok := open.BlockCloseForOpen()
  703. if !ok {
  704. /* Not an opening of a block, so no block found. */
  705. return false
  706. }
  707. me.DropLookahead()
  708. me.NewAstChildDescend(AstTypeBlock)
  709. res := me.ParseStatements()
  710. me.AstAscend(res)
  711. if res {
  712. me.Advance()
  713. if me.LookaheadType() != done {
  714. return me.ParseError()
  715. }
  716. me.DropLookahead()
  717. }
  718. return res
  719. }
  720. func (me *Parser) ParseParenthesis() bool {
  721. me.Advance()
  722. open := me.LookaheadType()
  723. done, ok := open.ParenthesisCloseForOpen()
  724. if !ok {
  725. /* Not an opening of a parenthesis, so no parenthesis found. */
  726. return false
  727. }
  728. me.DropLookahead()
  729. me.NewAstChildDescend(AstTypeParenthesis)
  730. res := me.ParseExpression()
  731. me.AstAscend(res)
  732. if res {
  733. me.Advance()
  734. if me.LookaheadType() != done {
  735. return me.ParseError()
  736. }
  737. me.DropLookahead()
  738. }
  739. return res
  740. }
  741. func (me *Parser) ParseWords() bool {
  742. me.NewAstChildDescend(AstTypeWords)
  743. res := me.ParseWord()
  744. for me.ParseWord() {
  745. }
  746. me.AstAscend(res)
  747. return res
  748. }
  749. func (me *Parser) ParseDefinition() bool {
  750. me.Advance()
  751. res := me.Consume(AstTypeDefinition, TokenKeywordDef)
  752. if !res {
  753. return false
  754. }
  755. res = res && (me.ParseWord() || me.ParseOperator())
  756. if !res {
  757. _ = me.ParseError()
  758. }
  759. res = res && me.ParseParametersNonempty()
  760. if !res {
  761. _ = me.ParseError()
  762. }
  763. me.AstAscend(res)
  764. return res
  765. }
  766. func (me *Parser) ParseParameter() bool {
  767. me.NewAstChildDescend(AstTypeParameter)
  768. res := me.ParseWordValue() || me.ParseOperator() ||
  769. me.ParseParenthesis() || me.ParseBlock()
  770. me.AstAscend(res)
  771. return res
  772. }
  773. func (me *Parser) ParseParameters() bool {
  774. for me.ParseParameter() {
  775. }
  776. return true
  777. }
  778. func (me *Parser) ParseError() bool {
  779. me.now.NewChild(AstTypeError, me.lookahead)
  780. fmt.Printf("Parse error: at %s\n", me.lookahead)
  781. return false
  782. }
  783. func (me *Parser) ParseExpression() bool {
  784. return (me.ParseWordValue() || me.ParseOperator()) && me.ParseParameters()
  785. }
  786. func (me *Parser) ParseStatement() bool {
  787. me.NewAstChildDescend(AstTypeStatement)
  788. /* First case is for an empty expression/statement. */
  789. res := me.ParseEOX() ||
  790. me.ParseDefinition() ||
  791. (me.ParseExpression() && me.ParseEOX()) ||
  792. me.ParseBlock()
  793. me.AstAscend(res)
  794. return res
  795. }
  796. func (me *Parser) ParseEOF() bool {
  797. return me.Consume(AstTypeEox, TokenEOF)
  798. }
  799. func (me *Parser) ParseStatements() bool {
  800. me.NewAstChildDescend(AstTypeStatements)
  801. res := me.ParseStatement()
  802. for me.ParseStatement() {
  803. }
  804. me.AstAscend(res)
  805. return res
  806. }
  807. func (me *Parser) ParseProgram() bool {
  808. return me.ParseStatements() && me.ParseEOF()
  809. }
  810. func NewParserForLexer(lexer *Lexer) *Parser {
  811. me := &Parser{}
  812. me.Ast = NewAst(AstTypeProgram)
  813. me.now = me.Ast
  814. me.Lexer = lexer
  815. me.Ast.Token = &Token{}
  816. go me.Lexer.Start()
  817. return me
  818. }
  819. func NewParserForText(text string) *Parser {
  820. lexer := OpenLexer(strings.NewReader(text))
  821. return NewParserForLexer(lexer)
  822. }
  823. func (me *Ast) DotID() string {
  824. return fmt.Sprintf("ast_%p", me)
  825. }
  826. func (me *Ast) Dotty() {
  827. g := graphviz.NewDigraph("rankdir", "LR")
  828. me.Walk(func(ast *Ast) *Ast {
  829. label := ast.AstType.String()
  830. if ast.Token != nil {
  831. label = label + "\n" + ast.Token.String()
  832. }
  833. g.AddNode(ast.DotID(), "label", label)
  834. if ast.Parent() != nil {
  835. g.AddEdgeByName(ast.Parent().(*Ast).DotID(), ast.DotID())
  836. }
  837. return nil
  838. })
  839. g.Dotty()
  840. }
  841. /*
  842. PROGRAM -> STATEMENTS.
  843. STATEMENTS -> STATEMENT STATEMENTS | .
  844. STATEMENT -> EXPRESSION EOX | DEFINITION | BLOCK .
  845. DEFINITION -> define WORDOP WORDOPS BLOCK.
  846. WORDOPS -> WORDOP WORDOPS | .
  847. EXPRESSION -> WORDVALUE PARAMETERSS.
  848. PARAMETERS -> PARAMETER PARAMETERS | .
  849. PARAMETER -> WORDVALUE | PARENTHESIS | BLOCK | operator.
  850. PARENTHESIS -> '(' EXPRESSION ')' | ot EXPRESSION ct.
  851. BLOCK -> oe STATEMENTS ce | do STATEMENTS end .
  852. WORDOP -> word | operator | a | the.
  853. WORDVALUE -> word | VALUE | a | the.
  854. VALUE -> string | number | symbol.
  855. EOX -> eol | period.
  856. )
  857. */
  858. type DefineType int
  859. const (
  860. DefineTypeNone = DefineType(iota)
  861. DefineTypeGo
  862. DefineTypeUser
  863. DefineTypeVar
  864. )
  865. type Var interface {
  866. }
  867. type DefinePattern struct {
  868. Parts []string
  869. }
  870. type GoDefineFunc func(runtime Runtime, args ...Var) Var
  871. type UserDefine struct {
  872. DefinePattern
  873. *Ast
  874. }
  875. type GoDefine struct {
  876. DefinePattern
  877. *GoDefineFunc
  878. }
  879. type Define struct {
  880. DefineType
  881. definition *Ast
  882. }
  883. type Environment struct {
  884. Parent *Environment
  885. }
  886. type Runtime struct {
  887. Environment
  888. }
  889. func main() {
  890. fmt.Println("Hello World!")
  891. }