raku.go 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002
  1. // raku
  2. /* Raku is an easy to use scripting language that can also be used easily interactively
  3. Desrired syntax (verified LL(1) on smlweb.cpsc.ucalgary.ca)
  4. PROGRAM -> STATEMENTS.
  5. STATEMENTS -> STATEMENT STATEMENTS | .
  6. STATEMENT -> EXPRESSION EOX | DEFINITION | BLOCK | EOX .
  7. DEFINITION -> define WORDS BLOCK.
  8. WORDS -> word WORDS | .
  9. EXPRESSION -> WORDVALUE MODIFIERS.
  10. MODIFIERS -> MODIFIER MODIFIERS | .
  11. OPERATION -> operator MODIFIER .
  12. MODIFIER -> OPERATION | WORDVALUE | PARENTHESIS | BLOCK.
  13. PARENTHESIS -> '(' EXPRESSION ')' | ot EXPRESSION ct.
  14. BLOCK -> oe STATEMENTS ce | do STATEMENTS end .
  15. WORDVALUE -> word | VALUE | a | the.
  16. VALUE -> string | number | symbol.
  17. EOX -> eol | period.
  18. Lexer:
  19. */
  20. package raku
  21. import (
  22. "bytes"
  23. "fmt"
  24. "io"
  25. "reflect"
  26. "runtime"
  27. "strings"
  28. "unicode"
  29. "github.com/beoran/woe/graphviz"
  30. "github.com/beoran/woe/monolog"
  31. "github.com/beoran/woe/tree"
  32. )
  33. type Value string
  34. type TokenType int64
  35. type Position struct {
  36. Index int
  37. Row int
  38. Column int
  39. }
  40. const (
  41. TokenPeriod TokenType = TokenType('.')
  42. TokenComma TokenType = TokenType(',')
  43. TokenSemicolon TokenType = TokenType(';')
  44. TokenColon TokenType = TokenType(':')
  45. TokenOpenParen TokenType = TokenType('(')
  46. TokenCloseParen TokenType = TokenType(')')
  47. TokenOpenBrace TokenType = TokenType('{')
  48. TokenCloseBrace TokenType = TokenType('}')
  49. TokenOpenBracket TokenType = TokenType('[')
  50. TokenCloseBracket TokenType = TokenType(']')
  51. TokenNone TokenType = 0
  52. TokenError TokenType = -1
  53. TokenWord TokenType = -2
  54. TokenEOL TokenType = -3
  55. TokenEOF TokenType = -4
  56. TokenNumber TokenType = -5
  57. TokenOperator TokenType = -6
  58. TokenString TokenType = -7
  59. TokenSymbol TokenType = -8
  60. TokenFirstKeyword TokenType = -9
  61. TokenKeywordA TokenType = -10
  62. TokenKeywordDo TokenType = -11
  63. TokenKeywordEnd TokenType = -12
  64. TokenKeywordThe TokenType = -13
  65. TokenKeywordTo TokenType = -14
  66. TokenLastKeyword TokenType = -15
  67. TokenLast TokenType = -15
  68. )
  69. type Token struct {
  70. TokenType
  71. Value
  72. Position
  73. }
  74. var tokenTypeMap map[TokenType]string = map[TokenType]string{
  75. TokenNone: "TokenNone",
  76. TokenError: "TokenError",
  77. TokenWord: "TokenWord",
  78. TokenEOL: "TokenEOL",
  79. TokenEOF: "TokenEOF",
  80. TokenNumber: "TokenNumber",
  81. TokenOperator: "TokenOperator",
  82. TokenString: "TokenString",
  83. TokenSymbol: "TokenSymbol",
  84. TokenKeywordA: "TokenKeywordA",
  85. TokenKeywordDo: "TokenKeywordDo",
  86. TokenKeywordEnd: "TokenKeywordEnd",
  87. TokenKeywordThe: "TokenKeywordThe",
  88. TokenKeywordTo: "TokenKeywordTo",
  89. }
  90. var keywordMap map[string]TokenType = map[string]TokenType{
  91. "a": TokenKeywordA,
  92. "an": TokenKeywordA,
  93. "do": TokenKeywordDo,
  94. "end": TokenKeywordEnd,
  95. "the": TokenKeywordThe,
  96. "to": TokenKeywordTo,
  97. }
  98. var sigilMap map[string]TokenType = map[string]TokenType{
  99. "[": TokenOpenBracket,
  100. "{": TokenOpenBrace,
  101. "(": TokenOpenParen,
  102. "]": TokenCloseBracket,
  103. "}": TokenCloseBrace,
  104. ")": TokenCloseParen,
  105. }
  106. func (me TokenType) String() string {
  107. name, found := tokenTypeMap[me]
  108. if found {
  109. return name
  110. } else {
  111. if (me > 0) && (me < 256) {
  112. return fmt.Sprintf("TokenChar<%c>", byte(me))
  113. }
  114. return fmt.Sprintf("Unknown Token %d", int(me))
  115. }
  116. }
  117. func (me Token) String() string {
  118. return fmt.Sprintf("Token: %s >%s< %d %d %d.", me.TokenType, string(me.Value), me.Index, me.Row, me.Column)
  119. }
  120. type TokenChannel chan *Token
  121. type Lexer struct {
  122. Reader io.Reader
  123. Current Position
  124. Last Position
  125. Token Token
  126. rule LexerRule
  127. Output TokenChannel
  128. buffer []byte
  129. runes []rune
  130. }
  131. type LexerRule func(lexer *Lexer) LexerRule
  132. func (me *Lexer) Emit(t TokenType, v Value) {
  133. tok := &Token{t, v, me.Current}
  134. me.Output <- tok
  135. }
  136. func (me *Lexer) Error(message string, args ...interface{}) {
  137. value := fmt.Sprintf(message, args...)
  138. monolog.Error("Lex Error: %s", value)
  139. me.Emit(TokenError, Value(value))
  140. }
  141. func LexError(me *Lexer) LexerRule {
  142. me.Error("Error")
  143. return nil
  144. }
  145. func (me *Lexer) SkipComment() bool {
  146. if me.Peek() == '#' {
  147. if me.Next() == '(' {
  148. return me.SkipNotIn(")")
  149. } else {
  150. return me.SkipNotIn("\r\n")
  151. }
  152. }
  153. return true
  154. }
  155. /* Returns whether or not a keyword was found, and if so, the TokenType
  156. of the keyword.*/
  157. func LookupKeyword(word string) (bool, TokenType) {
  158. kind, found := keywordMap[word]
  159. return found, kind
  160. }
  161. /* Returns whether or not a special operator or sigil was found, and if so,
  162. returns the TokenTyp of the sigil.*/
  163. func LookupSigil(sigil string) (bool, TokenType) {
  164. fmt.Printf("LookupSigil: %s\n", sigil)
  165. kind, found := sigilMap[sigil]
  166. return found, kind
  167. }
  168. func LexWord(me *Lexer) LexerRule {
  169. me.SkipNotIn(" \t\r\n'")
  170. iskw, kind := LookupKeyword(me.CurrentStringValue())
  171. if iskw {
  172. me.Found(kind)
  173. } else {
  174. me.Found(TokenWord)
  175. }
  176. return LexNormal
  177. }
  178. func LexSymbol(me *Lexer) LexerRule {
  179. me.SkipNotIn(" \t\r\n'")
  180. me.Found(TokenSymbol)
  181. return LexNormal
  182. }
  183. func LexNumber(me *Lexer) LexerRule {
  184. me.SkipNotIn(" \tBBBT\r\n")
  185. me.Found(TokenNumber)
  186. return LexNormal
  187. }
  188. func LexWhitespace(me *Lexer) LexerRule {
  189. me.SkipWhitespace()
  190. me.Advance()
  191. return LexNormal
  192. }
  193. func LexComment(me *Lexer) LexerRule {
  194. if !me.SkipComment() {
  195. me.Error("Unterminated comment")
  196. return LexError
  197. }
  198. me.Advance()
  199. return LexNormal
  200. }
  201. func LexPunctuator(me *Lexer) LexerRule {
  202. me.Found(TokenType(me.Peek()))
  203. return LexNormal
  204. }
  205. func LexEOL(me *Lexer) LexerRule {
  206. me.SkipIn("\r\n")
  207. me.Found(TokenEOL)
  208. return LexNormal
  209. }
  210. func LexOperator(me *Lexer) LexerRule {
  211. me.SkipNotIn(" \t\r\n")
  212. issig, kind := LookupSigil(me.CurrentStringValue())
  213. if issig {
  214. me.Found(kind)
  215. } else {
  216. me.Found(TokenOperator)
  217. }
  218. return LexNormal
  219. }
  220. func lexEscape(me *Lexer) error {
  221. _ = me.Next()
  222. return nil
  223. }
  224. func LexString(me *Lexer) LexerRule {
  225. open := me.Peek()
  226. do_escape := open == '"'
  227. peek := me.Next()
  228. me.Advance()
  229. for ; peek != '\000'; peek = me.Next() {
  230. if do_escape && peek == '\\' {
  231. if err := lexEscape(me); err != nil {
  232. return LexError
  233. }
  234. } else if peek == open {
  235. me.Found(TokenString)
  236. _ = me.Next()
  237. me.Advance()
  238. return LexNormal
  239. }
  240. }
  241. me.Error("Unexpected EOF in string.")
  242. return nil
  243. }
  244. func LexNumberOrOperator(me *Lexer) LexerRule {
  245. if unicode.IsDigit(me.Next()) {
  246. return LexNumber
  247. } else {
  248. _ = me.Previous()
  249. return LexOperator
  250. }
  251. }
  252. func LexNormal(me *Lexer) LexerRule {
  253. peek := me.Peek()
  254. if peek == '#' {
  255. return LexComment
  256. } else if strings.ContainsRune(" \t", peek) {
  257. return LexWhitespace
  258. } else if strings.ContainsRune(".,;:", peek) {
  259. return LexPunctuator
  260. } else if strings.ContainsRune("$", peek) {
  261. return LexSymbol
  262. } else if strings.ContainsRune("\r\n", peek) {
  263. return LexEOL
  264. } else if strings.ContainsRune("+-", peek) {
  265. return LexNumberOrOperator
  266. } else if strings.ContainsRune("\"`", peek) {
  267. return LexString
  268. } else if peek == '\000' {
  269. me.Emit(TokenEOF, "")
  270. return nil
  271. } else if unicode.IsLetter(peek) {
  272. return LexWord
  273. } else if unicode.IsDigit(peek) {
  274. return LexNumber
  275. } else {
  276. return LexOperator
  277. }
  278. }
  279. func OpenLexer(reader io.Reader) *Lexer {
  280. lexer := &Lexer{}
  281. lexer.Reader = reader
  282. lexer.Output = make(TokenChannel)
  283. // lexer.buffer = new(byte[1024])
  284. return lexer
  285. }
  286. func (me *Lexer) ReadReaderOnce() (bool, error) {
  287. buffer := make([]byte, 1024)
  288. n, err := me.Reader.Read(buffer)
  289. monolog.Debug("read %v %d %v\n", buffer[:n], n, err)
  290. if n > 0 {
  291. me.buffer = append(me.buffer, buffer[:n]...)
  292. monolog.Debug("append %s", me.buffer)
  293. }
  294. if err == io.EOF {
  295. return true, nil
  296. } else if err != nil {
  297. me.Error("Error reading from reader: %s", err)
  298. return true, err
  299. }
  300. return false, nil
  301. }
  302. func (me *Lexer) ReadReader() error {
  303. me.buffer = make([]byte, 0)
  304. more, err := me.ReadReaderOnce()
  305. for err == nil && more {
  306. more, err = me.ReadReaderOnce()
  307. }
  308. me.runes = bytes.Runes(me.buffer)
  309. return err
  310. }
  311. func (me *Lexer) Peek() rune {
  312. if (me.Current.Index) >= len(me.runes) {
  313. return '\000'
  314. }
  315. return me.runes[me.Current.Index]
  316. }
  317. func (me *Lexer) PeekNext() rune {
  318. if (me.Current.Index + 1) >= len(me.runes) {
  319. return '\000'
  320. }
  321. return me.runes[me.Current.Index+1]
  322. }
  323. func (me *Lexer) Next() rune {
  324. if me.Peek() == '\n' {
  325. me.Current.Column = 0
  326. me.Current.Row++
  327. }
  328. me.Current.Index++
  329. if me.Current.Index >= len(me.runes) {
  330. //me.Emit(TokenEOF, "")
  331. }
  332. return me.Peek()
  333. }
  334. func (me *Lexer) Previous() rune {
  335. if me.Current.Index > 0 {
  336. me.Current.Index--
  337. if me.Peek() == '\n' {
  338. me.Current.Column = 0
  339. me.Current.Row++
  340. }
  341. }
  342. return me.Peek()
  343. }
  344. func (me *Lexer) SkipRune() {
  345. _ = me.Next()
  346. }
  347. func (me *Lexer) SkipIn(set string) bool {
  348. for strings.ContainsRune(set, me.Next()) {
  349. monolog.Debug("SkipIn: %s %c\n", set, me.Peek())
  350. if me.Peek() == '\000' {
  351. return false
  352. }
  353. }
  354. return true
  355. }
  356. func (me *Lexer) SkipNotIn(set string) bool {
  357. _ = me.Next()
  358. for !strings.ContainsRune(set, me.Peek()) {
  359. if me.Next() == '\000' {
  360. return false
  361. }
  362. }
  363. return true
  364. }
  365. func (me *Lexer) SkipWhile(should_skip func(r rune) bool) bool {
  366. for should_skip(me.Peek()) {
  367. if me.Next() == '\000' {
  368. return false
  369. }
  370. }
  371. return true
  372. }
  373. func (me *Lexer) SkipWhitespace() {
  374. me.SkipIn(" \t")
  375. }
  376. func (me *Lexer) Advance() {
  377. me.Last = me.Current
  378. }
  379. func (me *Lexer) Rewind() {
  380. me.Current = me.Last
  381. }
  382. func (me *Lexer) CurrentRuneValue() []rune {
  383. return me.runes[me.Last.Index:me.Current.Index]
  384. }
  385. func (me *Lexer) CurrentStringValue() string {
  386. return string(me.CurrentRuneValue())
  387. }
  388. func (me *Lexer) Found(kind TokenType) {
  389. me.Emit(kind, Value(me.CurrentStringValue()))
  390. me.Advance()
  391. }
  392. func GetFunctionName(fun interface{}) string {
  393. return runtime.FuncForPC(reflect.ValueOf(fun).Pointer()).Name()
  394. }
  395. func (me *Lexer) Start() {
  396. if err := me.ReadReader(); err == nil || err == io.EOF {
  397. rule := LexNormal
  398. for rule != nil {
  399. monolog.Debug("Lexer Rule: %s\n", GetFunctionName(rule))
  400. rule = rule(me)
  401. }
  402. } else {
  403. me.Error("Could not read in input buffer: %s", err)
  404. }
  405. close(me.Output)
  406. }
  407. func (me *Lexer) TryLexing() {
  408. go me.Start()
  409. for token := range me.Output {
  410. monolog.Info("Token %s", token)
  411. }
  412. }
  413. type AstType int
  414. const (
  415. AstTypeProgram = AstType(iota)
  416. AstTypeStatements
  417. AstTypeStatement
  418. AstTypeDefinition
  419. AstTypeWords
  420. AstTypeExpression
  421. AstTypeWordExpression
  422. AstTypeWordCallop
  423. AstTypeOperation
  424. AstTypeOperations
  425. AstTypeCallArgs
  426. AstTypeValueExpression
  427. AstTypeValueCallop
  428. AstTypeParametersNonempty
  429. AstTypeParameters
  430. AstTypeParameter
  431. AstTypeBlock
  432. AstTypeWordValue
  433. AstTypeWord
  434. AstTypeValue
  435. AstTypeEox
  436. AstTypeOperator
  437. AstTypeError
  438. )
  439. var astTypeMap map[AstType]string = map[AstType]string{
  440. AstTypeProgram: "AstTypeProgram",
  441. AstTypeStatements: "AstTypeStatements",
  442. AstTypeStatement: "AstTypeStatement:",
  443. AstTypeDefinition: "AstTypeDefinition",
  444. AstTypeWords: "AstTypeWords",
  445. AstTypeExpression: "AstTypeExpression",
  446. AstTypeWordExpression: "AstTypeWordExpression",
  447. AstTypeWordCallop: "AstTypeWordCallop",
  448. AstTypeOperation: "AstTypeOperation",
  449. AstTypeOperations: "AstTypeOperations",
  450. AstTypeCallArgs: "AstTypeCallArgs",
  451. AstTypeValueExpression: "AstTypeValueExpression",
  452. AstTypeValueCallop: "AstTypeValueCallop",
  453. AstTypeParametersNonempty: "AstTypeParametersNonempty",
  454. AstTypeParameters: "AstTypeParameters",
  455. AstTypeParameter: "AstTypeParameter",
  456. AstTypeBlock: "AstTypeBlock",
  457. AstTypeWordValue: "AstTypeWordValue",
  458. AstTypeWord: "AstTypeWord",
  459. AstTypeValue: "AstTypeValue",
  460. AstTypeEox: "AstTypeEox",
  461. AstTypeOperator: "AstTypeOperator",
  462. AstTypeError: "AstTypeError",
  463. }
  464. func (me AstType) String() string {
  465. name, found := astTypeMap[me]
  466. if found {
  467. return name
  468. } else {
  469. return fmt.Sprintf("Unknown AstType %d", int(me))
  470. }
  471. }
  472. type Ast struct {
  473. tree.Node
  474. AstType
  475. *Token
  476. }
  477. func (me *Ast) NewChild(kind AstType, token *Token) *Ast {
  478. child := &Ast{}
  479. child.AstType = kind
  480. child.Token = token
  481. tree.AppendChild(me, child)
  482. return child
  483. }
  484. func (me *Ast) Walk(walker func(ast *Ast) *Ast) *Ast {
  485. node_res := tree.Walk(me,
  486. func(node tree.Noder) tree.Noder {
  487. ast_res := walker(node.(*Ast))
  488. if ast_res == nil {
  489. return nil
  490. } else {
  491. return ast_res
  492. }
  493. })
  494. if node_res != nil {
  495. return node_res.(*Ast)
  496. } else {
  497. return nil
  498. }
  499. }
  500. func (me *Ast) Remove() {
  501. _ = tree.Remove(me)
  502. }
  503. func NewAst(kind AstType) *Ast {
  504. ast := &Ast{}
  505. ast.AstType = kind
  506. ast.Token = nil
  507. return ast
  508. }
  509. type ParseAction func(parser *Parser) bool
  510. type RuleType int
  511. const (
  512. RuleTypeNone = RuleType(iota)
  513. RuleTypeAlternate
  514. RuleTypeSequence
  515. )
  516. type Rule struct {
  517. tree.Node
  518. Name string
  519. RuleType
  520. ParseAction
  521. }
  522. func NewRule(name string, ruty RuleType) *Rule {
  523. res := &Rule{}
  524. res.RuleType = ruty
  525. res.Name = name
  526. return res
  527. }
  528. func (me *Rule) NewChild(action ParseAction) *Rule {
  529. child := NewRule("foo", RuleTypeNone)
  530. tree.AppendChild(me, child)
  531. return child
  532. }
  533. func (me *Rule) Walk(walker func(rule *Rule) *Rule) *Rule {
  534. node_res := tree.Walk(me,
  535. func(node tree.Noder) tree.Noder {
  536. rule_res := walker(node.(*Rule))
  537. if rule_res == nil {
  538. return nil
  539. } else {
  540. return rule_res
  541. }
  542. })
  543. return node_res.(*Rule)
  544. }
  545. type Parser struct {
  546. *Ast
  547. *Lexer
  548. now *Ast
  549. lookahead *Token
  550. }
  551. func (me *Parser) SetupRules() {
  552. }
  553. func (me *Parser) Expect(types ...TokenType) bool {
  554. fmt.Print("Expecting: ", types, " from ", me.now.AstType, " have ", me.LookaheadType(), " \n")
  555. for _, t := range types {
  556. if me.LookaheadType() == t {
  557. fmt.Print("Found: ", t, "\n")
  558. return true
  559. }
  560. }
  561. fmt.Print("Not found.\n")
  562. return false
  563. }
  564. type Parsable interface {
  565. isParsable()
  566. }
  567. func (me TokenType) isParsable() {
  568. }
  569. func (me ParseAction) isParsable() {
  570. }
  571. /* Advance the lexer but only of there is no lookahead token already available in me.lookahead.
  572. */
  573. func (me *Parser) Advance() *Token {
  574. if me.lookahead == nil {
  575. me.lookahead = <-me.Lexer.Output
  576. }
  577. return me.lookahead
  578. }
  579. func (me *Parser) DropLookahead() {
  580. me.lookahead = nil
  581. }
  582. func (me *Parser) Lookahead() *Token {
  583. return me.lookahead
  584. }
  585. func (me *Parser) LookaheadType() TokenType {
  586. if me.lookahead == nil {
  587. return TokenError
  588. }
  589. return me.Lookahead().TokenType
  590. }
  591. func (me *Parser) Consume(atyp AstType, types ...TokenType) bool {
  592. me.Advance()
  593. res := me.Expect(types...)
  594. if res {
  595. me.NewAstChild(atyp)
  596. me.DropLookahead()
  597. }
  598. return res
  599. }
  600. func (me *Parser) ConsumeWithoutAst(types ...TokenType) bool {
  601. me.Advance()
  602. res := me.Expect(types...)
  603. if res {
  604. me.DropLookahead()
  605. }
  606. return res
  607. }
  608. /*
  609. func (me * Parser) OneOf(restype AstType, options ...Parsable) bool {
  610. res := false
  611. k, v := range options {
  612. switch option := v.Type {
  613. case TokenType: res := Consume(restype, option)
  614. case ParseAction: res := option(me)
  615. }
  616. }
  617. return res
  618. }
  619. */
  620. func (me *Parser) ParseEOX() bool {
  621. return me.ConsumeWithoutAst(TokenEOL, TokenPeriod)
  622. }
  623. func (me *Parser) ParseValue() bool {
  624. return me.Consume(AstTypeValue, TokenString, TokenNumber, TokenSymbol)
  625. }
  626. func (me *Parser) ParseWord() bool {
  627. return me.Consume(AstTypeWord, TokenWord, TokenKeywordA, TokenKeywordThe)
  628. }
  629. func (me *Parser) ParseWordValue() bool {
  630. me.NewAstChildDescend(AstTypeWordValue)
  631. res := me.ParseValue() || me.ParseWord()
  632. me.AstAscend(res)
  633. return res
  634. }
  635. func (me *Parser) ParseParameter() bool {
  636. me.NewAstChildDescend(AstTypeParameter)
  637. res := me.ParseWordValue() || me.ParseBlock()
  638. me.AstAscend(res)
  639. return res
  640. }
  641. func (me *Parser) ParseParametersNonempty() bool {
  642. res := false
  643. for me.ParseParameter() {
  644. res = true
  645. }
  646. return res
  647. }
  648. func (me *Parser) ParseParameters() bool {
  649. me.NewAstChildDescend(AstTypeParameters)
  650. _ = me.ParseParametersNonempty()
  651. me.AstAscend(true)
  652. return true
  653. }
  654. func (me *Parser) ParseCallArgs() bool {
  655. me.NewAstChildDescend(AstTypeCallArgs)
  656. res := me.ParseParameters() && me.ParseEOX()
  657. me.AstAscend(res)
  658. return res
  659. }
  660. func (me *Parser) ParseOperator() bool {
  661. return me.Consume(AstTypeOperator, TokenOperator)
  662. }
  663. func (me *Parser) ParseOperation() bool {
  664. me.NewAstChildDescend(AstTypeOperation)
  665. res := me.ParseOperator() && me.ParseParameter()
  666. me.AstAscend(res)
  667. return res
  668. }
  669. func (me *Parser) ParseOperations() bool {
  670. me.NewAstChildDescend(AstTypeOperations)
  671. res := me.ParseOperation()
  672. for me.ParseOperation() {
  673. }
  674. me.AstAscend(res)
  675. return res
  676. }
  677. func (me *Parser) ParseWordCallOp() bool {
  678. me.NewAstChildDescend(AstTypeWordCallop)
  679. res := me.ParseCallArgs() || me.ParseOperations()
  680. me.AstAscend(res)
  681. return res
  682. }
  683. func (me *Parser) ParseWordExpression() bool {
  684. me.NewAstChildDescend(AstTypeWordExpression)
  685. res := me.ParseWord() && me.ParseWordCallOp()
  686. me.AstAscend(res)
  687. return res
  688. }
  689. func (me *Parser) ParseValueCallOp() bool {
  690. me.NewAstChildDescend(AstTypeValueCallop)
  691. res := me.ParseCallArgs() || me.ParseOperations()
  692. me.AstAscend(res)
  693. return res
  694. }
  695. func (me *Parser) ParseValueExpression() bool {
  696. me.NewAstChildDescend(AstTypeValueExpression)
  697. res := me.ParseValue() && me.ParseValueCallOp()
  698. me.AstAscend(res)
  699. return false
  700. }
  701. func (me *Parser) NewAstChild(tyty AstType) *Ast {
  702. return me.now.NewChild(tyty, me.lookahead)
  703. }
  704. func (me *Parser) NewAstChildDescend(tyty AstType) {
  705. node := me.NewAstChild(tyty)
  706. me.now = node
  707. }
  708. func (me *Parser) AstAscend(keep bool) {
  709. if me.now.Parent() != nil {
  710. now := me.now
  711. me.now = now.Parent().(*Ast)
  712. if !keep {
  713. now.Remove()
  714. }
  715. }
  716. }
  717. func (me TokenType) CloseForOpen() (TokenType, bool) {
  718. switch me {
  719. case TokenOpenBrace:
  720. return TokenCloseBrace, true
  721. case TokenOpenBracket:
  722. return TokenCloseBracket, true
  723. case TokenOpenParen:
  724. return TokenCloseParen, true
  725. case TokenKeywordDo:
  726. return TokenKeywordEnd, true
  727. default:
  728. return TokenError, false
  729. }
  730. }
  731. func (me *Parser) ParseBlock() bool {
  732. me.Advance()
  733. open := me.LookaheadType()
  734. done, ok := open.CloseForOpen()
  735. if !ok {
  736. /* Not an opening of a block, so no block found. */
  737. return false
  738. }
  739. me.DropLookahead()
  740. me.NewAstChildDescend(AstTypeBlock)
  741. res := me.ParseStatements()
  742. me.AstAscend(res)
  743. if res {
  744. me.Advance()
  745. if me.LookaheadType() != done {
  746. return me.ParseError()
  747. }
  748. me.DropLookahead()
  749. }
  750. return res
  751. }
  752. func (me *Parser) ParseWords() bool {
  753. me.NewAstChildDescend(AstTypeWords)
  754. res := me.ParseWord()
  755. for me.ParseWord() {
  756. }
  757. me.AstAscend(res)
  758. return res
  759. }
  760. func (me *Parser) ParseDefinition() bool {
  761. me.Advance()
  762. res := me.Consume(AstTypeDefinition, TokenKeywordTo)
  763. if !res {
  764. return false
  765. }
  766. res = res && me.ParseWords()
  767. if !res {
  768. _ = me.ParseError()
  769. }
  770. res = res && me.ParseBlock()
  771. if !res {
  772. _ = me.ParseError()
  773. }
  774. me.AstAscend(true)
  775. return res
  776. }
  777. func (me *Parser) ParseError() bool {
  778. me.now.NewChild(AstTypeError, me.lookahead)
  779. fmt.Printf("Parse error: at %s\n", me.lookahead)
  780. return false
  781. }
  782. func (me *Parser) ParseExpression() bool {
  783. return me.ParseWordExpression() || me.ParseValueExpression()
  784. }
  785. func (me *Parser) ParseStatement() bool {
  786. me.NewAstChildDescend(AstTypeStatement)
  787. /* First case is for an empty expression/statement. */
  788. res := me.ParseEOX() || me.ParseDefinition() || me.ParseExpression() || me.ParseBlock()
  789. me.AstAscend(res)
  790. return res
  791. }
  792. func (me *Parser) ParseEOF() bool {
  793. return me.Consume(AstTypeEox, TokenEOF)
  794. }
  795. func (me *Parser) ParseStatements() bool {
  796. me.NewAstChildDescend(AstTypeStatements)
  797. res := me.ParseStatement()
  798. for me.ParseStatement() {
  799. }
  800. me.AstAscend(res)
  801. return res
  802. }
  803. func (me *Parser) ParseProgram() bool {
  804. return me.ParseStatements() && me.ParseEOF()
  805. }
  806. func NewParserForLexer(lexer *Lexer) *Parser {
  807. me := &Parser{}
  808. me.Ast = NewAst(AstTypeProgram)
  809. me.now = me.Ast
  810. me.Lexer = lexer
  811. me.Ast.Token = &Token{}
  812. go me.Lexer.Start()
  813. return me
  814. }
  815. func NewParserForText(text string) *Parser {
  816. lexer := OpenLexer(strings.NewReader(text))
  817. return NewParserForLexer(lexer)
  818. }
  819. func (me *Ast) DotID() string {
  820. return fmt.Sprintf("ast_%p", me)
  821. }
  822. func (me *Ast) Dotty() {
  823. g := graphviz.NewDigraph("rankdir", "LR")
  824. me.Walk(func(ast *Ast) *Ast {
  825. label := ast.AstType.String()
  826. if ast.Token != nil {
  827. label = label + "\n" + ast.Token.String()
  828. }
  829. g.AddNode(ast.DotID(), "label", label)
  830. if ast.Parent() != nil {
  831. g.AddEdgeByName(ast.Parent().(*Ast).DotID(), ast.DotID())
  832. }
  833. return nil
  834. })
  835. g.Dotty()
  836. }
  837. /*
  838. PROGRAM -> STATEMENTS.
  839. STATEMENTS -> STATEMENT STATEMENTS | .
  840. STATEMENT -> DEFINITION | EXPRESSION | BLOCK .
  841. DEFINITION -> to WORDS BLOCK.
  842. WORDS -> word WORDS | .
  843. EXPRESSION -> WORD_EXPRESSION | VALUE_EXPRESSION.
  844. WORD_EXPRESSION -> word WORD_CALLOP.
  845. WORD_CALLOP -> WORD_OPERATION | WORD_CALL.
  846. OPERATION -> operator PARAMETERS_NONEMPTY EOX.
  847. WORD_CALL -> PARAMETERS EOX.
  848. VALUE_EXPRESSION -> value VALUE_CALLOP.
  849. VALUE_CALLOP -> VALUE_OPERATION | VALUE_CALL.
  850. VALUE_CALL -> EOX.
  851. PARAMETERS_NONEMPTY -> PARAMETER PARAMETERS.
  852. PARAMETERS -> PARAMETERS_NONEMPTY | .
  853. PARAMETER -> BLOCK | WORDVALUE | OPERATION.
  854. BLOCK -> ob STATEMENTS cb | op STATEMENTS cp | oa STATEMENTS ca | do STATEMENTS end.
  855. WORDVALUE -> word | VALUE.
  856. VALUE -> string | number | symbol.
  857. EOX -> eol | period.
  858. AstNodeBlock = AstNodeType(iota)
  859. )
  860. */
  861. type Environment struct {
  862. Parent *Environment
  863. }
  864. func main() {
  865. fmt.Println("Hello World!")
  866. }