Support operators though expression chains.

README.md

@@ -11,9 +11,42 @@ feel.
 
 ## Syntax
 
-A Muesli program consists of blocks and/or chains, separated by newlines or 
+A Muesli program consists of blocks and chains, separated by newlines or 
 periods. A chain consists of one or more commands, linked together by operators.
 
+- A muesli program consists of statements.
+- A statement is either a block, or a command chain terminated by a newline, 
+  or just an empty line .
+- A command chain consists of commands chained together with operators.
+  A command chain is evaluated by looking up and calling the the first operator 
+  in the chain, which receives the first command and the rest of the chain as 
+  callable  substitutions, which have not been substituted yet. The operator 
+  implementation can then choose to evaluate the substitutions as needed. 
+  For example: ls | grep "foo" | sort > fopen "foo"
+  is evaluated as: > (| (| (ls) (grep "foo")) (sort)) (fopen "foo")
+- A command may be a direct command, a substitution, or a literal value.
+- A direct command consists of a name followed by zero or more parameters.
+  A direct command is evaluated as follows: the name is looked up
+  in the VM and executed if found and callable, with the given parameters,
+  and evaluates to the return value to the callable.
+  An error is raised if the name of the direct command is not found.
+- A substitution starts with an open parenthesis, followed by a command chain,
+  followed by a close parenthesis. a substitution followed by zero or more
+  parameters.
+  A substitution is evaluated as the return value of the command chain
+  inside the parenthesis of the substitution.
+- A bracketed list [ elem1 elem2 ... ]  is syntactic sugar for (list
+elem1 elem 2)
+- A dollar getter $varname is syntactic sugar for (get varname)
+- A equals setter =varname value is syntactic sugar for (set varname value)
+- Therefore, parenthesis, lists, getters and setters are allowed anywhere,
+  also in the beginning of the command with substitution semantics.
+- A literal value evaluates to itself.
+- A name also evaluates to itself when it is a parameter,
+  but if it is at the beginning of a direct command, the name is looked up
+  in the VM and called in stead if found, or an error is raised if not found.
+
+
 Blocks consist of chains grouped together in blocks between braces {}. The 
 commands in a block are not executed immediately but stored.
 
@@ -25,7 +58,7 @@ Every command has a return value, which can be of any type, and not just a
 number like in most shells. Furthermore commands have input and output, which 
 default to stdin and stdout, but can be redirected or reconfigured. 
 
-There are three types of commands, namely direct commands indirect commmands
+There are three types of commands, namely direct commands, substitutions,
 and literal commands. A direct command starts with a name, and is followed by 
 any number of parameters. Parameters may be which may be blocks, parenthesis, 
 lists, names or literals, or values. An indirect command starts with a 

ast.go

@@ -12,6 +12,7 @@ type AstMetaKindNone AstBasicMetaKind
 type AstMetaKindProgram AstBasicMetaKind
 type AstMetaKindStatements AstBasicMetaKind
 type AstMetaKindStatement AstBasicMetaKind
+type AstMetaKindOperation AstBasicMetaKind
 type AstMetaKindClosed AstBasicMetaKind
 type AstMetaKindSet AstBasicMetaKind
 type AstMetaKindGet AstBasicMetaKind
@@ -40,26 +41,22 @@ const (
 	AstKindProgram     = AstMetaKindProgram("Program")
 	AstKindStatements  = AstMetaKindStatements("Statements")
 	AstKindStatement   = AstMetaKindStatement("Statement")
-    // AstKindClosed      = AstMetaKindClosed("Closed")
-	AstKindSet         = AstMetaKindSet("Set")
+    AstKindOperation   = AstMetaKindOperation("Operation")
+ 	AstKindSet         = AstMetaKindSet("Set")
 	AstKindGet         = AstMetaKindGet("Get")
 	AstKindTarget      = AstMetaKindTarget("Target")
 	AstKindCommand     = AstMetaKindCommand("Command")
 	AstKindArguments   = AstMetaKindArguments("Arguments")
-	// AstKindArgument    = AstMetaKindArgument("Argument")
-	// AstKindExpression  = AstMetaKindExpression("Expression")
 	AstKindBlock       = AstMetaKindBlock("Block")
 	AstKindParenthesis = AstMetaKindParenthesis("Parenthesis")
 	AstKindList        = AstMetaKindList("List")
-	// AstKindCapture     = AstMetaKindCapture("Capture")
 	AstKindWordValue   = AstMetaKindWordValue("WordValue")
 	AstKindWord        = AstMetaKindWord("Word")
 	AstKindType        = AstMetaKindType("Type")
 	AstKindValue       = AstMetaKindValue("Value")
 	AstKindEnd         = AstMetaKindEnd("End")
 	AstKindError       = AstMetaKindError("Error")
-    // AstKindFlatten is a special ast kind that insruct the parser not to generate a token 
-	AstKindFlatten     = AstMetaKindFlatten("Flatten")
+    AstKindFlatten     = AstMetaKindFlatten("Flatten")
 )
 
 func (astkind AstBasicMetaKind) String() string {
@@ -74,6 +71,7 @@ func (astkind AstMetaKindNone) String() string        { return "AstNone       "
 func (astkind AstMetaKindProgram) String() string     { return "AstProgram    " }
 func (astkind AstMetaKindStatements) String() string  { return "AstStatements " }
 func (astkind AstMetaKindStatement) String() string   { return "AstStatement  " }
+func (astkind AstMetaKindOperation) String() string   { return "AstOperation  " }
 func (astkind AstMetaKindClosed) String() string      { return "AstClosed     " }
 func (astkind AstMetaKindSet) String() string         { return "AstSet        " }
 func (astkind AstMetaKindGet) String() string         { return "AstGet        " }
@@ -83,7 +81,7 @@ func (astkind AstMetaKindArguments) String() string   { return "AstArguments  "
 func (astkind AstMetaKindArgument) String() string    { return "AstArgument   " }
 func (astkind AstMetaKindExpression) String() string  { return "AstExpression " }
 func (astkind AstMetaKindBlock) String() string       { return "AstBlock      " }
-func (astkind AstMetaKindParenthesis) String() string { return "AstParenthesis" }
+func (astkind AstMetaKindParenthesis) String() string{ return "AstParenthesis"}
 func (astkind AstMetaKindList) String() string        { return "AstList       " }
 func (astkind AstMetaKindCapture) String() string     { return "AstCapture    " }
 func (astkind AstMetaKindWordValue) String() string   { return "AstWordValue  " }
@@ -98,6 +96,7 @@ func (astkind AstMetaKindNone) IsLeaf() bool        { return false }
 func (astkind AstMetaKindProgram) IsLeaf() bool     { return false }
 func (astkind AstMetaKindStatements) IsLeaf() bool  { return false }
 func (astkind AstMetaKindStatement) IsLeaf() bool   { return false }
+func (astkind AstMetaKindOperation) IsLeaf() bool   { return false }
 func (astkind AstMetaKindClosed) IsLeaf() bool      { return false }
 func (astkind AstMetaKindSet) IsLeaf() bool         { return false }
 func (astkind AstMetaKindGet) IsLeaf() bool         { return false }
@@ -148,6 +147,14 @@ func (astkind AstMetaKindStatement) Eval(vm *VM, ast Ast, val ...Value) []Value
 	return val
 }
 
+func (astkind AstMetaKindOperation) Eval(vm *VM, ast Ast, val ...Value) []Value {
+	operatorName := ast.Value()	
+	arguments := val
+	vm.Trace("Operator execute: %s %v", operatorName.String(), arguments)
+	res := vm.CallNamed(operatorName.String(), arguments...)
+    return res	
+}
+
 func (astkind AstMetaKindClosed) Eval(vm *VM, ast Ast, val ...Value) []Value {return ReturnEmpty() }
 func (astkind AstMetaKindSet) Eval(vm *VM, ast Ast, val ...Value) []Value {
 	values := val

parser.go

@@ -69,10 +69,9 @@ STATEMENTS -> STATEMENT STATEMENTS | .
 STATEMENT -> BLOCK | CHAIN eos | eos .
 CHAIN -> COMMAND LINKS . 
 LINKS -> LINK LINKS | .
-LINK -> operator COMMAND .  
-COMMAND -> DIRECT | INDIRECT | LITERAL . 
-DIRECT -> NAME PARAMETERS .
-INDIRECT -> SUBSTITUTION PARAMETERS .
+LINK -> operator EXPRESSION .  
+EXPRESSION -> COMMAND | SUBSTITUTION | LITERAL . 
+COMMAND -> NAME PARAMETERS .
 PARAMETER -> NAME | LITERAL | SUBSTITUTION | BLOCK .
 SUBSTITUTION -> PARENTHESIS | GETTER | SETTER | LIST .
 PARENTHESIS -> openparen CHAIN closeparen .
@@ -238,15 +237,22 @@ func AstKindForToken(token Token) AstKind {
 	}
 }
 
-func (parser *Parser) ParseWordValue() *Ast {
-    parser.LogDebug("ParseWordValue: %s\n", parser.current.String())
+func (parser *Parser) ParseLiteral() *Ast {
+    parser.LogDebug("ParseLiteral: %s\n", parser.current.String())
 
 	value := parser.Require(TokenKindInteger, TokenKindString,
-		TokenKindBoolean, TokenKindNil, TokenKindFloat, TokenKindSymbol,
-		TokenKindType, TokenKindWord)
-	
+		TokenKindBoolean, TokenKindNil, TokenKindFloat)
 	astKind := AstKindForToken(value)
-	
+
+	return parser.NewAst(astKind, nil, EmptyAstArray(), value)
+}
+
+func (parser *Parser) ParseName() *Ast {
+    parser.LogDebug("ParseName: %s\n", parser.current.String())
+
+	value := parser.Require(TokenKindWord, TokenKindType, TokenKindSymbol)
+	astKind := AstKindForToken(value)
+
 	return parser.NewAst(astKind, nil, EmptyAstArray(), value)
 }
 
@@ -254,10 +260,10 @@ func (parser *Parser) ParseArgument() *Ast {
     parser.LogDebug("ParseArgument: %s\n", parser.current.String())
 
 	switch {
-			case parser.NextIsGet(): return parser.ParseGet()
-			case parser.NextIsSet(): return parser.ParseSet()
-			case parser.NextIsClosed(): return parser.ParseClosed()
-			case parser.NextIsWordValue(): return parser.ParseWordValue()
+			case parser.NextIsName(): return parser.ParseName()
+			case parser.NextIsBlock(): return parser.ParseBlock()
+			case parser.NextIsSubstitution(): return parser.ParseSubstitution()
+			case parser.NextIsLiteral(): return parser.ParseLiteral()
 			default: parser.Panicf("error: in argument: expected $, =, }, word or value")
 			return nil
 	}
@@ -311,9 +317,9 @@ func (parser *Parser) ParseParenthesis() *Ast {
 
 
 	ast := NewAstWithToken(AstKindParenthesis, op)
-	expr := parser.ParseExpression()
+	expr := parser.ParseChain()
 	if expr.IsNone() {
-		return parser.NewAstError("expected expression")
+		return parser.NewAstError("expected chain")
 	}
 	if AstIsError(expr) {
 		return expr
@@ -350,24 +356,24 @@ func (parser *Parser) ParseBlock() *Ast {
 	return ast
 }
 
+func (parser *Parser) RequireName() Token {
+    return parser.Require(TokenKindWord, TokenKindType, TokenKindSymbol)
+}
+
 /* Parses the target of a set expression */
 func (parser *Parser) ParseTarget() *Ast {
     parser.LogDebug("ParseTarget: %s\n", parser.current.String())
     var target *Ast
     
     switch {
-		case parser.NextIs(TokenKindWord, TokenKindType, TokenKindSymbol):
+		case parser.NextIsName():
 			// Direct target to a set
-			token := parser.Require(TokenKindWord, TokenKindType, TokenKindSymbol)
+			token := parser.RequireName()
 			target = NewAstWithToken(AstKindTarget, token)
-		case parser.NextIsGet(): 
+		case parser.NextIsSubstitution():
 			// Indirect target
 			target = NewAstWithToken(AstKindTarget, parser.current)
-			target.AppendChild(parser.ParseGet())
-		case parser.NextIsClosed():
-			// Indirect target
-			target = NewAstWithToken(AstKindTarget, parser.current)
-			target.AppendChild(parser.ParseClosed())
+			target.AppendChild(parser.ParseSubstitution())
 		default: 
 			parser.Panicf("Malformed setter.")
 			return nil
@@ -388,18 +394,13 @@ func (parser *Parser) ParseOrigin() *Ast {
     var target *Ast
     
     switch {
-		case parser.NextIs(TokenKindWord, TokenKindType, TokenKindSymbol):
+		case parser.NextIsName():
 			// Direct target to a set
-			token := parser.Require(TokenKindWord, TokenKindType, TokenKindSymbol)		
+			token := parser.RequireName()		
 			target = NewAstWithToken(AstKindTarget, token)
-		case parser.NextIsGet(): 
-			// Indirect target
-			target = NewAstWithToken(AstKindTarget, parser.current)
-			target.AppendChild(parser.ParseGet())
-		case parser.NextIsClosed():
-			// Indirect target
+		case parser.NextIsSubstitution(): 
 			target = NewAstWithToken(AstKindTarget, parser.current)
-			target.AppendChild(parser.ParseClosed())
+			target.AppendChild(parser.ParseSubstitution())
 		default: 
 			parser.Panicf("Malformed getter")
 			return nil
@@ -439,14 +440,15 @@ func (parser *Parser) ParseCommand() *Ast {
 	return command
 }
 
-func (parser *Parser) ParseClosed() *Ast {
-    parser.LogDebug("ParseClosed: %s\n", parser.current.String())
+func (parser *Parser) ParseSubstitution() *Ast {
+    parser.LogDebug("ParseSubstitution: %s\n", parser.current.String())
     switch {
-		case parser.NextIs(TokenKindOpenBlock): return parser.ParseBlock()
 		case parser.NextIs(TokenKindOpenParen): return parser.ParseParenthesis()
 		case parser.NextIs(TokenKindOpenList): return parser.ParseList()
+		case parser.NextIs(TokenKindGet): return parser.ParseGet()
+		case parser.NextIs(TokenKindSet): return parser.ParseSet()
 		default:
-			parser.Panicf("Syntax error in closed, expected {, (, [")
+			parser.Panicf("Syntax error in substitution, expected $, =, (, [")
 			return nil
 	}
 }
@@ -457,9 +459,8 @@ func (parser *Parser) ParseExpression() *Ast {
     
     switch {
 		case parser.NextIsWord(): return parser.ParseCommand()
-		case parser.NextIsSet(): return parser.ParseSet()
-		case parser.NextIsGet(): return parser.ParseGet()
-		case parser.NextIsValue(): return parser.ParseCommand()
+		case parser.NextIsSubstitution(): return parser.ParseSubstitution()
+		case parser.NextIsLiteral(): return parser.ParseLiteral()
 		default: 
 			parser.Panicf("Syntax error in expression, expected word, $, =, value")
 			return nil
@@ -502,8 +503,14 @@ func (parser Parser) NextIs(kinds ...TokenKind) bool {
     return false
 } 
 
-func (parser Parser) NextIsClosed() bool {
-    return parser.NextIs(TokenKindOpenBlock, TokenKindOpenList, TokenKindOpenParen)
+func (parser Parser) NextIsBlock() bool {
+    return parser.NextIs(TokenKindOpenBlock)
+    
+}
+
+func (parser Parser) NextIsSubstitution() bool {
+    return parser.NextIs(TokenKindOpenList, TokenKindOpenParen, 
+    TokenKindSet, TokenKindGet)
 }
 
 func (parser Parser) NextIsWord() bool {
@@ -518,29 +525,29 @@ func (parser Parser) NextIsSet() bool {
     return parser.NextIs(TokenKindSet)
 }
 
-func (parser Parser) NextIsValue() bool {
-    return parser.NextIs(TokenKindString, TokenKindType, 
-    TokenKindInteger, TokenKindFloat, TokenKindBoolean, TokenKindNil,
-    TokenKindSymbol)
+func (parser Parser) NextIsLiteral() bool {
+    return parser.NextIs(TokenKindString, TokenKindInteger, TokenKindFloat, 
+    TokenKindBoolean, TokenKindNil)
+}
+
+func (parser Parser) NextIsName() bool {
+    return parser.NextIs(TokenKindWord, TokenKindType, TokenKindSymbol)
 }
 
 func (parser Parser) NextIsOperator() bool {
     return parser.NextIs(TokenKindOperator)
 }
 
-func (parser Parser) NextIsWordValue() bool {
-    return parser.NextIs(TokenKindWord, TokenKindString, TokenKindType, 
-    TokenKindInteger, TokenKindFloat, TokenKindBoolean, TokenKindNil,
-    TokenKindSymbol)
+func (parser Parser) NextIsArgument() bool {
+    return parser.NextIsName() || parser.NextIsLiteral() || 
+        parser.NextIsSubstitution() || parser.NextIsBlock() 
 }
 
-func (parser Parser) NextIsArgument() bool {
-    return parser.NextIs(TokenKindOpenBlock, TokenKindOpenList, TokenKindOpenParen, 
-    TokenKindSet, TokenKindGet, TokenKindWord, TokenKindString, TokenKindType,
-    TokenKindInteger, TokenKindFloat, TokenKindBoolean, TokenKindNil,
-    TokenKindSymbol)
+func (parser Parser) NextIsCommand() bool {
+    return parser.NextIsName()
 }
 
+
 func (parser Parser) NextIsErrorSkipped() bool {
     return parser.NextIs(TokenKindCloseBlock, TokenKindCloseList, TokenKindCloseParen, 
     TokenKindEOX, TokenKindEOF)
@@ -555,30 +562,71 @@ func (parser Parser) NextIsEOF() bool {
 }
 
 func (parser Parser) NextIsExpression() bool {
-    return parser.NextIs(TokenKindWord,
-                    TokenKindGet,
-                    TokenKindSet,
-                    TokenKindString, 
-                    TokenKindType, 
-                    TokenKindInteger, 
-                    TokenKindFloat, 
-                    TokenKindBoolean, TokenKindNil,
-                    TokenKindSymbol)
+    return parser.NextIsCommand() || parser.NextIsSubstitution() || 
+        parser.NextIsLiteral()
+}
+
+func (parser Parser) NextIsChain() bool {
+	return parser.NextIsExpression()
 }
 
+
 func (parser Parser) NextIsStatement() bool {
-	return parser.NextIsExpression() || parser.NextIsClosed() || parser.NextIsEOX()
+	return parser.NextIsChain() || parser.NextIsBlock() || parser.NextIsEOX()
 }
 
+func (parser *Parser) ParseChain() *Ast {
+    expression := parser.ParseExpression()
+    if !parser.NextIsOperator() {
+        return expression
+    } 
+    
+    var expressions = []*Ast{ expression }
+    var operators = []Token {}
+    for parser.NextIsOperator() {
+        oper := parser.Require(TokenKindOperator)
+        expression := parser.ParseExpression()
+        if expression == nil {
+            parser.Panicf("Expected expression after operator.")
+        }
+        expressions = append(expressions, expression)
+        operators = append(operators, oper)
+    }
+    // now there are N expressions and N - 1 operators, iterate backwards
+    // for easy composition
+    var chain *Ast
+    
+    for i, j := len(expressions) - 1, len(operators) - 1 ; i >= 0 && j >= 0 ; i, j = i - 1 , j - 1 {
+        expression := expressions[i]
+        oper := operators[j]
+        if chain == nil {
+            expression2 := expressions[i-1]
+            subst2 := NewAstWithToken(AstKindParenthesis, oper)
+            subst2.AppendChildren(expression)
+            subst1 := NewAstWithToken(AstKindParenthesis, oper)
+            subst1.AppendChildren(expression2)
+            chain = NewAstWithToken(AstKindOperation, oper)
+            chain.AppendChildren(subst1, subst2)
+            i--
+        } else {
+            subst := NewAstWithToken(AstKindParenthesis, oper)
+            subst.AppendChildren(expression)
+            newChain := NewAstWithToken(AstKindOperation, oper)
+            newChain.AppendChildren(subst, chain)
+            chain = newChain
+        }
+    }
+    return chain
+} 
 
 func (parser *Parser) ParseStatement() *Ast {
     parser.LogDebug("ParseStatement: %s\n", parser.current.String())
     switch  {
-        case parser.NextIsClosed(): return parser.ParseClosed()
-        case parser.NextIsExpression(): return parser.ParseExpressionStatement()
+        case parser.NextIsBlock(): return parser.ParseBlock()
+        case parser.NextIsExpression(): return parser.ParseChain()
         case parser.NextIsEOX(): return parser.ParseEmptyStatement()
         default:
-        parser.Panicf("Expected closed, expression, or EOX in statement")
+        parser.Panicf("Expected block, command, or EOX in statement")
         return parser.NewAstError("Expected closed, expression, or EOX in statement")
     }
 }