lourenco/mgmt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

lang: New function engine

Browse Source 
This mega patch primarily introduces a new function engine. The main
reasons for this new engine are:

1) Massively improved performance with lock-contended graphs.

Certain large function graphs could have very high lock-contention which
turned out to be much slower than I would have liked. This new algorithm
happens to be basically lock-free, so that's another helpful
improvement.

2) Glitch-free function graphs.

The function graphs could "glitch" (an FRP term) which could be
undesirable in theory. In practice this was never really an issue, and
I've not explicitly guaranteed that the new graphs are provably
glitch-free, but in practice things are a lot more consistent.

3) Simpler graph shape.

The new graphs don't require the private channels. This makes
understanding the graphs a lot easier.

4) Branched graphs only run half.

Previously we would run two pure side of an if statement, and while this
was mostly meant as an early experiment, it stayed in for far too long
and now was the right time to remove this. This also means our graphs
are much smaller and more efficient too.

Note that this changed the function API slightly. Everything has been
ported. It's possible that we introduce a new API in the future, but it
is unexpected to cause removal of the two current APIs.

In addition, we finally split out the "schedule" aspect from
world.schedule(). The "pick me" aspects now happen in a separate
resource, rather than as a yucky side-effect in the function. This also
lets us more precisely choose when we're scheduled, and we can observe
without being chosen too.

As usual many thanks to Sam for helping through some of the algorithmic
graph shape issues!
This commit is contained in:
James Shubin
2025-09-09 02:46:59 -04:00
parent 1e2db5b8c5
commit 790b7199ca
109 changed files with 3632 additions and 6904 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
lang/core/collect.go
View File

@@ -100,10 +100,8 @@ type CollectFunc struct {
init *interfaces.Init
last types.Value // last value received to use for diff
args []types.Value
kind string
result types.Value // last calculated output
input chan string // stream of inputs
kind *string // the active kind
watchChan chan error
}
@@ -292,13 +290,13 @@ func (obj *CollectFunc) Info() *interfaces.Info {
// Init runs some startup code for this function.
func (obj *CollectFunc) Init(init *interfaces.Init) error {
obj.init = init
obj.input = make(chan string)
obj.watchChan = make(chan error) // XXX: sender should close this, but did I implement that part yet???
return nil
}
// Stream returns the changing values that this func has over time.
func (obj *CollectFunc) Stream(ctx context.Context) error {
defer close(obj.init.Output) // the sender closes
ctx, cancel := context.WithCancel(ctx)
defer cancel() // important so that we cleanup the watch when exiting
for {
@@ -306,50 +304,35 @@ func (obj *CollectFunc) Stream(ctx context.Context) error {
// TODO: should this first chan be run as a priority channel to
// avoid some sort of glitch? is that even possible? can our
// hostname check with reality (below) fix that?
case input, ok := <-obj.init.Input:
case kind, ok := <-obj.input:
if !ok {
obj.init.Input = nil // don't infinite loop back
continue // no more inputs, but don't return!
obj.input = nil // don't infinite loop back
return fmt.Errorf("unexpected close")
}
//if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil {
// return errwrap.Wrapf(err, "wrong function input")
//}
if obj.last != nil && input.Cmp(obj.last) == nil {
continue // value didn't change, skip it
}
obj.last = input // store for next
args, err := interfaces.StructToCallableArgs(input) // []types.Value, error)
if err != nil {
return err
}
obj.args = args
kind := args[0].Str()
if kind == "" {
return fmt.Errorf("can't use an empty kind")
}
if obj.init.Debug {
obj.init.Logf("kind: %s", kind)
if obj.kind != nil && *obj.kind == kind {
continue // nothing changed
}
// TODO: support changing the key over time?
if obj.kind == "" {
obj.kind = kind // store it
if obj.kind == nil {
obj.kind = &kind // store
var err error
// Don't send a value right away, wait for the
// first Watch startup event to get one!
obj.watchChan, err = obj.init.World.ResWatch(ctx, obj.kind) // watch for var changes
obj.watchChan, err = obj.init.World.ResWatch(ctx, kind) // watch for var changes
if err != nil {
return err
}
} else if obj.kind != kind {
return fmt.Errorf("can't change kind, previously: `%s`", obj.kind)
continue // we get values on the watch chan, not here!
}
continue // we get values on the watch chan, not here!
if *obj.kind == kind {
continue // skip duplicates
}
// *obj.kind != kind
return fmt.Errorf("can't change kind, previously: `%s`", *obj.kind)
case err, ok := <-obj.watchChan:
if !ok { // closed
@@ -360,27 +343,13 @@ func (obj *CollectFunc) Stream(ctx context.Context) error {
return nil
}
if err != nil {
return errwrap.Wrapf(err, "channel watch failed on `%s`", obj.kind)
return errwrap.Wrapf(err, "channel watch failed on `%s`", *obj.kind)
}
result, err := obj.Call(ctx, obj.args) // get the value...
if err != nil {
if err := obj.init.Event(ctx); err != nil { // send event
return err
}
// if the result is still the same, skip sending an update...
if obj.result != nil && result.Cmp(obj.result) == nil {
continue // result didn't change
}
obj.result = result // store new result
case <-ctx.Done():
return nil
}
select {
case obj.init.Output <- obj.result: // send
// pass
case <-ctx.Done():
return nil
}
@@ -402,6 +371,21 @@ func (obj *CollectFunc) Call(ctx context.Context, args []types.Value) (types.Val
return nil, fmt.Errorf("invalid resource kind: %s", kind)
}
// Check before we send to a chan where we'd need Stream to be running.
if obj.init == nil {
return nil, funcs.ErrCantSpeculate
}
if obj.init.Debug {
obj.init.Logf("kind: %s", kind)
}
select {
case obj.input <- kind:
case <-ctx.Done():
return nil, ctx.Err()
}
filters := []*engine.ResFilter{}
arg := args[1]
@@ -453,10 +437,6 @@ func (obj *CollectFunc) Call(ctx context.Context, args []types.Value) (types.Val
}
}
if obj.init == nil {
return nil, funcs.ErrCantSpeculate
}
list := types.NewList(obj.Info().Sig.Out) // collectFuncOutType
if len(filters) == 0 {

1012
lang/core/core_test.go
View File

File diff suppressed because it is too large Load Diff

27
lang/core/datetime/now.go
View File

@@ -31,7 +31,6 @@ package coredatetime
import (
"context"
"fmt"
"time"
"github.com/purpleidea/mgmt/lang/funcs"
@@ -82,22 +81,8 @@ func (obj *Now) Init(init *interfaces.Init) error {
return nil
}
// Stream returns the changing values that this fact has over time.
// Stream starts a mainloop and runs Event when it's time to Call() again.
func (obj *Now) Stream(ctx context.Context) error {
defer close(obj.init.Output) // always signal when we're done
// We always wait for our initial event to start.
select {
case _, ok := <-obj.init.Input:
if ok {
return fmt.Errorf("unexpected input")
}
obj.init.Input = nil
case <-ctx.Done():
return nil
}
// XXX: this might be an interesting fact to write because:
// 1) will the sleeps from the ticker be in sync with the second ticker?
// 2) if we care about a less precise interval (eg: minute changes) can
@@ -124,17 +109,9 @@ func (obj *Now) Stream(ctx context.Context) error {
return nil
}
result, err := obj.Call(ctx, nil)
if err != nil {
if err := obj.init.Event(ctx); err != nil {
return err
}
select {
case obj.init.Output <- result:
case <-ctx.Done():
return nil
}
}
}

130
lang/core/datetime/str_now.go Normal file
View File

@@ -0,0 +1,130 @@
// Mgmt
// Copyright (C) James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
//
// Additional permission under GNU GPL version 3 section 7
//
// If you modify this program, or any covered work, by linking or combining it
// with embedded mcl code and modules (and that the embedded mcl code and
// modules which link with this program, contain a copy of their source code in
// the authoritative form) containing parts covered by the terms of any other
// license, the licensors of this program grant you additional permission to
// convey the resulting work. Furthermore, the licensors of this program grant
// the original author, James Shubin, additional permission to update this
// additional permission if he deems it necessary to achieve the goals of this
// additional permission.
package coredatetime
import (
"context"
"strconv"
"time"
"github.com/purpleidea/mgmt/lang/funcs"
"github.com/purpleidea/mgmt/lang/interfaces"
"github.com/purpleidea/mgmt/lang/types"
)
const (
// StrNowFuncName is the name this fact is registered as. It's still a
// Func Name because this is the name space the fact is actually using.
StrNowFuncName = "str_now"
)
func init() {
funcs.ModuleRegister(ModuleName, StrNowFuncName, func() interfaces.Func { return &StrNow{} }) // must register the fact and name
}
// StrNow is a fact which returns the current date and time.
type StrNow struct {
init *interfaces.Init
}
// String returns a simple name for this fact. This is needed so this struct can
// satisfy the pgraph.Vertex interface.
func (obj *StrNow) String() string {
return NowFuncName
}
// Validate makes sure we've built our struct properly.
func (obj *StrNow) Validate() error {
return nil
}
// Info returns some static info about itself.
func (obj *StrNow) Info() *interfaces.Info {
return &interfaces.Info{
Pure: false, // non-constant facts can't be pure!
Memo: false,
Fast: false,
Spec: false,
Sig: types.NewType("func() str"),
}
}
// Init runs some startup code for this fact.
func (obj *StrNow) Init(init *interfaces.Init) error {
obj.init = init
return nil
}
// Stream returns the changing values that this fact has over time.
func (obj *StrNow) Stream(ctx context.Context) error {
// XXX: this might be an interesting fact to write because:
// 1) will the sleeps from the ticker be in sync with the second ticker?
// 2) if we care about a less precise interval (eg: minute changes) can
// we set this up so it doesn't tick as often? -- Yes (make this a function or create a limit function to wrap this)
// 3) is it best to have a delta timer that wakes up before it's needed
// and calculates how much longer to sleep for?
ticker := time.NewTicker(time.Duration(1) * time.Second)
//ticker := time.NewTicker(time.Duration(1) * time.Millisecond)
//ticker := time.NewTicker(time.Duration(1) * time.Microsecond)
//ticker := time.NewTicker(time.Duration(1) * time.Nanosecond)
defer ticker.Stop()
// streams must generate an initial event on startup
// even though ticker will send one, we want to be faster to first event
startChan := make(chan struct{}) // start signal
close(startChan) // kick it off!
for {
select {
case <-startChan:
startChan = nil // disable
case <-ticker.C: // received the timer event
// pass
case <-ctx.Done():
return nil
}
if err := obj.init.Event(ctx); err != nil {
return err
}
}
}
// Call this fact and return the value if it is possible to do so at this time.
func (obj *StrNow) Call(ctx context.Context, args []types.Value) (types.Value, error) {
return &types.StrValue{ // seconds since 1970 as a string...
V: strconv.FormatInt(time.Now().Unix(), 10), // .UTC() not necessary
//V: strconv.FormatInt(time.Now().UnixMilli(), 10), // .UTC() not necessary
//V: strconv.FormatInt(time.Now().UnixMicro(), 10), // .UTC() not necessary
//V: strconv.FormatInt(time.Now().UnixNano(), 10), // .UTC() not necessary
}, nil
}

58
lang/core/deploy/abspath.go
View File

@@ -114,62 +114,6 @@ func (obj *AbsPathFunc) Init(init *interfaces.Init) error {
return nil
}
// Stream returns the changing values that this func has over time.
func (obj *AbsPathFunc) Stream(ctx context.Context) error {
defer close(obj.init.Output) // the sender closes
for {
select {
case input, ok := <-obj.init.Input:
if !ok {
obj.init.Input = nil // don't infinite loop back
continue // no more inputs, but don't return!
}
//if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil {
// return errwrap.Wrapf(err, "wrong function input")
//}
if obj.last != nil && input.Cmp(obj.last) == nil {
continue // value didn't change, skip it
}
obj.last = input // store for next
args, err := interfaces.StructToCallableArgs(input) // []types.Value, error)
if err != nil {
return err
}
obj.args = args
path := args[0].Str()
// TODO: add validation for absolute path?
if obj.path != nil && *obj.path == path {
continue // nothing changed
}
obj.path = &path
result, err := obj.Call(ctx, obj.args)
if err != nil {
return err
}
// if the result is still the same, skip sending an update...
if obj.result != nil && result.Cmp(obj.result) == nil {
continue // result didn't change
}
obj.result = result // store new result
case <-ctx.Done():
return nil
}
select {
case obj.init.Output <- obj.result: // send
// pass
case <-ctx.Done():
return nil
}
}
}
// Copy is implemented so that the obj.built value is not lost if we copy this
// function.
func (obj *AbsPathFunc) Copy() interfaces.Func {
@@ -187,6 +131,8 @@ func (obj *AbsPathFunc) Call(ctx context.Context, args []types.Value) (types.Val
}
path := args[0].Str()
// TODO: add validation for absolute path?
if obj.data == nil {
return nil, funcs.ErrCantSpeculate
}

62
lang/core/deploy/readfile.go
View File

@@ -61,11 +61,6 @@ var _ interfaces.DataFunc = &ReadFileFunc{}
type ReadFileFunc struct {
init *interfaces.Init
data *interfaces.FuncData
last types.Value // last value received to use for diff
args []types.Value
filename *string // the active filename
result types.Value // last calculated output
}
// String returns a simple name for this function. This is needed so this struct
@@ -115,63 +110,6 @@ func (obj *ReadFileFunc) Init(init *interfaces.Init) error {
return nil
}
// Stream returns the changing values that this func has over time.
func (obj *ReadFileFunc) Stream(ctx context.Context) error {
defer close(obj.init.Output) // the sender closes
for {
select {
case input, ok := <-obj.init.Input:
if !ok {
obj.init.Input = nil // don't infinite loop back
continue // no more inputs, but don't return!
}
//if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil {
// return errwrap.Wrapf(err, "wrong function input")
//}
if obj.last != nil && input.Cmp(obj.last) == nil {
continue // value didn't change, skip it
}
obj.last = input // store for next
args, err := interfaces.StructToCallableArgs(input) // []types.Value, error)
if err != nil {
return err
}
obj.args = args
filename := args[0].Str()
// TODO: add validation for absolute path?
// TODO: add check for empty string
if obj.filename != nil && *obj.filename == filename {
continue // nothing changed
}
obj.filename = &filename
result, err := obj.Call(ctx, obj.args)
if err != nil {
return err
}
// if the result is still the same, skip sending an update...
if obj.result != nil && result.Cmp(obj.result) == nil {
continue // result didn't change
}
obj.result = result // store new result
case <-ctx.Done():
return nil
}
select {
case obj.init.Output <- obj.result: // send
// pass
case <-ctx.Done():
return nil
}
}
}
// Copy is implemented so that the obj.built value is not lost if we copy this
// function.
func (obj *ReadFileFunc) Copy() interfaces.Func {

62
lang/core/deploy/readfileabs.go
View File

@@ -61,11 +61,6 @@ var _ interfaces.DataFunc = &ReadFileAbsFunc{}
type ReadFileAbsFunc struct {
init *interfaces.Init
data *interfaces.FuncData
last types.Value // last value received to use for diff
args []types.Value
filename *string // the active filename
result types.Value // last calculated output
}
// String returns a simple name for this function. This is needed so this struct
@@ -115,63 +110,6 @@ func (obj *ReadFileAbsFunc) Init(init *interfaces.Init) error {
return nil
}
// Stream returns the changing values that this func has over time.
func (obj *ReadFileAbsFunc) Stream(ctx context.Context) error {
defer close(obj.init.Output) // the sender closes
for {
select {
case input, ok := <-obj.init.Input:
if !ok {
obj.init.Input = nil // don't infinite loop back
continue // no more inputs, but don't return!
}
//if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil {
// return errwrap.Wrapf(err, "wrong function input")
//}
if obj.last != nil && input.Cmp(obj.last) == nil {
continue // value didn't change, skip it
}
obj.last = input // store for next
args, err := interfaces.StructToCallableArgs(input) // []types.Value, error)
if err != nil {
return err
}
obj.args = args
filename := args[0].Str()
// TODO: add validation for absolute path?
// TODO: add check for empty string
if obj.filename != nil && *obj.filename == filename {
continue // nothing changed
}
obj.filename = &filename
result, err := obj.Call(ctx, obj.args)
if err != nil {
return err
}
// if the result is still the same, skip sending an update...
if obj.result != nil && result.Cmp(obj.result) == nil {
continue // result didn't change
}
obj.result = result // store new result
case <-ctx.Done():
return nil
}
select {
case obj.init.Output <- obj.result: // send
// pass
case <-ctx.Done():
return nil
}
}
}
// Copy is implemented so that the obj.built value is not lost if we copy this
// function.
func (obj *ReadFileAbsFunc) Copy() interfaces.Func {

29
lang/core/example/flipflop.go
View File

@@ -31,7 +31,6 @@ package coreexample
import (
"context"
"fmt"
"sync"
"time"
@@ -88,22 +87,8 @@ func (obj *FlipFlop) Init(init *interfaces.Init) error {
return nil
}
// Stream returns the changing values that this fact has over time.
// Stream starts a mainloop and runs Event when it's time to Call() again.
func (obj *FlipFlop) Stream(ctx context.Context) error {
defer close(obj.init.Output) // always signal when we're done
// We always wait for our initial event to start.
select {
case _, ok := <-obj.init.Input:
if ok {
return fmt.Errorf("unexpected input")
}
obj.init.Input = nil
case <-ctx.Done():
return nil
}
// TODO: don't hard code 5 sec interval
ticker := time.NewTicker(time.Duration(5) * time.Second)
defer ticker.Stop()
@@ -125,20 +110,12 @@ func (obj *FlipFlop) Stream(ctx context.Context) error {
return nil
}
result, err := obj.Call(ctx, nil)
if err != nil {
return err
}
obj.mutex.Lock()
obj.value = !obj.value // flip it
obj.mutex.Unlock()
select {
case obj.init.Output <- result:
case <-ctx.Done():
return nil
if err := obj.init.Event(ctx); err != nil {
return err
}
}
}

73
lang/core/example/vumeter.go
View File

@@ -37,7 +37,6 @@ import (
"path/filepath"
"strconv"
"strings"
"sync"
"syscall"
"time"
@@ -147,66 +146,36 @@ func (obj *VUMeterFunc) Init(init *interfaces.Init) error {
// Stream returns the changing values that this func has over time.
func (obj *VUMeterFunc) Stream(ctx context.Context) error {
defer close(obj.init.Output) // the sender closes
ticker := newTicker()
ticker := time.NewTicker(time.Duration(1) * time.Second)
defer ticker.Stop()
// FIXME: this goChan seems to work better than the ticker :)
// this is because we have a ~1sec delay in capturing the value in exec
goChan := make(chan struct{})
once := &sync.Once{}
onceFunc := func() { close(goChan) } // only run once!
// streams must generate an initial event on startup
// even though ticker will send one, we want to be faster to first event
startChan := make(chan struct{}) // start signal
close(startChan) // kick it off!
// XXX: We have a back pressure problem! Call takes ~1sec to run. But
// since we moved to a buffered event channel, we now generate new
// events every second, rather than every second _after_ the previous
// event was consumed... This means, we're bombaring the engine with
// more events than it can ever process. Add a backpressure mechanism so
// that we're always draining the event channel. We could do that here
// per-function for rare cases like this, and/or we could try and fix it
// in the engine.
for {
select {
case input, ok := <-obj.init.Input:
if !ok {
obj.init.Input = nil // don't infinite loop back
continue // no more inputs, but don't return!
}
//if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil {
// return errwrap.Wrapf(err, "wrong function input")
//}
case <-startChan:
startChan = nil // disable
if obj.last != nil && input.Cmp(obj.last) == nil {
continue // value didn't change, skip it
}
obj.last = input // store for next
args, err := interfaces.StructToCallableArgs(input) // []types.Value, error)
if err != nil {
return err
}
obj.args = args
once.Do(onceFunc)
continue // we must wrap around and go in through goChan
//case <-ticker.C: // received the timer event
case <-goChan: // triggers constantly
if obj.last == nil {
continue // still waiting for input values
}
result, err := obj.Call(ctx, obj.args)
if err != nil {
return err
}
// if the result is still the same, skip sending an update...
if obj.result != nil && result.Cmp(obj.result) == nil {
continue // result didn't change
}
obj.result = result // store new result
case <-ticker.C: // received the timer event
// pass
case <-ctx.Done():
return nil
}
select {
case obj.init.Output <- obj.result: // send
// pass
case <-ctx.Done():
return nil
if err := obj.init.Event(ctx); err != nil {
return err
}
}
}

47
lang/core/fmt/printf.go
View File

@@ -315,53 +315,6 @@ func (obj *PrintfFunc) Init(init *interfaces.Init) error {
return nil
}
// Stream returns the changing values that this func has over time.
func (obj *PrintfFunc) Stream(ctx context.Context) error {
defer close(obj.init.Output) // the sender closes
for {
select {
case input, ok := <-obj.init.Input:
if !ok {
return nil // can't output any more
}
//if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil {
// return errwrap.Wrapf(err, "wrong function input")
//}
if obj.last != nil && input.Cmp(obj.last) == nil {
continue // value didn't change, skip it
}
obj.last = input // store for next
args, err := interfaces.StructToCallableArgs(input) // []types.Value, error)
if err != nil {
return err
}
result, err := obj.Call(ctx, args)
if err != nil {
return err
}
// if the result is still the same, skip sending an update...
if obj.result != nil && result.Cmp(obj.result) == nil {
continue // result didn't change
}
obj.result = result // store new result
case <-ctx.Done():
return nil
}
select {
case obj.init.Output <- obj.result: // send
// pass
case <-ctx.Done():
return nil
}
}
}
// Copy is implemented so that the obj.Type value is not lost if we copy this
// function.
func (obj *PrintfFunc) Copy() interfaces.Func {

50
lang/core/golang/template.go
View File

@@ -84,9 +84,6 @@ type TemplateFunc struct {
built bool // was this function built yet?
init *interfaces.Init
last types.Value // last value received to use for diff
result types.Value // last calculated output
}
// String returns a simple name for this function. This is needed so this struct
@@ -348,53 +345,6 @@ Loop:
return buf.String(), nil
}
// Stream returns the changing values that this func has over time.
func (obj *TemplateFunc) Stream(ctx context.Context) error {
defer close(obj.init.Output) // the sender closes
for {
select {
case input, ok := <-obj.init.Input:
if !ok {
return nil // can't output any more
}
//if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil {
// return errwrap.Wrapf(err, "wrong function input")
//}
if obj.last != nil && input.Cmp(obj.last) == nil {
continue // value didn't change, skip it
}
obj.last = input // store for next
args, err := interfaces.StructToCallableArgs(input) // []types.Value, error)
if err != nil {
return err
}
result, err := obj.Call(ctx, args)
if err != nil {
return err
}
// if the result is still the same, skip sending an update...
if obj.result != nil && result.Cmp(obj.result) == nil {
continue // result didn't change
}
obj.result = result // store new result
case <-ctx.Done():
return nil
}
select {
case obj.init.Output <- obj.result: // send
// pass
case <-ctx.Done():
return nil
}
}
}
// Copy is implemented so that the obj.built value is not lost if we copy this
// function.
func (obj *TemplateFunc) Copy() interfaces.Func {

240
lang/core/history.go
View File

@@ -32,6 +32,8 @@ package core // TODO: should this be in its own individual package?
import (
"context"
"fmt"
"sync"
"time"
"github.com/purpleidea/mgmt/lang/funcs"
"github.com/purpleidea/mgmt/lang/interfaces"
@@ -46,6 +48,9 @@ const (
// arg names...
historyArgNameValue = "value"
historyArgNameIndex = "index"
// factor helps us sample much faster for precision reasons.
factor = 10
)
func init() {
@@ -54,25 +59,39 @@ func init() {
var _ interfaces.BuildableFunc = &HistoryFunc{} // ensure it meets this expectation
// HistoryFunc is special function which returns the Nth oldest value seen. It
// must store up incoming values until it gets enough to return the desired one.
// A restart of the program, will expunge the stored state. This obviously takes
// more memory, the further back you wish to index. A change in the index var is
// generally not useful, but it is permitted. Moving it to a smaller value will
// cause older index values to be expunged. If this is undesirable, a max count
// could be added. This was not implemented with efficiency in mind. Since some
// functions might not send out un-changed values, it might also make sense to
// implement a *time* based hysteresis, since this only looks at the last N
// changed values. A time based hysteresis would tick every precision-width, and
// store whatever the latest value at that time is.
// HistoryFunc is special function which returns the value N milliseconds ago.
// It must store up incoming values until it gets enough to return the desired
// one. If it doesn't yet have a value, it will initially return the oldest
// value it can. A restart of the program, will expunge the stored state. This
// obviously takes more memory, the further back you wish to index. A change in
// the index var is generally not useful, but it is permitted. Moving it to a
// smaller value will cause older index values to be expunged. If this is
// undesirable, a max count could be added. This was not implemented with
// efficiency in mind. This implements a *time* based hysteresis, since
// previously this only looked at the last N changed values. Since some
// functions might not send out un-changed values, it might make more sense this
// way. This time based hysteresis should tick every precision-width, and store
// whatever the latest value at that time is. This is implemented wrong, because
// we can't guarantee the sampling interval is constant, and it's also wasteful.
// We should implement a better version that keeps track of the time, so that we
// can pick the closest one and also not need to store duplicates.
// XXX: This function needs another look. We likely we to snapshot everytime we
// get a new value in obj.Call instead of having a ticker.
type HistoryFunc struct {
Type *types.Type // type of input value (same as output type)
init *interfaces.Init
history []types.Value // goes from newest (index->0) to oldest (len()-1)
input chan int64
delay *int64
result types.Value // last calculated output
value types.Value // last value
buffer []*valueWithTimestamp
interval int
retention int
ticker *time.Ticker
mutex *sync.Mutex // don't need an rwmutex since only one reader
}
// String returns a simple name for this function. This is needed so this struct
@@ -168,68 +187,165 @@ func (obj *HistoryFunc) Info() *interfaces.Info {
// Init runs some startup code for this function.
func (obj *HistoryFunc) Init(init *interfaces.Init) error {
obj.init = init
obj.input = make(chan int64)
obj.mutex = &sync.Mutex{}
return nil
}
// Stream returns the changing values that this func has over time.
func (obj *HistoryFunc) Stream(ctx context.Context) error {
defer close(obj.init.Output) // the sender closes
obj.ticker = time.NewTicker(1) // build it however (non-zero to avoid panic!)
defer obj.ticker.Stop() // double stop is safe
obj.ticker.Stop() // begin with a stopped ticker
select {
case <-obj.ticker.C: // drain if needed
default:
}
for {
select {
case input, ok := <-obj.init.Input:
case delay, ok := <-obj.input:
if !ok {
return nil // can't output any more
obj.input = nil // don't infinite loop back
return fmt.Errorf("unexpected close")
}
//if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil {
// return errwrap.Wrapf(err, "wrong function input")
// obj.delay is only used here for duplicate detection,
// and while similar to obj.interval, we don't reuse it
// because we don't want a race condition reading delay
if obj.delay != nil && *obj.delay == delay {
continue // nothing changed
}
obj.delay = &delay
obj.reinit(int(delay)) // starts ticker!
case <-obj.ticker.C: // received the timer event
obj.store()
// XXX: We deadlock here if the select{} in obj.Call
// runs at the same time and the event obj.ag is
// unbuffered. Should the engine buffer?
// XXX: If we send events, we basically infinite loop :/
// XXX: Didn't look into the feedback mechanism yet.
//if err := obj.init.Event(ctx); err != nil {
// return err
//}
//if obj.last != nil && input.Cmp(obj.last) == nil {
// continue // value didn't change, skip it
//}
//obj.last = input // store for next
index := int(input.Struct()[historyArgNameIndex].Int())
value := input.Struct()[historyArgNameValue]
var result types.Value
if index < 0 {
return fmt.Errorf("can't use a negative index of %d", index)
}
// 1) truncate history so length equals index
if len(obj.history) > index {
// remove all but first N elements, where N == index
obj.history = obj.history[:index]
}
// 2) (un)shift (add our new value to the beginning)
obj.history = append([]types.Value{value}, obj.history...)
// 3) are we ready to output a sufficiently old value?
if index >= len(obj.history) {
continue // not enough history is stored yet...
}
// 4) read one off the back
result = obj.history[len(obj.history)-1]
// TODO: do we want to do this?
// if the result is still the same, skip sending an update...
if obj.result != nil && result.Cmp(obj.result) == nil {
continue // result didn't change
}
obj.result = result // store new result
case <-ctx.Done():
return nil
}
select {
case obj.init.Output <- obj.result: // send
// pass
case <-ctx.Done():
return nil
}
}
}
func (obj *HistoryFunc) reinit(delay int) {
obj.mutex.Lock()
defer obj.mutex.Unlock()
if obj.buffer == nil {
}
obj.interval = delay
obj.retention = delay + 10000 // XXX: arbitrary
obj.buffer = []*valueWithTimestamp{}
duration := delay / factor // XXX: sample more often than delay?
// Start sampler...
if duration == 0 { // can't be zero or ticker will panic
duration = 100 // XXX: 1ms is probably too fast
}
obj.ticker.Reset(time.Duration(duration) * time.Millisecond)
}
func (obj *HistoryFunc) store() {
obj.mutex.Lock()
defer obj.mutex.Unlock()
val := obj.value.Copy() // copy
now := time.Now()
v := &valueWithTimestamp{
Timestamp: now,
Value: val,
}
obj.buffer = append(obj.buffer, v) // newer values go at the end
retention := time.Duration(obj.retention) * time.Millisecond
// clean up old entries
cutoff := now.Add(-retention)
i := 0
for ; i < len(obj.buffer); i++ {
if obj.buffer[i].Timestamp.After(cutoff) {
break
}
}
obj.buffer = obj.buffer[i:]
}
func (obj *HistoryFunc) peekAgo(ms int) types.Value {
obj.mutex.Lock()
defer obj.mutex.Unlock()
if obj.buffer == nil { // haven't started yet
return nil
}
if len(obj.buffer) == 0 { // no data exists yet
return nil
}
target := time.Now().Add(-time.Duration(ms) * time.Millisecond)
for i := len(obj.buffer) - 1; i >= 0; i-- {
if !obj.buffer[i].Timestamp.After(target) {
return obj.buffer[i].Value
}
}
// If no value found, return the oldest one.
return obj.buffer[0].Value
}
// Call this function with the input args and return the value if it is possible
// to do so at this time.
func (obj *HistoryFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) {
if len(args) < 2 {
return nil, fmt.Errorf("not enough args")
}
value := args[0]
interval := args[1].Int() // ms (used to be index)
if interval < 0 {
return nil, fmt.Errorf("can't use a negative interval of %d", interval)
}
// Check before we send to a chan where we'd need Stream to be running.
if obj.init == nil {
return nil, funcs.ErrCantSpeculate
}
obj.mutex.Lock()
obj.value = value // store a copy
obj.mutex.Unlock()
// XXX: we deadlock here if obj.init.Event also runs at the same time!
// XXX: ...only if it's unbuffered of course. Should the engine buffer?
select {
case obj.input <- interval: // inform the delay interval
case <-ctx.Done():
return nil, ctx.Err()
}
val := obj.peekAgo(int(interval)) // contains mutex
if val == nil { // don't have a value yet, return self...
return obj.value, nil
}
return val, nil
}
// valueWithTimestamp stores a value alongside the time it was recorded.
type valueWithTimestamp struct {
Timestamp time.Time
Value types.Value
}

312
lang/core/iter/filter.go
View File

@@ -77,10 +77,8 @@ type FilterFunc struct {
listType *types.Type
// outputChan is an initially-nil channel from which we receive output
// lists from the subgraph. This channel is reset when the subgraph is
// recreated.
outputChan chan types.Value
argFuncs []interfaces.Func
outputFunc interfaces.Func
}
// String returns a simple name for this function. This is needed so this struct
@@ -99,6 +97,11 @@ func (obj *FilterFunc) ArgGen(index int) (string, error) {
}
// helper
//
// NOTE: The expression signature is shown here, but the actual "signature" of
// this in the function graph returns the "dummy" value because we do the same
// this that we do with ExprCall for example. That means that this function is
// one of very few where the actual expr signature is different from the func!
func (obj *FilterFunc) sig() *types.Type {
// func(inputs []?1, function func(?1) bool) []?1
typ := "?1"
@@ -121,6 +124,60 @@ func (obj *FilterFunc) sig() *types.Type {
// runs.
func (obj *FilterFunc) Build(typ *types.Type) (*types.Type, error) {
// typ is the KindFunc signature we're trying to build...
if typ.Kind != types.KindFunc {
return nil, fmt.Errorf("input type must be of kind func")
}
if len(typ.Ord) != 2 {
return nil, fmt.Errorf("the map needs exactly two args")
}
if typ.Map == nil {
return nil, fmt.Errorf("the map is nil")
}
tInputs, exists := typ.Map[typ.Ord[0]]
if !exists || tInputs == nil {
return nil, fmt.Errorf("first argument was missing")
}
tFunction, exists := typ.Map[typ.Ord[1]]
if !exists || tFunction == nil {
return nil, fmt.Errorf("second argument was missing")
}
if tInputs.Kind != types.KindList {
return nil, fmt.Errorf("first argument must be of kind list")
}
if tFunction.Kind != types.KindFunc {
return nil, fmt.Errorf("second argument must be of kind func")
}
if typ.Out == nil {
return nil, fmt.Errorf("return type must be specified")
}
if typ.Out.Kind != types.KindList {
return nil, fmt.Errorf("return argument must be a list")
}
if len(tFunction.Ord) != 1 {
return nil, fmt.Errorf("the functions map needs exactly one arg")
}
if tFunction.Map == nil {
return nil, fmt.Errorf("the functions map is nil")
}
tArg, exists := tFunction.Map[tFunction.Ord[0]]
if !exists || tArg == nil {
return nil, fmt.Errorf("the functions first argument was missing")
}
if err := tArg.Cmp(tInputs.Val); err != nil {
return nil, errwrap.Wrapf(err, "the functions arg type must match the input list contents type")
}
if tFunction.Out == nil {
return nil, fmt.Errorf("return type of function must be specified")
}
if err := tFunction.Out.Cmp(types.TypeBool); err != nil {
return nil, errwrap.Wrapf(err, "return type of function must be a bool")
}
// TODO: Do we need to be extra careful and check that this matches?
// unificationUtil.UnifyCmp(typ, obj.sig()) != nil {}
@@ -130,11 +187,22 @@ func (obj *FilterFunc) Build(typ *types.Type) (*types.Type, error) {
return obj.sig(), nil
}
// SetShape tells the function about some special graph engine pointers.
func (obj *FilterFunc) SetShape(argFuncs []interfaces.Func, outputFunc interfaces.Func) {
obj.argFuncs = argFuncs
obj.outputFunc = outputFunc
}
// Validate tells us if the input struct takes a valid form.
func (obj *FilterFunc) Validate() error {
if obj.Type == nil {
return fmt.Errorf("type is not yet known")
}
if obj.argFuncs == nil || obj.outputFunc == nil {
return fmt.Errorf("function did not receive shape information")
}
return nil
}
@@ -145,7 +213,7 @@ func (obj *FilterFunc) Info() *interfaces.Info {
Pure: false, // XXX: what if the input function isn't pure?
Memo: false,
Fast: false,
Spec: false,
Spec: false, // must be false with the current graph shape code
Sig: obj.sig(), // helper
Err: obj.Validate(),
}
@@ -162,125 +230,6 @@ func (obj *FilterFunc) Init(init *interfaces.Init) error {
return nil
}
// Stream returns the changing values that this func has over time.
func (obj *FilterFunc) Stream(ctx context.Context) error {
// Every time the FuncValue or the length of the list changes, recreate
// the subgraph, by calling the FuncValue N times on N nodes, each of
// which extracts one of the N values in the list.
defer close(obj.init.Output) // the sender closes
// A Func to send input lists to the subgraph. The Txn.Erase() call
// ensures that this Func is not removed when the subgraph is recreated,
// so that the function graph can propagate the last list we received to
// the subgraph.
inputChan := make(chan types.Value)
subgraphInput := &structs.ChannelBasedSourceFunc{
Name: "subgraphInput",
Source: obj,
Chan: inputChan,
Type: obj.listType,
}
obj.init.Txn.AddVertex(subgraphInput)
if err := obj.init.Txn.Commit(); err != nil {
return errwrap.Wrapf(err, "commit error in Stream")
}
obj.init.Txn.Erase() // prevent the next Reverse() from removing subgraphInput
defer func() {
close(inputChan)
obj.init.Txn.Reverse()
obj.init.Txn.DeleteVertex(subgraphInput)
obj.init.Txn.Commit()
}()
obj.outputChan = nil
canReceiveMoreFuncValuesOrInputLists := true
canReceiveMoreOutputLists := true
for {
if !canReceiveMoreFuncValuesOrInputLists && !canReceiveMoreOutputLists {
//break
return nil
}
select {
case input, ok := <-obj.init.Input:
if !ok {
obj.init.Input = nil // block looping back here
canReceiveMoreFuncValuesOrInputLists = false
continue
}
if obj.last != nil && input.Cmp(obj.last) == nil {
continue // value didn't change, skip it
}
obj.last = input // store for next
value, exists := input.Struct()[filterArgNameFunction]
if !exists {
return fmt.Errorf("programming error, can't find edge")
}
newFuncValue, ok := value.(*full.FuncValue)
if !ok {
return fmt.Errorf("programming error, can't convert to *FuncValue")
}
newInputList, exists := input.Struct()[filterArgNameInputs]
if !exists {
return fmt.Errorf("programming error, can't find edge")
}
// If we have a new function or the length of the input
// list has changed, then we need to replace the
// subgraph with a new one that uses the new function
// the correct number of times.
// It's important to have this compare step to avoid
// redundant graph replacements which slow things down,
// but also cause the engine to lock, which can preempt
// the process scheduler, which can cause duplicate or
// unnecessary re-sending of values here, which causes
// the whole process to repeat ad-nauseum.
n := len(newInputList.List())
if newFuncValue != obj.lastFuncValue || n != obj.lastInputListLength {
obj.lastFuncValue = newFuncValue
obj.lastInputListLength = n
// replaceSubGraph uses the above two values
if err := obj.replaceSubGraph(subgraphInput); err != nil {
return errwrap.Wrapf(err, "could not replace subgraph")
}
canReceiveMoreOutputLists = true
}
// send the new input list to the subgraph
select {
case inputChan <- newInputList:
case <-ctx.Done():
return nil
}
case outputList, ok := <-obj.outputChan:
// send the new output list downstream
if !ok {
obj.outputChan = nil
canReceiveMoreOutputLists = false
continue
}
select {
case obj.init.Output <- outputList:
case <-ctx.Done():
return nil
}
case <-ctx.Done():
return nil
}
}
}
func (obj *FilterFunc) replaceSubGraph(subgraphInput interfaces.Func) error {
// replaceSubGraph creates a subgraph which first splits the input list
// into 'n' nodes. Then it applies 'newFuncValue' to each, and sends
@@ -314,16 +263,9 @@ func (obj *FilterFunc) replaceSubGraph(subgraphInput interfaces.Func) error {
// "filterCombineList0" -> "outputListFunc"
// "filterCombineList1" -> "outputListFunc"
// "filterCombineList2" -> "outputListFunc"
// "outputListFunc" -> "filterSubgraphOutput"
//
// "outputListFunc" -> "funcSubgraphOutput"
// }
const channelBasedSinkFuncArgNameEdgeName = structs.ChannelBasedSinkFuncArgName // XXX: not sure if the specific name matters.
// We pack the value pairs into structs that look like this...
structType := types.NewType(fmt.Sprintf("struct{v %s; b bool}", obj.Type.String()))
getArgName := func(i int) string {
return fmt.Sprintf("outputElem%d", i)
}
argNameInputList := "inputList"
// delete the old subgraph
if err := obj.init.Txn.Reverse(); err != nil {
@@ -331,15 +273,26 @@ func (obj *FilterFunc) replaceSubGraph(subgraphInput interfaces.Func) error {
}
// create the new subgraph
obj.outputChan = make(chan types.Value)
subgraphOutput := &structs.ChannelBasedSinkFunc{
Name: "filterSubgraphOutput",
Target: obj,
EdgeName: channelBasedSinkFuncArgNameEdgeName,
Chan: obj.outputChan,
Type: obj.listType,
// XXX: Should we move creation of funcSubgraphOutput into Init() ?
funcSubgraphOutput := &structs.OutputFunc{ // the new graph shape thing!
//Textarea: obj.Textarea,
Name: "funcSubgraphOutput",
Type: obj.sig().Out,
EdgeName: structs.OutputFuncArgName,
}
obj.init.Txn.AddVertex(subgraphOutput)
obj.init.Txn.AddVertex(funcSubgraphOutput)
obj.init.Txn.AddEdge(funcSubgraphOutput, obj.outputFunc, &interfaces.FuncEdge{Args: []string{structs.OutputFuncArgName}}) // "out"
// XXX: hack add this edge that I thought would happen in call.go
obj.init.Txn.AddEdge(obj, funcSubgraphOutput, &interfaces.FuncEdge{Args: []string{structs.OutputFuncDummyArgName}}) // "dummy"
// We pack the value pairs into structs that look like this...
structType := types.NewType(fmt.Sprintf("struct{v %s; b bool}", obj.Type.String()))
getArgName := func(i int) string {
return fmt.Sprintf("outputElem%d", i)
}
argNameInputList := "inputList"
m := make(map[string]*types.Type)
ord := []string{}
@@ -385,10 +338,9 @@ func (obj *FilterFunc) replaceSubGraph(subgraphInput interfaces.Func) error {
},
)
edge := &interfaces.FuncEdge{Args: []string{structs.OutputFuncArgName}} // "out"
obj.init.Txn.AddVertex(outputListFunc)
obj.init.Txn.AddEdge(outputListFunc, subgraphOutput, &interfaces.FuncEdge{
Args: []string{channelBasedSinkFuncArgNameEdgeName},
})
obj.init.Txn.AddEdge(outputListFunc, funcSubgraphOutput, edge)
for i := 0; i < obj.lastInputListLength; i++ {
i := i
@@ -414,7 +366,7 @@ func (obj *FilterFunc) replaceSubGraph(subgraphInput interfaces.Func) error {
)
obj.init.Txn.AddVertex(inputElemFunc)
outputElemFunc, err := obj.lastFuncValue.CallWithFuncs(obj.init.Txn, []interfaces.Func{inputElemFunc})
outputElemFunc, err := obj.lastFuncValue.CallWithFuncs(obj.init.Txn, []interfaces.Func{inputElemFunc}, funcSubgraphOutput)
if err != nil {
return errwrap.Wrapf(err, "could not call obj.lastFuncValue.CallWithFuncs()")
}
@@ -462,6 +414,76 @@ func (obj *FilterFunc) replaceSubGraph(subgraphInput interfaces.Func) error {
return obj.init.Txn.Commit()
}
// Call this function with the input args and return the value if it is possible
// to do so at this time.
func (obj *FilterFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) {
if len(args) < 2 {
return nil, fmt.Errorf("not enough args")
}
// Check before we send to a chan where we'd need Stream to be running.
if obj.init == nil {
return nil, funcs.ErrCantSpeculate
}
// Need this before we can *really* run this properly.
if len(obj.argFuncs) != 2 {
return nil, funcs.ErrCantSpeculate
//return nil, fmt.Errorf("unexpected input arg length")
}
newInputList := args[0]
value := args[1]
newFuncValue, ok := value.(*full.FuncValue)
if !ok {
return nil, fmt.Errorf("programming error, can't convert to *FuncValue")
}
a := obj.last != nil && newInputList.Cmp(obj.last) == nil
b := obj.lastFuncValue != nil && newFuncValue == obj.lastFuncValue
if a && b {
return types.NewNil(), nil // dummy value
}
obj.last = newInputList // store for next
obj.lastFuncValue = newFuncValue
// Every time the FuncValue or the length of the list changes, recreate
// the subgraph, by calling the FuncValue N times on N nodes, each of
// which extracts one of the N values in the list.
n := len(newInputList.List())
c := n == obj.lastInputListLength
if b && c {
return types.NewNil(), nil // dummy value
}
obj.lastInputListLength = n
if b && !c { // different length list
return types.NewNil(), nil // dummy value
}
// If we have a new function or the length of the input list has
// changed, then we need to replace the subgraph with a new one that
// uses the new function the correct number of times.
subgraphInput := obj.argFuncs[0]
// replaceSubGraph uses the above two values
if err := obj.replaceSubGraph(subgraphInput); err != nil {
return nil, errwrap.Wrapf(err, "could not replace subgraph")
}
return nil, interfaces.ErrInterrupt
}
// Cleanup runs after that function was removed from the graph.
func (obj *FilterFunc) Cleanup(ctx context.Context) error {
obj.init.Txn.Reverse()
//obj.init.Txn.DeleteVertex(subgraphInput) // XXX: should we delete it?
return obj.init.Txn.Commit()
}
// Copy is implemented so that the type value is not lost if we copy this
// function.
func (obj *FilterFunc) Copy() interfaces.Func {

247
lang/core/iter/map.go
View File

@@ -81,10 +81,8 @@ type MapFunc struct {
inputListType *types.Type
outputListType *types.Type
// outputChan is an initially-nil channel from which we receive output
// lists from the subgraph. This channel is reset when the subgraph is
// recreated.
outputChan chan types.Value
argFuncs []interfaces.Func
outputFunc interfaces.Func
}
// String returns a simple name for this function. This is needed so this struct
@@ -103,6 +101,11 @@ func (obj *MapFunc) ArgGen(index int) (string, error) {
}
// helper
//
// NOTE: The expression signature is shown here, but the actual "signature" of
// this in the function graph returns the "dummy" value because we do the same
// this that we do with ExprCall for example. That means that this function is
// one of very few where the actual expr signature is different from the func!
func (obj *MapFunc) sig() *types.Type {
// func(inputs []?1, function func(?1) ?2) []?2
tIi := "?1"
@@ -186,17 +189,31 @@ func (obj *MapFunc) Build(typ *types.Type) (*types.Type, error) {
return nil, errwrap.Wrapf(err, "return type of function must match returned list contents type")
}
// TODO: Do we need to be extra careful and check that this matches?
// unificationUtil.UnifyCmp(typ, obj.sig()) != nil {}
obj.Type = tInputs.Val // or tArg
obj.RType = tFunction.Out // or typ.Out.Val
return obj.sig(), nil
}
// SetShape tells the function about some special graph engine pointers.
func (obj *MapFunc) SetShape(argFuncs []interfaces.Func, outputFunc interfaces.Func) {
obj.argFuncs = argFuncs
obj.outputFunc = outputFunc
}
// Validate tells us if the input struct takes a valid form.
func (obj *MapFunc) Validate() error {
if obj.Type == nil || obj.RType == nil {
return fmt.Errorf("type is not yet known")
}
if obj.argFuncs == nil || obj.outputFunc == nil {
return fmt.Errorf("function did not receive shape information")
}
return nil
}
@@ -207,7 +224,7 @@ func (obj *MapFunc) Info() *interfaces.Info {
Pure: false, // XXX: what if the input function isn't pure?
Memo: false,
Fast: false,
Spec: false,
Spec: false, // must be false with the current graph shape code
Sig: obj.sig(), // helper
Err: obj.Validate(),
}
@@ -225,124 +242,6 @@ func (obj *MapFunc) Init(init *interfaces.Init) error {
return nil
}
// Stream returns the changing values that this func has over time.
func (obj *MapFunc) Stream(ctx context.Context) error {
// Every time the FuncValue or the length of the list changes, recreate the
// subgraph, by calling the FuncValue N times on N nodes, each of which
// extracts one of the N values in the list.
defer close(obj.init.Output) // the sender closes
// A Func to send input lists to the subgraph. The Txn.Erase() call ensures
// that this Func is not removed when the subgraph is recreated, so that the
// function graph can propagate the last list we received to the subgraph.
inputChan := make(chan types.Value)
subgraphInput := &structs.ChannelBasedSourceFunc{
Name: "subgraphInput",
Source: obj,
Chan: inputChan,
Type: obj.inputListType,
}
obj.init.Txn.AddVertex(subgraphInput)
if err := obj.init.Txn.Commit(); err != nil {
return errwrap.Wrapf(err, "commit error in Stream")
}
obj.init.Txn.Erase() // prevent the next Reverse() from removing subgraphInput
defer func() {
close(inputChan)
obj.init.Txn.Reverse()
obj.init.Txn.DeleteVertex(subgraphInput)
obj.init.Txn.Commit()
}()
obj.outputChan = nil
canReceiveMoreFuncValuesOrInputLists := true
canReceiveMoreOutputLists := true
for {
if !canReceiveMoreFuncValuesOrInputLists && !canReceiveMoreOutputLists {
//break
return nil
}
select {
case input, ok := <-obj.init.Input:
if !ok {
obj.init.Input = nil // block looping back here
canReceiveMoreFuncValuesOrInputLists = false
continue
}
if obj.last != nil && input.Cmp(obj.last) == nil {
continue // value didn't change, skip it
}
obj.last = input // store for next
value, exists := input.Struct()[mapArgNameFunction]
if !exists {
return fmt.Errorf("programming error, can't find edge")
}
newFuncValue, ok := value.(*full.FuncValue)
if !ok {
return fmt.Errorf("programming error, can't convert to *FuncValue")
}
newInputList, exists := input.Struct()[mapArgNameInputs]
if !exists {
return fmt.Errorf("programming error, can't find edge")
}
// If we have a new function or the length of the input
// list has changed, then we need to replace the
// subgraph with a new one that uses the new function
// the correct number of times.
// It's important to have this compare step to avoid
// redundant graph replacements which slow things down,
// but also cause the engine to lock, which can preempt
// the process scheduler, which can cause duplicate or
// unnecessary re-sending of values here, which causes
// the whole process to repeat ad-nauseum.
n := len(newInputList.List())
if newFuncValue != obj.lastFuncValue || n != obj.lastInputListLength {
obj.lastFuncValue = newFuncValue
obj.lastInputListLength = n
// replaceSubGraph uses the above two values
if err := obj.replaceSubGraph(subgraphInput); err != nil {
return errwrap.Wrapf(err, "could not replace subgraph")
}
canReceiveMoreOutputLists = true
}
// send the new input list to the subgraph
select {
case inputChan <- newInputList:
case <-ctx.Done():
return nil
}
case outputList, ok := <-obj.outputChan:
// send the new output list downstream
if !ok {
obj.outputChan = nil
canReceiveMoreOutputLists = false
continue
}
select {
case obj.init.Output <- outputList:
case <-ctx.Done():
return nil
}
case <-ctx.Done():
return nil
}
}
}
func (obj *MapFunc) replaceSubGraph(subgraphInput interfaces.Func) error {
// Create a subgraph which splits the input list into 'n' nodes, applies
// 'newFuncValue' to each, then combines the 'n' outputs back into a
@@ -363,11 +262,9 @@ func (obj *MapFunc) replaceSubGraph(subgraphInput interfaces.Func) error {
// "outputElem1" -> "outputListFunc"
// "outputElem2" -> "outputListFunc"
//
// "outputListFunc" -> "mapSubgraphOutput"
// "outputListFunc" -> "funcSubgraphOutput"
// }
const channelBasedSinkFuncArgNameEdgeName = structs.ChannelBasedSinkFuncArgName // XXX: not sure if the specific name matters.
// delete the old subgraph
if err := obj.init.Txn.Reverse(); err != nil {
return errwrap.Wrapf(err, "could not Reverse")
@@ -375,15 +272,18 @@ func (obj *MapFunc) replaceSubGraph(subgraphInput interfaces.Func) error {
// create the new subgraph
obj.outputChan = make(chan types.Value)
subgraphOutput := &structs.ChannelBasedSinkFunc{
Name: "mapSubgraphOutput",
Target: obj,
EdgeName: channelBasedSinkFuncArgNameEdgeName,
Chan: obj.outputChan,
Type: obj.outputListType,
// XXX: Should we move creation of funcSubgraphOutput into Init() ?
funcSubgraphOutput := &structs.OutputFunc{ // the new graph shape thing!
//Textarea: obj.Textarea,
Name: "funcSubgraphOutput",
Type: obj.sig().Out,
EdgeName: structs.OutputFuncArgName,
}
obj.init.Txn.AddVertex(subgraphOutput)
obj.init.Txn.AddVertex(funcSubgraphOutput)
obj.init.Txn.AddEdge(funcSubgraphOutput, obj.outputFunc, &interfaces.FuncEdge{Args: []string{structs.OutputFuncArgName}}) // "out"
// XXX: hack add this edge that I thought would happen in call.go
obj.init.Txn.AddEdge(obj, funcSubgraphOutput, &interfaces.FuncEdge{Args: []string{structs.OutputFuncDummyArgName}}) // "dummy"
m := make(map[string]*types.Type)
ord := []string{}
@@ -398,6 +298,7 @@ func (obj *MapFunc) replaceSubGraph(subgraphInput interfaces.Func) error {
Ord: ord,
Out: obj.outputListType,
}
outputListFunc := structs.SimpleFnToDirectFunc(
"mapOutputList",
&types.FuncValue{
@@ -413,10 +314,9 @@ func (obj *MapFunc) replaceSubGraph(subgraphInput interfaces.Func) error {
},
)
edge := &interfaces.FuncEdge{Args: []string{structs.OutputFuncArgName}} // "out"
obj.init.Txn.AddVertex(outputListFunc)
obj.init.Txn.AddEdge(outputListFunc, subgraphOutput, &interfaces.FuncEdge{
Args: []string{channelBasedSinkFuncArgNameEdgeName},
})
obj.init.Txn.AddEdge(outputListFunc, funcSubgraphOutput, edge)
for i := 0; i < obj.lastInputListLength; i++ {
i := i
@@ -434,6 +334,7 @@ func (obj *MapFunc) replaceSubGraph(subgraphInput interfaces.Func) error {
return nil, fmt.Errorf("inputElemFunc: expected a ListValue argument")
}
// Extract the correct list element.
return list.List()[i], nil
},
T: types.NewType(fmt.Sprintf("func(inputList %s) %s", obj.inputListType, obj.Type)),
@@ -441,7 +342,7 @@ func (obj *MapFunc) replaceSubGraph(subgraphInput interfaces.Func) error {
)
obj.init.Txn.AddVertex(inputElemFunc)
outputElemFunc, err := obj.lastFuncValue.CallWithFuncs(obj.init.Txn, []interfaces.Func{inputElemFunc})
outputElemFunc, err := obj.lastFuncValue.CallWithFuncs(obj.init.Txn, []interfaces.Func{inputElemFunc}, funcSubgraphOutput)
if err != nil {
return errwrap.Wrapf(err, "could not call obj.lastFuncValue.CallWithFuncs()")
}
@@ -457,6 +358,76 @@ func (obj *MapFunc) replaceSubGraph(subgraphInput interfaces.Func) error {
return obj.init.Txn.Commit()
}
// Call this function with the input args and return the value if it is possible
// to do so at this time.
func (obj *MapFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) {
if len(args) < 2 {
return nil, fmt.Errorf("not enough args")
}
// Check before we send to a chan where we'd need Stream to be running.
if obj.init == nil {
return nil, funcs.ErrCantSpeculate
}
// Need this before we can *really* run this properly.
if len(obj.argFuncs) != 2 {
return nil, funcs.ErrCantSpeculate
//return nil, fmt.Errorf("unexpected input arg length")
}
newInputList := args[0]
value := args[1]
newFuncValue, ok := value.(*full.FuncValue)
if !ok {
return nil, fmt.Errorf("programming error, can't convert to *FuncValue")
}
a := obj.last != nil && newInputList.Cmp(obj.last) == nil
b := obj.lastFuncValue != nil && newFuncValue == obj.lastFuncValue
if a && b {
return types.NewNil(), nil // dummy value
}
obj.last = newInputList // store for next
obj.lastFuncValue = newFuncValue
// Every time the FuncValue or the length of the list changes, recreate
// the subgraph, by calling the FuncValue N times on N nodes, each of
// which extracts one of the N values in the list.
n := len(newInputList.List())
c := n == obj.lastInputListLength
if b && c {
return types.NewNil(), nil // dummy value
}
obj.lastInputListLength = n
if b && !c { // different length list
return types.NewNil(), nil // dummy value
}
// If we have a new function or the length of the input list has
// changed, then we need to replace the subgraph with a new one that
// uses the new function the correct number of times.
subgraphInput := obj.argFuncs[0]
// replaceSubGraph uses the above two values
if err := obj.replaceSubGraph(subgraphInput); err != nil {
return nil, errwrap.Wrapf(err, "could not replace subgraph")
}
return nil, interfaces.ErrInterrupt
}
// Cleanup runs after that function was removed from the graph.
func (obj *MapFunc) Cleanup(ctx context.Context) error {
obj.init.Txn.Reverse()
//obj.init.Txn.DeleteVertex(subgraphInput) // XXX: should we delete it?
return obj.init.Txn.Commit()
}
// Copy is implemented so that the type values are not lost if we copy this
// function.
func (obj *MapFunc) Copy() interfaces.Func {

44
lang/core/iter/range.go
View File

@@ -47,7 +47,6 @@ const (
RangeFuncName = "range"
)
var _ interfaces.CallableFunc = &RangeFunc{}
var _ interfaces.BuildableFunc = &RangeFunc{}
// RangeFunc is a function that ranges over elements on a list according to
@@ -175,49 +174,6 @@ func (obj *RangeFunc) Init(init *interfaces.Init) error {
return nil
}
// Stream returns the changing values that this func has over time.
func (obj *RangeFunc) Stream(ctx context.Context) error {
defer close(obj.init.Output) // closing the sender
for {
select {
case input, ok := <-obj.init.Input:
if !ok {
return nil // we don't have more inputs
}
if obj.last != nil && input.Cmp(obj.last) == nil {
continue // nothing has changed, skip it
}
obj.last = input // storing the input for comparison
args, err := interfaces.StructToCallableArgs(input)
if err != nil {
return err
}
result, err := obj.Call(ctx, args)
if err != nil {
return err
}
if obj.result != nil && result.Cmp(obj.result) == nil {
continue // if the result didn't change, we don't need to update
}
obj.result = result // store new result
case <-ctx.Done():
return nil
}
select {
case obj.init.Output <- obj.result: // sending new result
case <-ctx.Done():
return nil
}
}
}
// Call returns the result of this function.
func (obj *RangeFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) {
if len(args) == 1 { // we only have stop, assume start is 0 and step is 1

43
lang/core/local/pool.go
View File

@@ -115,48 +115,6 @@ func (obj *PoolFunc) Init(init *interfaces.Init) error {
return nil
}
// Stream returns the changing values that this func has over time.
func (obj *PoolFunc) Stream(ctx context.Context) error {
defer close(obj.init.Output) // the sender closes
var value types.Value
for {
select {
case input, ok := <-obj.init.Input:
if !ok {
obj.init.Input = nil // don't infinite loop back
continue // no more inputs, but don't return!
}
//if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil {
// return errwrap.Wrapf(err, "wrong function input")
//}
if obj.last != nil && input.Cmp(obj.last) == nil {
continue // value didn't change, skip it
}
obj.last = input // store for next
args, err := interfaces.StructToCallableArgs(input) // []types.Value, error)
if err != nil {
return err
}
result, err := obj.Call(ctx, args)
if err != nil {
return err
}
value = result
case <-ctx.Done():
return nil
}
select {
case obj.init.Output <- value:
case <-ctx.Done():
return nil
}
}
}
// Copy is implemented so that the obj.built value is not lost if we copy this
// function.
func (obj *PoolFunc) Copy() interfaces.Func {
@@ -181,6 +139,7 @@ func (obj *PoolFunc) Call(ctx context.Context, args []types.Value) (types.Value,
if obj.init == nil {
return nil, funcs.ErrCantSpeculate
}
result, err := obj.init.Local.Pool(ctx, namespace, uid, nil)
if err != nil {
return nil, err

42
lang/core/local/vardir.go
View File

@@ -122,48 +122,6 @@ func (obj *VarDirFunc) Init(init *interfaces.Init) error {
return nil
}
// Stream returns the changing values that this func has over time.
func (obj *VarDirFunc) Stream(ctx context.Context) error {
defer close(obj.init.Output) // the sender closes
var value types.Value
for {
select {
case input, ok := <-obj.init.Input:
if !ok {
obj.init.Input = nil // don't infinite loop back
continue // no more inputs, but don't return!
}
//if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil {
// return errwrap.Wrapf(err, "wrong function input")
//}
if obj.last != nil && input.Cmp(obj.last) == nil {
continue // value didn't change, skip it
}
obj.last = input // store for next
args, err := interfaces.StructToCallableArgs(input) // []types.Value, error)
if err != nil {
return err
}
result, err := obj.Call(ctx, args)
if err != nil {
return err
}
value = result
case <-ctx.Done():
return nil
}
select {
case obj.init.Output <- value:
case <-ctx.Done():
return nil
}
}
}
// Copy is implemented so that the obj.built value is not lost if we copy this
// function.
func (obj *VarDirFunc) Copy() interfaces.Func {

20
lang/core/lookup.go
View File

@@ -183,11 +183,6 @@ func (obj *LookupFunc) Build(typ *types.Type) (*types.Type, error) {
return nil, err
}
if _, ok := f.(interfaces.CallableFunc); !ok {
// programming error
return nil, fmt.Errorf("not a CallableFunc")
}
bf, ok := f.(interfaces.BuildableFunc)
if !ok {
// programming error
@@ -248,23 +243,10 @@ func (obj *LookupFunc) Init(init *interfaces.Init) error {
return obj.fn.Init(init)
}
// Stream returns the changing values that this func has over time.
func (obj *LookupFunc) Stream(ctx context.Context) error {
if obj.fn == nil {
return fmt.Errorf("function not built correctly")
}
return obj.fn.Stream(ctx)
}
// Call returns the result of this function.
func (obj *LookupFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) {
if obj.fn == nil {
return nil, funcs.ErrCantSpeculate
}
cf, ok := obj.fn.(interfaces.CallableFunc)
if !ok {
// programming error
return nil, fmt.Errorf("not a CallableFunc")
}
return cf.Call(ctx, args)
return obj.fn.Call(ctx, args)
}

20
lang/core/lookup_default.go
View File

@@ -128,11 +128,6 @@ func (obj *LookupDefaultFunc) Build(typ *types.Type) (*types.Type, error) {
return nil, err
}
if _, ok := f.(interfaces.CallableFunc); !ok {
// programming error
return nil, fmt.Errorf("not a CallableFunc")
}
bf, ok := f.(interfaces.BuildableFunc)
if !ok {
// programming error
@@ -194,23 +189,10 @@ func (obj *LookupDefaultFunc) Init(init *interfaces.Init) error {
return obj.fn.Init(init)
}
// Stream returns the changing values that this func has over time.
func (obj *LookupDefaultFunc) Stream(ctx context.Context) error {
if obj.fn == nil {
return fmt.Errorf("function not built correctly")
}
return obj.fn.Stream(ctx)
}
// Call returns the result of this function.
func (obj *LookupDefaultFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) {
if obj.fn == nil {
return nil, funcs.ErrCantSpeculate
}
cf, ok := obj.fn.(interfaces.CallableFunc)
if !ok {
// programming error
return nil, fmt.Errorf("not a CallableFunc")
}
return cf.Call(ctx, args)
return obj.fn.Call(ctx, args)
}

63
lang/core/os/modinfo.go
View File

@@ -64,8 +64,8 @@ type ModinfoLoadedFunc struct {
init *interfaces.Init
last types.Value // last value received to use for diff
input chan string // stream of inputs
modulename *string // the active module name
result types.Value // last calculated output
}
// String returns a simple name for this function. This is needed so this struct
@@ -103,13 +103,12 @@ func (obj *ModinfoLoadedFunc) Info() *interfaces.Info {
// Init runs some startup code for this function.
func (obj *ModinfoLoadedFunc) Init(init *interfaces.Init) error {
obj.init = init
obj.input = make(chan string)
return nil
}
// Stream returns the changing values that this func has over time.
func (obj *ModinfoLoadedFunc) Stream(ctx context.Context) error {
defer close(obj.init.Output) // the sender closes
// create new watcher
// XXX: does this file produce inotify events?
recWatcher := &recwatch.RecWatcher{
@@ -127,22 +126,12 @@ func (obj *ModinfoLoadedFunc) Stream(ctx context.Context) error {
for {
select {
case input, ok := <-obj.init.Input:
case modulename, ok := <-obj.input:
if !ok {
obj.init.Input = nil // don't infinite loop back
continue // no more inputs, but don't return!
obj.input = nil // don't infinite loop back
return fmt.Errorf("unexpected close")
}
//if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil {
// return errwrap.Wrapf(err, "wrong function input")
//}
if obj.last != nil && input.Cmp(obj.last) == nil {
continue // value didn't change, skip it
}
obj.last = input // store for next
modulename := input.Struct()[modinfoLoadedArgNameModule].Str()
// TODO: add check for empty string
if obj.modulename != nil && *obj.modulename == modulename {
continue // nothing changed
}
@@ -156,33 +145,13 @@ func (obj *ModinfoLoadedFunc) Stream(ctx context.Context) error {
return errwrap.Wrapf(err, "error event received")
}
if obj.last == nil {
if obj.modulename == nil {
continue // still waiting for input values
}
case <-ctx.Done():
return nil
}
args, err := interfaces.StructToCallableArgs(obj.last) // []types.Value, error)
if err != nil {
return err
}
result, err := obj.Call(ctx, args)
if err != nil {
return err
}
// if the result is still the same, skip sending an update...
if obj.result != nil && result.Cmp(obj.result) == nil {
continue // result didn't change
}
obj.result = result // store new result
select {
case obj.init.Output <- obj.result: // send
// pass
if err := obj.init.Event(ctx); err != nil { // send event
return err
}
case <-ctx.Done():
return nil
@@ -198,6 +167,20 @@ func (obj *ModinfoLoadedFunc) Call(ctx context.Context, args []types.Value) (typ
}
modulename := args[0].Str()
// Check before we send to a chan where we'd need Stream to be running.
if obj.init == nil {
return nil, funcs.ErrCantSpeculate
}
// Tell the Stream what we're watching now... This doesn't block because
// Stream should always be ready to consume unless it's closing down...
// If it dies, then a ctx closure should come soon.
select {
case obj.input <- modulename:
case <-ctx.Done():
return nil, ctx.Err()
}
m, err := lsmod.LsMod()
if err != nil {
return nil, errwrap.Wrapf(err, "error reading modules")

115
lang/core/os/readfile.go
View File

@@ -60,15 +60,13 @@ func init() {
// package.
type ReadFileFunc struct {
init *interfaces.Init
last types.Value // last value received to use for diff
recWatcher *recwatch.RecWatcher
events chan error // internal events
wg *sync.WaitGroup
args []types.Value
input chan string // stream of inputs
filename *string // the active filename
result types.Value // last calculated output
}
// String returns a simple name for this function. This is needed so this struct
@@ -106,6 +104,7 @@ func (obj *ReadFileFunc) Info() *interfaces.Info {
// Init runs some startup code for this function.
func (obj *ReadFileFunc) Init(init *interfaces.Init) error {
obj.init = init
obj.input = make(chan string)
obj.events = make(chan error)
obj.wg = &sync.WaitGroup{}
return nil
@@ -113,8 +112,8 @@ func (obj *ReadFileFunc) Init(init *interfaces.Init) error {
// Stream returns the changing values that this func has over time.
func (obj *ReadFileFunc) Stream(ctx context.Context) error {
defer close(obj.init.Output) // the sender closes
defer close(obj.events) // clean up for fun
//defer close(obj.input) // if we close, this is a race with the sender
defer close(obj.events) // clean up for fun
defer obj.wg.Wait()
defer func() {
if obj.recWatcher != nil {
@@ -124,24 +123,21 @@ func (obj *ReadFileFunc) Stream(ctx context.Context) error {
}()
for {
select {
case input, ok := <-obj.init.Input:
case filename, ok := <-obj.input:
if !ok {
obj.init.Input = nil // don't infinite loop back
continue // no more inputs, but don't return!
obj.input = nil // don't infinite loop back
return fmt.Errorf("unexpected close")
}
//if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil {
// return errwrap.Wrapf(err, "wrong function input")
//}
if obj.last != nil && input.Cmp(obj.last) == nil {
continue // value didn't change, skip it
}
obj.last = input // store for next
filename := input.Struct()[readFileArgNameFilename].Str()
// TODO: add validation for absolute path?
// TODO: add check for empty string
if obj.filename != nil && *obj.filename == filename {
//select {
//case obj.ack <- struct{}{}:
//case <-ctx.Done():
// // don't block here on shutdown
// return
//default:
// // pass, in case we didn't Call()
//}
continue // nothing changed
}
obj.filename = &filename
@@ -192,6 +188,17 @@ func (obj *ReadFileFunc) Stream(ctx context.Context) error {
}
}
// XXX: should we send an ACK event to
// Call() right here? This should ideally
// be from the Startup event of recWatcher
//select {
//case obj.ack <- struct{}{}:
//case <-ctx.Done():
// // don't block here on shutdown
// return
//default:
// // pass, in case we didn't Call()
//}
select {
case obj.events <- err:
// send event...
@@ -213,37 +220,12 @@ func (obj *ReadFileFunc) Stream(ctx context.Context) error {
return errwrap.Wrapf(err, "error event received")
}
if obj.last == nil {
continue // still waiting for input values
}
args, err := interfaces.StructToCallableArgs(obj.last) // []types.Value, error)
if err != nil {
return err
}
obj.args = args
result, err := obj.Call(ctx, obj.args)
if err != nil {
if err := obj.init.Event(ctx); err != nil { // send event
return err
}
// if the result is still the same, skip sending an update...
if obj.result != nil && result.Cmp(obj.result) == nil {
continue // result didn't change
}
obj.result = result // store new result
case <-ctx.Done():
return nil
}
select {
case obj.init.Output <- obj.result: // send
// pass
case <-ctx.Done():
return nil
return ctx.Err()
}
}
}
@@ -256,6 +238,30 @@ func (obj *ReadFileFunc) Call(ctx context.Context, args []types.Value) (types.Va
}
filename := args[0].Str()
// TODO: add validation for absolute path?
// TODO: add check for empty string
// Check before we send to a chan where we'd need Stream to be running.
if obj.init == nil {
return nil, funcs.ErrCantSpeculate
}
// Tell the Stream what we're watching now... This doesn't block because
// Stream should always be ready to consume unless it's closing down...
// If it dies, then a ctx closure should come soon.
select {
case obj.input <- filename:
case <-ctx.Done():
return nil, ctx.Err()
}
// XXX: Should we make sure the Stream is ready before we continue here?
//select {
//case <-obj.ack:
// // received
//case <-ctx.Done():
// return nil, ctx.Err()
//}
// read file...
content, err := os.ReadFile(filename)
if err != nil {
@@ -266,3 +272,20 @@ func (obj *ReadFileFunc) Call(ctx context.Context, args []types.Value) (types.Va
V: string(content), // convert to string
}, nil
}
// Cleanup runs after that function was removed from the graph.
func (obj *ReadFileFunc) Cleanup(ctx context.Context) error {
// Even if the filename stops changing, we never shutdown Stream because
// those file contents may change. Theoretically if someone sends us an
// empty string, and then it shuts down we could close.
//if obj.filename == "" { // we require obj.ack to not have a race here
// close(obj.exit) // add a channel into that Stream
//}
return nil
}
// Done is a message from the engine to tell us that no more Call's are coming.
func (obj *ReadFileFunc) Done() error {
close(obj.input) // At this point we know obj.input won't be used.
return nil
}

117
lang/core/os/readfilewait.go
View File

@@ -62,15 +62,13 @@ func init() {
// will eventually be deprecated when the function graph error system is stable.
type ReadFileWaitFunc struct {
init *interfaces.Init
last types.Value // last value received to use for diff
recWatcher *recwatch.RecWatcher
events chan error // internal events
wg *sync.WaitGroup
args []types.Value
input chan string // stream of inputs
filename *string // the active filename
result types.Value // last calculated output
}
// String returns a simple name for this function. This is needed so this struct
@@ -108,6 +106,7 @@ func (obj *ReadFileWaitFunc) Info() *interfaces.Info {
// Init runs some startup code for this function.
func (obj *ReadFileWaitFunc) Init(init *interfaces.Init) error {
obj.init = init
obj.input = make(chan string)
obj.events = make(chan error)
obj.wg = &sync.WaitGroup{}
return nil
@@ -115,8 +114,8 @@ func (obj *ReadFileWaitFunc) Init(init *interfaces.Init) error {
// Stream returns the changing values that this func has over time.
func (obj *ReadFileWaitFunc) Stream(ctx context.Context) error {
defer close(obj.init.Output) // the sender closes
defer close(obj.events) // clean up for fun
//defer close(obj.input) // if we close, this is a race with the sender
defer close(obj.events) // clean up for fun
defer obj.wg.Wait()
defer func() {
if obj.recWatcher != nil {
@@ -126,24 +125,21 @@ func (obj *ReadFileWaitFunc) Stream(ctx context.Context) error {
}()
for {
select {
case input, ok := <-obj.init.Input:
case filename, ok := <-obj.input:
if !ok {
obj.init.Input = nil // don't infinite loop back
continue // no more inputs, but don't return!
obj.input = nil // don't infinite loop back
return fmt.Errorf("unexpected close")
}
//if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil {
// return errwrap.Wrapf(err, "wrong function input")
//}
if obj.last != nil && input.Cmp(obj.last) == nil {
continue // value didn't change, skip it
}
obj.last = input // store for next
filename := input.Struct()[readFileWaitArgNameFilename].Str()
// TODO: add validation for absolute path?
// TODO: add check for empty string
if obj.filename != nil && *obj.filename == filename {
//select {
//case obj.ack <- struct{}{}:
//case <-ctx.Done():
// // don't block here on shutdown
// return
//default:
// // pass, in case we didn't Call()
//}
continue // nothing changed
}
obj.filename = &filename
@@ -194,6 +190,17 @@ func (obj *ReadFileWaitFunc) Stream(ctx context.Context) error {
}
}
// XXX: should we send an ACK event to
// Call() right here? This should ideally
// be from the Startup event of recWatcher
//select {
//case obj.ack <- struct{}{}:
//case <-ctx.Done():
// // don't block here on shutdown
// return
//default:
// // pass, in case we didn't Call()
//}
select {
case obj.events <- err:
// send event...
@@ -215,37 +222,12 @@ func (obj *ReadFileWaitFunc) Stream(ctx context.Context) error {
return errwrap.Wrapf(err, "error event received")
}
if obj.last == nil {
continue // still waiting for input values
}
args, err := interfaces.StructToCallableArgs(obj.last) // []types.Value, error)
if err != nil {
return err
}
obj.args = args
result, err := obj.Call(ctx, obj.args)
if err != nil {
if err := obj.init.Event(ctx); err != nil { // send event
return err
}
// if the result is still the same, skip sending an update...
if obj.result != nil && result.Cmp(obj.result) == nil {
continue // result didn't change
}
obj.result = result // store new result
case <-ctx.Done():
return nil
}
select {
case obj.init.Output <- obj.result: // send
// pass
case <-ctx.Done():
return nil
return ctx.Err()
}
}
}
@@ -258,6 +240,30 @@ func (obj *ReadFileWaitFunc) Call(ctx context.Context, args []types.Value) (type
}
filename := args[0].Str()
// TODO: add validation for absolute path?
// TODO: add check for empty string
// Check before we send to a chan where we'd need Stream to be running.
if obj.init == nil {
return nil, funcs.ErrCantSpeculate
}
// Tell the Stream what we're watching now... This doesn't block because
// Stream should always be ready to consume unless it's closing down...
// If it dies, then a ctx closure should come soon.
select {
case obj.input <- filename:
case <-ctx.Done():
return nil, ctx.Err()
}
// XXX: Should we make sure the Stream is ready before we continue here?
//select {
//case <-obj.ack:
// // received
//case <-ctx.Done():
// return nil, ctx.Err()
//}
// read file...
content, err := os.ReadFile(filename)
if err != nil && !os.IsNotExist(err) { // ignore file not found errors
@@ -269,6 +275,23 @@ func (obj *ReadFileWaitFunc) Call(ctx context.Context, args []types.Value) (type
//}
return &types.StrValue{
V: s,
V: s, // convert to string
}, nil
}
// Cleanup runs after that function was removed from the graph.
func (obj *ReadFileWaitFunc) Cleanup(ctx context.Context) error {
// Even if the filename stops changing, we never shutdown Stream because
// those file contents may change. Theoretically if someone sends us an
// empty string, and then it shuts down we could close.
//if obj.filename == "" { // we require obj.ack to not have a race here
// close(obj.exit) // add a channel into that Stream
//}
return nil
}
// Done is a message from the engine to tell us that no more Call's are coming.
func (obj *ReadFileWaitFunc) Done() error {
close(obj.input) // At this point we know obj.input won't be used.
return nil
}

114
lang/core/os/system.go
View File

@@ -47,6 +47,10 @@ const (
// arg names...
systemArgNameCmd = "cmd"
// SystemFuncBufferLength is the number of lines we can buffer before we
// block. If you need a larger value, please let us know your use-case.
SystemFuncBufferLength = 1024
)
func init() {
@@ -60,9 +64,25 @@ func init() {
// Note that in the likely case in which the process emits several lines one
// after the other, the downstream resources might not run for every line unless
// the "Meta:realize" metaparam is set to true.
//
// Furthermore, there is no guarantee that every intermediate line will be seen,
// particularly if there is no delay between them. Only the last line is
// guaranteed. As a result, it is not recommend to use this for timing or
// coordination. If you are using this for an intermediate value, or a
// non-declarative system, then it's likely you are using this wrong.
type SystemFunc struct {
init *interfaces.Init
cancel context.CancelFunc
input chan string // stream of inputs
last *string // the active command
output *string // the last output
values chan string
count int
mutex *sync.Mutex
}
// String returns a simple name for this function. This is needed so this struct
@@ -101,18 +121,18 @@ func (obj *SystemFunc) Info() *interfaces.Info {
// Init runs some startup code for this function.
func (obj *SystemFunc) Init(init *interfaces.Init) error {
obj.init = init
obj.input = make(chan string)
obj.values = make(chan string, SystemFuncBufferLength)
obj.mutex = &sync.Mutex{}
return nil
}
// Stream returns the changing values that this func has over time.
func (obj *SystemFunc) Stream(ctx context.Context) error {
func (obj *SystemFunc) Stream(ctx context.Context) (reterr error) {
// XXX: this implementation is a bit awkward especially with the port to
// the Stream(context.Context) signature change. This is a straight port
// but we could refactor this eventually.
// Close the output chan to signal that no more values are coming.
defer close(obj.init.Output)
// A channel which closes when the current process exits, on its own
// or due to cancel(). The channel is only closed once all the pending
// stdout and stderr lines have been processed.
@@ -140,7 +160,7 @@ func (obj *SystemFunc) Stream(ctx context.Context) error {
for {
select {
case input, more := <-obj.init.Input:
case shellCommand, more := <-obj.input:
if !more {
// Wait until the current process exits and all of its
// stdout is sent downstream.
@@ -151,7 +171,11 @@ func (obj *SystemFunc) Stream(ctx context.Context) error {
return nil
}
}
shellCommand := input.Struct()[systemArgNameCmd].Str()
if obj.last != nil && *obj.last == shellCommand {
continue // nothing changed
}
obj.last = &shellCommand
// Kill the previous command, if any.
if obj.cancel != nil {
@@ -193,8 +217,22 @@ func (obj *SystemFunc) Stream(ctx context.Context) error {
stdoutScanner := bufio.NewScanner(stdoutReader)
for stdoutScanner.Scan() {
outputValue := &types.StrValue{V: stdoutScanner.Text()}
obj.init.Output <- outputValue
s := stdoutScanner.Text()
obj.mutex.Lock()
obj.count++
obj.mutex.Unlock()
select {
case obj.values <- s: // buffered
case <-ctx.Done():
// don't block here on shutdown
reterr = ctx.Err() // return err
return
}
if err := obj.init.Event(ctx); err != nil { // send event
reterr = err // return err
return
}
}
}()
@@ -220,8 +258,66 @@ func (obj *SystemFunc) Stream(ctx context.Context) error {
wg.Wait()
close(processedChan)
}()
case <-ctx.Done():
return nil
return ctx.Err()
}
}
}
// Call this function with the input args and return the value if it is possible
// to do so at this time.
func (obj *SystemFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) {
if len(args) < 1 {
return nil, fmt.Errorf("not enough args")
}
cmd := args[0].Str()
// Check before we send to a chan where we'd need Stream to be running.
if obj.init == nil {
return nil, funcs.ErrCantSpeculate
}
// Tell the Stream what we're watching now... This doesn't block because
// Stream should always be ready to consume unless it's closing down...
// If it dies, then a ctx closure should come soon.
select {
case obj.input <- cmd:
case <-ctx.Done():
return nil, ctx.Err()
}
obj.mutex.Lock()
// If there are no values, and we've previously received a value then...
if obj.count == 0 && obj.output != nil {
s := *obj.output
obj.mutex.Unlock()
return &types.StrValue{
V: s,
}, nil
}
obj.count-- // we might be briefly negative
obj.mutex.Unlock()
// We know a value must be coming (or the command blocks) so we wait...
select {
case s, ok := <-obj.values:
if !ok {
return nil, fmt.Errorf("unexpected close")
}
obj.output = &s // store
return &types.StrValue{
V: s,
}, nil
case <-ctx.Done():
return nil, ctx.Err()
}
}
// Done is a message from the engine to tell us that no more Call's are coming.
func (obj *SystemFunc) Done() error {
close(obj.input) // At this point we know obj.input won't be used.
return nil
}

114
lang/core/random1.go
View File

@@ -33,6 +33,7 @@ import (
"context"
"crypto/rand"
"fmt"
"math"
"math/big"
"github.com/purpleidea/mgmt/lang/funcs"
@@ -55,20 +56,22 @@ func init() {
funcs.Register(Random1FuncName, func() interfaces.Func { return &Random1Func{} })
}
// Random1Func returns one random string of a certain length.
// XXX: return a stream instead, and combine this with a first(?) function which
// takes the first value and then puts backpressure on the stream. This should
// notify parent functions somehow that their values are no longer required so
// that they can shutdown if possible. Maybe it should be returning a stream of
// floats [0,1] as well, which someone can later map to the alphabet that they
// want. Should random() take an interval to know how often to spit out values?
// It could also just do it once per second, and we could filter for less. If we
// want something high precision, we could add that in the future... We could
// name that "random" and this one can be "random1" until we deprecate it.
// Random1Func returns one random string of a certain length. If you change the
// length, then it will produce a new random value.
type Random1Func struct {
// XXX: To produce a stream of random values every N seconds, make a
// built-in function or use the dual <|> hack below?
// XXX: Maybe it should be returning a stream of floats [0,1] as well,
// which someone can later map to the alphabet that they want. Should
// random() take an interval to know how often to spit out values? It
// could also just do it once per second, and we could filter for less.
// If we want something high precision, we could add that in the future.
// We could name that "random" and this one can be "random1" until we
// deprecate it.
init *interfaces.Init
finished bool // did we send the random string?
length uint16 // last length
result string // last random
}
// String returns a simple name for this function. This is needed so this struct
@@ -136,49 +139,50 @@ func (obj *Random1Func) Init(init *interfaces.Init) error {
return nil
}
// Stream returns the single value that was generated and then closes.
func (obj *Random1Func) Stream(ctx context.Context) error {
defer close(obj.init.Output) // the sender closes
var result string
for {
select {
case input, ok := <-obj.init.Input:
if !ok {
return nil // can't output any more
}
//if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil {
// return errwrap.Wrapf(err, "wrong function input")
//}
if obj.finished {
// TODO: continue instead?
return fmt.Errorf("you can only pass a single input to random")
}
length := input.Struct()[random1ArgNameLength].Int()
// TODO: if negative, randomly pick a length ?
if length < 0 {
return fmt.Errorf("can't generate a negative length")
}
var err error
if result, err = generate(uint16(length)); err != nil {
return err // no errwrap needed b/c helper func
}
case <-ctx.Done():
return nil
}
select {
case obj.init.Output <- &types.StrValue{
V: result,
}:
// we only send one value, then wait for input to close
obj.finished = true
case <-ctx.Done():
return nil
}
// Call this function with the input args and return the value if it is possible
// to do so at this time.
func (obj *Random1Func) Call(ctx context.Context, args []types.Value) (types.Value, error) {
if len(args) < 1 {
return nil, fmt.Errorf("not enough args")
}
length := args[0].Int()
if length < 0 || length > math.MaxUint16 {
// On error, reset the cached values. This *may* be useful if we
// want to use the future "except" operator to produce an stream
// of random values-- we could flip flop between two "random1()"
// functions to successively get a val from one, while resetting
// the other one. Which happens right here... Here's an example:
//
// $now = datetime.now()
// $len = 8 # length of rand
// # alternate every second
// $out = if math.mod($now, 2) == 0 {
// random1($len) <|> random1(-1)
// } else {
// random1(-1) <|> random1($len)
// }
//
// Perhaps it's just better to have a core rand stream function?
obj.length = 0
obj.result = ""
return nil, fmt.Errorf("can't generate an invalid length")
}
if uint16(length) == obj.length { // same, so use cached value
return &types.StrValue{
V: obj.result,
}, nil
}
obj.length = uint16(length) // cache
result, err := generate(uint16(length))
if err != nil {
return nil, err // no errwrap needed b/c helper func
}
obj.result = result // cache
return &types.StrValue{
V: result,
}, nil
}

56
lang/core/struct_lookup.go
View File

@@ -283,62 +283,6 @@ func (obj *StructLookupFunc) Init(init *interfaces.Init) error {
return nil
}
// Stream returns the changing values that this func has over time.
func (obj *StructLookupFunc) Stream(ctx context.Context) error {
defer close(obj.init.Output) // the sender closes
for {
select {
case input, ok := <-obj.init.Input:
if !ok {
return nil // can't output any more
}
//if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil {
// return errwrap.Wrapf(err, "wrong function input")
//}
if obj.last != nil && input.Cmp(obj.last) == nil {
continue // value didn't change, skip it
}
obj.last = input // store for next
st := (input.Struct()[structLookupArgNameStruct]).(*types.StructValue)
field := input.Struct()[structLookupArgNameField].Str()
if field == "" {
return fmt.Errorf("received empty field")
}
if obj.field == "" {
// This can happen at compile time too. Bonus!
obj.field = field // store first field
}
if field != obj.field {
return fmt.Errorf("input field changed from: `%s`, to: `%s`", obj.field, field)
}
result, exists := st.Lookup(obj.field)
if !exists {
return fmt.Errorf("could not lookup field: `%s` in struct", field)
}
// if previous input was `2 + 4`, but now it
// changed to `1 + 5`, the result is still the
// same, so we can skip sending an update...
if obj.result != nil && result.Cmp(obj.result) == nil {
continue // result didn't change
}
obj.result = result // store new result
case <-ctx.Done():
return nil
}
select {
case obj.init.Output <- obj.result: // send
case <-ctx.Done():
return nil
}
}
}
// Call returns the result of this function.
func (obj *StructLookupFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) {
if len(args) < 2 {

65
lang/core/struct_lookup_optional.go
View File

@@ -280,71 +280,6 @@ func (obj *StructLookupOptionalFunc) Init(init *interfaces.Init) error {
return nil
}
// Stream returns the changing values that this func has over time.
func (obj *StructLookupOptionalFunc) Stream(ctx context.Context) error {
defer close(obj.init.Output) // the sender closes
for {
select {
case input, ok := <-obj.init.Input:
if !ok {
return nil // can't output any more
}
//if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil {
// return errwrap.Wrapf(err, "wrong function input")
//}
if obj.last != nil && input.Cmp(obj.last) == nil {
continue // value didn't change, skip it
}
obj.last = input // store for next
st := (input.Struct()[structLookupOptionalArgNameStruct]).(*types.StructValue)
field := input.Struct()[structLookupOptionalArgNameField].Str()
optional := input.Struct()[structLookupOptionalArgNameOptional]
if field == "" {
return fmt.Errorf("received empty field")
}
if obj.field == "" {
// This can happen at compile time too. Bonus!
obj.field = field // store first field
}
if field != obj.field {
return fmt.Errorf("input field changed from: `%s`, to: `%s`", obj.field, field)
}
// We know the result of this lookup statically at
// compile time, but for simplicity we check each time
// here anyways. Maybe one day there will be a fancy
// reason why this might vary over time.
var result types.Value
val, exists := st.Lookup(obj.field)
if exists {
result = val
} else {
result = optional
}
// if previous input was `2 + 4`, but now it
// changed to `1 + 5`, the result is still the
// same, so we can skip sending an update...
if obj.result != nil && result.Cmp(obj.result) == nil {
continue // result didn't change
}
obj.result = result // store new result
case <-ctx.Done():
return nil
}
select {
case obj.init.Output <- obj.result: // send
case <-ctx.Done():
return nil
}
}
}
// Call returns the result of this function.
func (obj *StructLookupOptionalFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) {
if len(args) < 3 {

37
lang/core/sys/cpucount.go
View File

@@ -33,7 +33,6 @@ package coresys
import (
"context"
"fmt"
"os"
"regexp"
"strconv"
@@ -97,24 +96,10 @@ func (obj *CPUCount) Init(init *interfaces.Init) error {
return nil
}
// Stream returns the changing values that this fact has over time. It will
// first poll sysfs to get the initial cpu count, and then receives UEvents from
// the kernel as CPUs are added/removed.
// Stream starts a mainloop and runs Event when it's time to Call() again. It
// will first poll sysfs to get the initial cpu count, and then receives UEvents
// from the kernel as CPUs are added/removed.
func (obj CPUCount) Stream(ctx context.Context) error {
defer close(obj.init.Output) // signal when we're done
// We always wait for our initial event to start.
select {
case _, ok := <-obj.init.Input:
if ok {
return fmt.Errorf("unexpected input")
}
obj.init.Input = nil
case <-ctx.Done():
return nil
}
ss, err := socketset.NewSocketSet(rtmGrps, socketFile, unix.NETLINK_KOBJECT_UEVENT)
if err != nil {
return errwrap.Wrapf(err, "error creating socket set")
@@ -182,23 +167,9 @@ func (obj CPUCount) Stream(ctx context.Context) error {
return nil
}
result, err := obj.Call(ctx, nil)
if err != nil {
if err := obj.init.Event(ctx); err != nil {
return err
}
// if the result is still the same, skip sending an update...
if obj.result != nil && result.Cmp(obj.result) == nil {
continue // result didn't change
}
obj.result = result // store new result
select {
case obj.init.Output <- result:
case <-ctx.Done():
return nil
}
}
}

41
lang/core/sys/cpucount_test.go
View File

@@ -32,50 +32,9 @@
package coresys
import (
"context"
"testing"
"github.com/purpleidea/mgmt/lang/interfaces"
"github.com/purpleidea/mgmt/lang/types"
)
func TestSimple(t *testing.T) {
fact := &CPUCount{}
input := make(chan types.Value)
close(input) // kick it off!
output := make(chan types.Value)
err := fact.Init(&interfaces.Init{
Input: input,
Output: output,
Logf: func(format string, v ...interface{}) {
t.Logf("cpucount_test: "+format, v...)
},
})
if err != nil {
t.Errorf("could not init CPUCount")
return
}
ctx, cancel := context.WithCancel(context.Background())
go func() {
defer cancel()
Loop:
for {
select {
case cpus := <-output:
t.Logf("CPUS: %d\n", cpus.Int())
break Loop
}
}
}()
// now start the stream
if err := fact.Stream(ctx); err != nil {
t.Error(err)
}
}
func TestParseCPUList(t *testing.T) {
var cpulistTests = []struct {
desc string

27
lang/core/sys/hostname.go
View File

@@ -94,20 +94,6 @@ func (obj *Hostname) Init(init *interfaces.Init) error {
// Stream returns the single value that this fact has, and then closes.
func (obj *Hostname) Stream(ctx context.Context) error {
defer close(obj.init.Output) // signal that we're done sending
// We always wait for our initial event to start.
select {
case _, ok := <-obj.init.Input:
if ok {
return fmt.Errorf("unexpected input")
}
obj.init.Input = nil
case <-ctx.Done():
return nil
}
recurse := false // single file
recWatcher, err := recwatch.NewRecWatcher("/etc/hostname", recurse)
if err != nil {
@@ -165,19 +151,9 @@ func (obj *Hostname) Stream(ctx context.Context) error {
return nil
}
// NOTE: We ask the actual machine instead of using obj.init.Hostname
value, err := obj.Call(ctx, nil)
if err != nil {
if err := obj.init.Event(ctx); err != nil {
return err
}
select {
case obj.init.Output <- value:
// pass
case <-ctx.Done():
return nil
}
}
}
@@ -191,6 +167,7 @@ func (obj *Hostname) Call(ctx context.Context, args []types.Value) (types.Value,
hostnameObject := conn.Object(hostname1Iface, hostname1Path)
// NOTE: We ask the actual machine instead of using obj.init.Hostname
h, err := obj.getHostnameProperty(hostnameObject, "Hostname")
if err != nil {
return nil, err

26
lang/core/sys/load.go
View File

@@ -85,22 +85,8 @@ func (obj *Load) Init(init *interfaces.Init) error {
return nil
}
// Stream returns the changing values that this fact has over time.
// Stream starts a mainloop and runs Event when it's time to Call() again.
func (obj *Load) Stream(ctx context.Context) error {
defer close(obj.init.Output) // always signal when we're done
// We always wait for our initial event to start.
select {
case _, ok := <-obj.init.Input:
if ok {
return fmt.Errorf("unexpected input")
}
obj.init.Input = nil
case <-ctx.Done():
return nil
}
// it seems the different values only update once every 5
// seconds, so that's as often as we need to refresh this!
// TODO: lookup this value if it's something configurable
@@ -124,17 +110,9 @@ func (obj *Load) Stream(ctx context.Context) error {
return nil
}
result, err := obj.Call(ctx, nil)
if err != nil {
if err := obj.init.Event(ctx); err != nil {
return err
}
select {
case obj.init.Output <- result:
case <-ctx.Done():
return nil
}
}
}

31
lang/core/sys/uptime.go
View File

@@ -31,7 +31,6 @@ package coresys
import (
"context"
"fmt"
"time"
"github.com/purpleidea/mgmt/lang/funcs"
@@ -83,22 +82,8 @@ func (obj *Uptime) Init(init *interfaces.Init) error {
return nil
}
// Stream returns the changing values that this fact has over time.
// Stream starts a mainloop and runs Event when it's time to Call() again.
func (obj *Uptime) Stream(ctx context.Context) error {
defer close(obj.init.Output)
// We always wait for our initial event to start.
select {
case _, ok := <-obj.init.Input:
if ok {
return fmt.Errorf("unexpected input")
}
obj.init.Input = nil
case <-ctx.Done():
return nil
}
ticker := time.NewTicker(time.Duration(1) * time.Second)
defer ticker.Stop()
@@ -112,24 +97,16 @@ func (obj *Uptime) Stream(ctx context.Context) error {
case <-startChan:
startChan = nil // disable
case <-ticker.C:
// send
case <-ticker.C: // received the timer event
// pass
case <-ctx.Done():
return nil
}
result, err := obj.Call(ctx, nil)
if err != nil {
if err := obj.init.Event(ctx); err != nil {
return err
}
select {
case obj.init.Output <- result:
case <-ctx.Done():
return nil
}
}
}

34
lang/core/test/fastcount.go
View File

@@ -31,7 +31,6 @@ package coretest
import (
"context"
"fmt"
"sync"
"github.com/purpleidea/mgmt/lang/funcs"
@@ -87,38 +86,23 @@ func (obj *FastCount) Init(init *interfaces.Init) error {
return nil
}
// Stream returns the changing values that this fact has over time.
// Stream starts a mainloop and runs Event when it's time to Call() again.
func (obj *FastCount) Stream(ctx context.Context) error {
defer close(obj.init.Output) // always signal when we're done
// We always wait for our initial event to start.
select {
case _, ok := <-obj.init.Input:
if ok {
return fmt.Errorf("unexpected input")
}
obj.init.Input = nil
case <-ctx.Done():
return nil
}
// streams must generate an initial event on startup
for {
result, err := obj.Call(ctx, nil)
if err != nil {
return err
select {
case <-ctx.Done():
return nil
default:
// run free
}
obj.mutex.Lock()
obj.count++
obj.mutex.Unlock()
select {
case obj.init.Output <- result:
case <-ctx.Done():
return nil
if err := obj.init.Event(ctx); err != nil {
return err
}
}
}

33
lang/core/test/oneinstance.go
View File

@@ -31,7 +31,6 @@ package coretest
import (
"context"
"fmt"
"sync"
"github.com/purpleidea/mgmt/lang/funcs"
@@ -268,38 +267,6 @@ func (obj *OneInstance) Init(init *interfaces.Init) error {
return nil
}
// Stream returns the changing values that this fact has over time.
func (obj *OneInstance) Stream(ctx context.Context) error {
obj.init.Logf("Stream of `%s` @ %p", obj.Name, obj)
defer close(obj.init.Output) // always signal when we're done
// We always wait for our initial event to start.
select {
case _, ok := <-obj.init.Input:
if ok {
return fmt.Errorf("unexpected input")
}
obj.init.Input = nil
case <-ctx.Done():
return nil
}
result, err := obj.Call(ctx, nil)
if err != nil {
return err
}
select {
case obj.init.Output <- result:
case <-ctx.Done():
return nil
}
return nil
}
// Call this fact and return the value if it is possible to do so at this time.
func (obj *OneInstance) Call(ctx context.Context, args []types.Value) (types.Value, error) {
return &types.StrValue{

93
lang/core/value/get.go
View File

@@ -77,7 +77,7 @@ func init() {
funcs.ModuleRegister(ModuleName, GetFloatFuncName, func() interfaces.Func { return &GetFunc{Type: types.TypeFloat} })
}
var _ interfaces.CallableFunc = &GetFunc{}
var _ interfaces.StreamableFunc = &GetFunc{}
// GetFunc is special function which looks up the stored `Any` field in the
// value resource that it gets it from. If it is initialized with a fixed Type
@@ -90,11 +90,8 @@ type GetFunc struct {
init *interfaces.Init
key string
args []types.Value
last types.Value
result types.Value // last calculated output
input chan string // stream of inputs
key *string // the active key
watchChan chan struct{}
}
@@ -216,13 +213,13 @@ func (obj *GetFunc) Info() *interfaces.Info {
// Init runs some startup code for this function.
func (obj *GetFunc) Init(init *interfaces.Init) error {
obj.init = init
obj.input = make(chan string)
obj.watchChan = make(chan struct{}) // sender closes this when Stream ends
return nil
}
// Stream returns the changing values that this func has over time.
func (obj *GetFunc) Stream(ctx context.Context) error {
defer close(obj.init.Output) // the sender closes
ctx, cancel := context.WithCancel(ctx)
defer cancel() // important so that we cleanup the watch when exiting
for {
@@ -230,51 +227,36 @@ func (obj *GetFunc) Stream(ctx context.Context) error {
// TODO: should this first chan be run as a priority channel to
// avoid some sort of glitch? is that even possible? can our
// hostname check with reality (below) fix that?
case input, ok := <-obj.init.Input:
case key, ok := <-obj.input:
if !ok {
obj.init.Input = nil // don't infinite loop back
continue // no more inputs, but don't return!
obj.input = nil // don't infinite loop back
return fmt.Errorf("unexpected close")
}
//if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil {
// return errwrap.Wrapf(err, "wrong function input")
//}
if obj.last != nil && input.Cmp(obj.last) == nil {
continue // value didn't change, skip it
}
obj.last = input // store for next
args, err := interfaces.StructToCallableArgs(input) // []types.Value, error)
if err != nil {
return err
}
obj.args = args
key := args[0].Str()
if key == "" {
return fmt.Errorf("can't use an empty key")
}
if obj.init.Debug {
obj.init.Logf("key: %s", key)
if obj.key != nil && *obj.key == key {
continue // nothing changed
}
// We don't support changing the key over time, since it
// might cause the type to need to be changed.
if obj.key == "" {
obj.key = key // store it
if obj.key == nil {
obj.key = &key // store it
var err error
// Don't send a value right away, wait for the
// first ValueWatch startup event to get one!
obj.watchChan, err = obj.init.Local.ValueWatch(ctx, obj.key) // watch for var changes
obj.watchChan, err = obj.init.Local.ValueWatch(ctx, key) // watch for var changes
if err != nil {
return err
}
} else if obj.key != key {
return fmt.Errorf("can't change key, previously: `%s`", obj.key)
continue // we get values on the watch chan, not here!
}
continue // we get values on the watch chan, not here!
if *obj.key == key {
continue // skip duplicates
}
// *obj.key != key
return fmt.Errorf("can't change key, previously: `%s`", *obj.key)
case _, ok := <-obj.watchChan:
if !ok { // closed
@@ -284,24 +266,10 @@ func (obj *GetFunc) Stream(ctx context.Context) error {
// return errwrap.Wrapf(err, "channel watch failed on `%s`", obj.key)
//}
result, err := obj.Call(ctx, obj.args) // get the value...
if err != nil {
if err := obj.init.Event(ctx); err != nil { // send event
return err
}
// if the result is still the same, skip sending an update...
if obj.result != nil && result.Cmp(obj.result) == nil {
continue // result didn't change
}
obj.result = result // store new result
case <-ctx.Done():
return nil
}
select {
case obj.init.Output <- obj.result: // send
// pass
case <-ctx.Done():
return nil
}
@@ -316,6 +284,18 @@ func (obj *GetFunc) Call(ctx context.Context, args []types.Value) (types.Value,
return nil, fmt.Errorf("not enough args")
}
key := args[0].Str()
if key == "" {
return nil, fmt.Errorf("can't use an empty key")
}
// Check before we send to a chan where we'd need Stream to be running.
if obj.init == nil {
return nil, funcs.ErrCantSpeculate
}
if obj.init.Debug {
obj.init.Logf("key: %s", key)
}
typ, exists := obj.Info().Sig.Out.Map[getFieldNameValue] // type of value field
if !exists || typ == nil {
@@ -323,6 +303,12 @@ func (obj *GetFunc) Call(ctx context.Context, args []types.Value) (types.Value,
return nil, fmt.Errorf("missing type for %s field", getFieldNameValue)
}
select {
case obj.input <- key:
case <-ctx.Done():
return nil, ctx.Err()
}
// The API will pull from the on-disk stored cache if present... This
// value comes from the field in the Value resource... We only have an
// on-disk cache because since functions load before resources do, we'd
@@ -331,9 +317,6 @@ func (obj *GetFunc) Call(ctx context.Context, args []types.Value) (types.Value,
// step that might be needed if the value started out empty...
// TODO: We could even add a stored: bool field in the returned struct!
isReady := true // assume true
if obj.init == nil {
return nil, funcs.ErrCantSpeculate
}
val, err := obj.init.Local.ValueGet(ctx, key)
if err != nil {
return nil, errwrap.Wrapf(err, "channel read failed on `%s`", key)

89
lang/core/world/collect/res.go
View File

@@ -75,10 +75,8 @@ func init() {
type ResFunc struct {
init *interfaces.Init
last types.Value // last value received to use for diff
args []types.Value
kind string
result types.Value // last calculated output
input chan string // stream of inputs
kind *string // the active kind
watchChan chan error
}
@@ -128,13 +126,13 @@ func (obj *ResFunc) Info() *interfaces.Info {
// Init runs some startup code for this function.
func (obj *ResFunc) Init(init *interfaces.Init) error {
obj.init = init
obj.input = make(chan string)
obj.watchChan = make(chan error) // XXX: sender should close this, but did I implement that part yet???
return nil
}
// Stream returns the changing values that this func has over time.
func (obj *ResFunc) Stream(ctx context.Context) error {
defer close(obj.init.Output) // the sender closes
ctx, cancel := context.WithCancel(ctx)
defer cancel() // important so that we cleanup the watch when exiting
for {
@@ -142,50 +140,35 @@ func (obj *ResFunc) Stream(ctx context.Context) error {
// TODO: should this first chan be run as a priority channel to
// avoid some sort of glitch? is that even possible? can our
// hostname check with reality (below) fix that?
case input, ok := <-obj.init.Input:
case kind, ok := <-obj.input:
if !ok {
obj.init.Input = nil // don't infinite loop back
continue // no more inputs, but don't return!
obj.input = nil // don't infinite loop back
return fmt.Errorf("unexpected close")
}
//if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil {
// return errwrap.Wrapf(err, "wrong function input")
//}
if obj.last != nil && input.Cmp(obj.last) == nil {
continue // value didn't change, skip it
}
obj.last = input // store for next
args, err := interfaces.StructToCallableArgs(input) // []types.Value, error)
if err != nil {
return err
}
obj.args = args
kind := args[0].Str()
if kind == "" {
return fmt.Errorf("can't use an empty kind")
}
if obj.init.Debug {
obj.init.Logf("kind: %s", kind)
if obj.kind != nil && *obj.kind == kind {
continue // nothing changed
}
// TODO: support changing the key over time?
if obj.kind == "" {
obj.kind = kind // store it
if obj.kind == nil {
obj.kind = &kind // store
var err error
// Don't send a value right away, wait for the
// first Watch startup event to get one!
obj.watchChan, err = obj.init.World.ResWatch(ctx, obj.kind) // watch for var changes
obj.watchChan, err = obj.init.World.ResWatch(ctx, kind) // watch for var changes
if err != nil {
return err
}
} else if obj.kind != kind {
return fmt.Errorf("can't change kind, previously: `%s`", obj.kind)
continue // we get values on the watch chan, not here!
}
continue // we get values on the watch chan, not here!
if *obj.kind == kind {
continue // skip duplicates
}
// *obj.kind != kind
return fmt.Errorf("can't change kind, previously: `%s`", *obj.kind)
case err, ok := <-obj.watchChan:
if !ok { // closed
@@ -196,27 +179,13 @@ func (obj *ResFunc) Stream(ctx context.Context) error {
return nil
}
if err != nil {
return errwrap.Wrapf(err, "channel watch failed on `%s`", obj.kind)
return errwrap.Wrapf(err, "channel watch failed on `%s`", *obj.kind)
}
result, err := obj.Call(ctx, obj.args) // get the value...
if err != nil {
if err := obj.init.Event(ctx); err != nil { // send event
return err
}
// if the result is still the same, skip sending an update...
if obj.result != nil && result.Cmp(obj.result) == nil {
continue // result didn't change
}
obj.result = result // store new result
case <-ctx.Done():
return nil
}
select {
case obj.init.Output <- obj.result: // send
// pass
case <-ctx.Done():
return nil
}
@@ -238,6 +207,21 @@ func (obj *ResFunc) Call(ctx context.Context, args []types.Value) (types.Value,
return nil, fmt.Errorf("invalid resource kind: %s", kind)
}
// Check before we send to a chan where we'd need Stream to be running.
if obj.init == nil {
return nil, funcs.ErrCantSpeculate
}
if obj.init.Debug {
obj.init.Logf("kind: %s", kind)
}
select {
case obj.input <- kind:
case <-ctx.Done():
return nil, ctx.Err()
}
filters := []*engine.ResFilter{}
filter := &engine.ResFilter{
Kind: kind,
@@ -246,9 +230,6 @@ func (obj *ResFunc) Call(ctx context.Context, args []types.Value) (types.Value,
}
filters = append(filters, filter)
if obj.init == nil {
return nil, funcs.ErrCantSpeculate
}
resOutput, err := obj.init.World.ResCollect(ctx, filters)
if err != nil {
return nil, err

214
lang/core/world/exchange.go
View File

@@ -1,214 +0,0 @@
// Mgmt
// Copyright (C) James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
//
// Additional permission under GNU GPL version 3 section 7
//
// If you modify this program, or any covered work, by linking or combining it
// with embedded mcl code and modules (and that the embedded mcl code and
// modules which link with this program, contain a copy of their source code in
// the authoritative form) containing parts covered by the terms of any other
// license, the licensors of this program grant you additional permission to
// convey the resulting work. Furthermore, the licensors of this program grant
// the original author, James Shubin, additional permission to update this
// additional permission if he deems it necessary to achieve the goals of this
// additional permission.
package coreworld
import (
"context"
"fmt"
"github.com/purpleidea/mgmt/lang/funcs"
"github.com/purpleidea/mgmt/lang/interfaces"
"github.com/purpleidea/mgmt/lang/types"
"github.com/purpleidea/mgmt/util/errwrap"
)
const (
// ExchangeFuncName is the name this function is registered as.
ExchangeFuncName = "exchange"
// arg names...
exchangeArgNameNamespace = "namespace"
exchangeArgNameValue = "value"
)
func init() {
funcs.ModuleRegister(ModuleName, ExchangeFuncName, func() interfaces.Func { return &ExchangeFunc{} })
}
// ExchangeFunc is special function which returns all the values of a given key
// in the exposed world, and sets it's own.
type ExchangeFunc struct {
init *interfaces.Init
namespace string
last types.Value
result types.Value // last calculated output
watchChan chan error
}
// String returns a simple name for this function. This is needed so this struct
// can satisfy the pgraph.Vertex interface.
func (obj *ExchangeFunc) String() string {
return ExchangeFuncName
}
// ArgGen returns the Nth arg name for this function.
func (obj *ExchangeFunc) ArgGen(index int) (string, error) {
seq := []string{exchangeArgNameNamespace, exchangeArgNameValue}
if l := len(seq); index >= l {
return "", fmt.Errorf("index %d exceeds arg length of %d", index, l)
}
return seq[index], nil
}
// Validate makes sure we've built our struct properly. It is usually unused for
// normal functions that users can use directly.
func (obj *ExchangeFunc) Validate() error {
return nil
}
// Info returns some static info about itself.
func (obj *ExchangeFunc) Info() *interfaces.Info {
return &interfaces.Info{
Pure: false, // definitely false
Memo: false,
Fast: false,
Spec: false,
// TODO: do we want to allow this to be statically polymorphic,
// and have value be any type we might want?
// output is map of: hostname => value
Sig: types.NewType(fmt.Sprintf("func(%s str, %s str) map{str: str}", exchangeArgNameNamespace, exchangeArgNameValue)),
Err: obj.Validate(),
}
}
// Init runs some startup code for this function.
func (obj *ExchangeFunc) Init(init *interfaces.Init) error {
obj.init = init
obj.watchChan = make(chan error) // XXX: sender should close this, but did I implement that part yet???
return nil
}
// Stream returns the changing values that this func has over time.
func (obj *ExchangeFunc) Stream(ctx context.Context) error {
defer close(obj.init.Output) // the sender closes
ctx, cancel := context.WithCancel(ctx)
defer cancel()
for {
select {
// TODO: should this first chan be run as a priority channel to
// avoid some sort of glitch? is that even possible? can our
// hostname check with reality (below) fix that?
case input, ok := <-obj.init.Input:
if !ok {
obj.init.Input = nil // don't infinite loop back
continue // no more inputs, but don't return!
}
//if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil {
// return errwrap.Wrapf(err, "wrong function input")
//}
if obj.last != nil && input.Cmp(obj.last) == nil {
continue // value didn't change, skip it
}
obj.last = input // store for next
namespace := input.Struct()[exchangeArgNameNamespace].Str()
if namespace == "" {
return fmt.Errorf("can't use an empty namespace")
}
if obj.init.Debug {
obj.init.Logf("namespace: %s", namespace)
}
// TODO: support changing the namespace over time...
// TODO: possibly removing our stored value there first!
if obj.namespace == "" {
obj.namespace = namespace // store it
var err error
obj.watchChan, err = obj.init.World.StrMapWatch(ctx, obj.namespace) // watch for var changes
if err != nil {
return err
}
} else if obj.namespace != namespace {
return fmt.Errorf("can't change namespace, previously: `%s`", obj.namespace)
}
value := input.Struct()[exchangeArgNameValue].Str()
if obj.init.Debug {
obj.init.Logf("value: %+v", value)
}
if err := obj.init.World.StrMapSet(ctx, obj.namespace, value); err != nil {
return errwrap.Wrapf(err, "namespace write error of `%s` to `%s`", value, obj.namespace)
}
continue // we get values on the watch chan, not here!
case err, ok := <-obj.watchChan:
if !ok { // closed
// XXX: if we close, perhaps the engine is
// switching etcd hosts and we should retry?
// maybe instead we should get an "etcd
// reconnect" signal, and the lang will restart?
return nil
}
if err != nil {
return errwrap.Wrapf(err, "channel watch failed on `%s`", obj.namespace)
}
keyMap, err := obj.init.World.StrMapGet(ctx, obj.namespace)
if err != nil {
return errwrap.Wrapf(err, "channel read failed on `%s`", obj.namespace)
}
var result types.Value
d := types.NewMap(obj.Info().Sig.Out)
for k, v := range keyMap {
key := &types.StrValue{V: k}
val := &types.StrValue{V: v}
if err := d.Add(key, val); err != nil {
return errwrap.Wrapf(err, "map could not add key `%s`, val: `%s`", k, v)
}
}
result = d // put map into interface type
// if the result is still the same, skip sending an update...
if obj.result != nil && result.Cmp(obj.result) == nil {
continue // result didn't change
}
obj.result = result // store new result
case <-ctx.Done():
return nil
}
select {
case obj.init.Output <- obj.result: // send
// pass
case <-ctx.Done():
return nil
}
}
}

98
lang/core/world/getval.go
View File

@@ -55,20 +55,15 @@ func init() {
funcs.ModuleRegister(ModuleName, GetValFuncName, func() interfaces.Func { return &GetValFunc{} })
}
var _ interfaces.CallableFunc = &GetValFunc{}
var _ interfaces.StreamableFunc = &GetValFunc{}
// GetValFunc is special function which returns the value of a given key in the
// exposed world.
type GetValFunc struct {
init *interfaces.Init
key string
args []types.Value
last types.Value
result types.Value // last calculated output
watchChan chan error
input chan string // stream of inputs
key *string // the active key
}
// String returns a simple name for this function. This is needed so this struct
@@ -109,66 +104,53 @@ func (obj *GetValFunc) Info() *interfaces.Info {
// Init runs some startup code for this function.
func (obj *GetValFunc) Init(init *interfaces.Init) error {
obj.init = init
obj.watchChan = make(chan error) // XXX: sender should close this, but did I implement that part yet???
obj.input = make(chan string)
return nil
}
// Stream returns the changing values that this func has over time.
func (obj *GetValFunc) Stream(ctx context.Context) error {
defer close(obj.init.Output) // the sender closes
ctx, cancel := context.WithCancel(ctx)
defer cancel() // important so that we cleanup the watch when exiting
watchChan := make(chan error) // XXX: sender should close this, but did I implement that part yet???
for {
select {
// TODO: should this first chan be run as a priority channel to
// avoid some sort of glitch? is that even possible? can our
// hostname check with reality (below) fix that?
case input, ok := <-obj.init.Input:
case key, ok := <-obj.input:
if !ok {
obj.init.Input = nil // don't infinite loop back
continue // no more inputs, but don't return!
obj.input = nil // don't infinite loop back
return fmt.Errorf("unexpected close")
}
//if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil {
// return errwrap.Wrapf(err, "wrong function input")
//}
if obj.last != nil && input.Cmp(obj.last) == nil {
continue // value didn't change, skip it
}
obj.last = input // store for next
args, err := interfaces.StructToCallableArgs(input) // []types.Value, error)
if err != nil {
return err
}
obj.args = args
key := args[0].Str()
if key == "" {
return fmt.Errorf("can't use an empty key")
}
if obj.init.Debug {
obj.init.Logf("key: %s", key)
if obj.key != nil && *obj.key == key {
continue // nothing changed
}
// TODO: support changing the key over time...
if obj.key == "" {
obj.key = key // store it
if obj.key == nil {
obj.key = &key // store
var err error
// Don't send a value right away, wait for the
// first ValueWatch startup event to get one!
obj.watchChan, err = obj.init.World.StrWatch(ctx, obj.key) // watch for var changes
watchChan, err = obj.init.World.StrWatch(ctx, key) // watch for var changes
if err != nil {
return err
}
} else if obj.key != key {
return fmt.Errorf("can't change key, previously: `%s`", obj.key)
continue // we get values on the watch chan, not here!
}
continue // we get values on the watch chan, not here!
if *obj.key == key {
continue // skip duplicates
}
case err, ok := <-obj.watchChan:
// *obj.key != key
return fmt.Errorf("can't change key, previously: `%s`", *obj.key)
case err, ok := <-watchChan:
if !ok { // closed
// XXX: if we close, perhaps the engine is
// switching etcd hosts and we should retry?
@@ -177,27 +159,13 @@ func (obj *GetValFunc) Stream(ctx context.Context) error {
return nil
}
if err != nil {
return errwrap.Wrapf(err, "channel watch failed on `%s`", obj.key)
return errwrap.Wrapf(err, "channel watch failed on `%s`", *obj.key)
}
result, err := obj.Call(ctx, obj.args) // get the value...
if err != nil {
if err := obj.init.Event(ctx); err != nil { // send event
return err
}
// if the result is still the same, skip sending an update...
if obj.result != nil && result.Cmp(obj.result) == nil {
continue // result didn't change
}
obj.result = result // store new result
case <-ctx.Done():
return nil
}
select {
case obj.init.Output <- obj.result: // send
// pass
case <-ctx.Done():
return nil
}
@@ -213,9 +181,25 @@ func (obj *GetValFunc) Call(ctx context.Context, args []types.Value) (types.Valu
}
key := args[0].Str()
exists := true // assume true
if key == "" {
return nil, fmt.Errorf("can't use an empty key")
}
// Check before we send to a chan where we'd need Stream to be running.
if obj.init == nil {
return nil, funcs.ErrCantSpeculate
}
if obj.init.Debug {
obj.init.Logf("key: %s", key)
}
select {
case obj.input <- key:
case <-ctx.Done():
return nil, ctx.Err()
}
val, err := obj.init.World.StrGet(ctx, key)
if err != nil && obj.init.World.StrIsNotExist(err) {
exists = false // val doesn't exist

106
lang/core/world/kvlookup.go
View File

@@ -51,18 +51,17 @@ func init() {
funcs.ModuleRegister(ModuleName, KVLookupFuncName, func() interfaces.Func { return &KVLookupFunc{} })
}
var _ interfaces.CallableFunc = &KVLookupFunc{}
var _ interfaces.StreamableFunc = &KVLookupFunc{}
// KVLookupFunc is special function which returns all the values of a given key
// in the exposed world. It is similar to exchange, but it does not set a key.
// Since exchange has been deprecated, you will want to use this in conjunction
// with a resource to set the desired value.
type KVLookupFunc struct {
init *interfaces.Init
namespace string
args []types.Value
last types.Value
result types.Value // last calculated output
input chan string // stream of inputs
namespace *string // the active namespace
watchChan chan error
}
@@ -104,74 +103,43 @@ func (obj *KVLookupFunc) Info() *interfaces.Info {
// Init runs some startup code for this function.
func (obj *KVLookupFunc) Init(init *interfaces.Init) error {
obj.init = init
obj.input = make(chan string)
obj.watchChan = make(chan error) // XXX: sender should close this, but did I implement that part yet???
return nil
}
// Stream returns the changing values that this func has over time.
func (obj *KVLookupFunc) Stream(ctx context.Context) error {
defer close(obj.init.Output) // the sender closes
ctx, cancel := context.WithCancel(ctx)
defer cancel()
defer cancel() // important so that we cleanup the watch when exiting
for {
select {
// TODO: should this first chan be run as a priority channel to
// avoid some sort of glitch? is that even possible? can our
// hostname check with reality (below) fix that?
case input, ok := <-obj.init.Input:
case namespace, ok := <-obj.input:
if !ok {
obj.init.Input = nil // don't infinite loop back
continue // no more inputs, but don't return!
}
//if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil {
// return errwrap.Wrapf(err, "wrong function input")
//}
if obj.last != nil && input.Cmp(obj.last) == nil {
continue // value didn't change, skip it
}
obj.last = input // store for next
args, err := interfaces.StructToCallableArgs(input) // []types.Value, error)
if err != nil {
return err
}
obj.args = args
namespace := args[0].Str()
if namespace == "" {
return fmt.Errorf("can't use an empty namespace")
}
if obj.init.Debug {
obj.init.Logf("namespace: %s", namespace)
obj.input = nil // don't infinite loop back
return fmt.Errorf("unexpected close")
}
// TODO: support changing the namespace over time...
// TODO: possibly removing our stored value there first!
if obj.namespace == "" {
obj.namespace = namespace // store it
if obj.namespace == nil {
obj.namespace = &namespace // store it
var err error
obj.watchChan, err = obj.init.World.StrMapWatch(ctx, obj.namespace) // watch for var changes
obj.watchChan, err = obj.init.World.StrMapWatch(ctx, namespace) // watch for var changes
if err != nil {
return err
}
result, err := obj.Call(ctx, obj.args) // build the map...
if err != nil {
return err
}
select {
case obj.init.Output <- result: // send one!
// pass
case <-ctx.Done():
return nil
}
} else if obj.namespace != namespace {
return fmt.Errorf("can't change namespace, previously: `%s`", obj.namespace)
continue // we get values on the watch chan, not here!
}
continue // we get values on the watch chan, not here!
if *obj.namespace == namespace {
continue // skip duplicates
}
// *obj.namespace != namespace
return fmt.Errorf("can't change namespace, previously: `%s`", *obj.namespace)
case err, ok := <-obj.watchChan:
if !ok { // closed
@@ -182,27 +150,13 @@ func (obj *KVLookupFunc) Stream(ctx context.Context) error {
return nil
}
if err != nil {
return errwrap.Wrapf(err, "channel watch failed on `%s`", obj.namespace)
return errwrap.Wrapf(err, "channel watch failed on `%s`", *obj.namespace)
}
result, err := obj.Call(ctx, obj.args) // build the map...
if err != nil {
if err := obj.init.Event(ctx); err != nil { // send event
return err
}
// if the result is still the same, skip sending an update...
if obj.result != nil && result.Cmp(obj.result) == nil {
continue // result didn't change
}
obj.result = result // store new result
case <-ctx.Done():
return nil
}
select {
case obj.init.Output <- obj.result: // send
// pass
case <-ctx.Done():
return nil
}
@@ -217,9 +171,25 @@ func (obj *KVLookupFunc) Call(ctx context.Context, args []types.Value) (types.Va
return nil, fmt.Errorf("not enough args")
}
namespace := args[0].Str()
if namespace == "" {
return nil, fmt.Errorf("can't use an empty namespace")
}
// Check before we send to a chan where we'd need Stream to be running.
if obj.init == nil {
return nil, funcs.ErrCantSpeculate
}
if obj.init.Debug {
obj.init.Logf("namespace: %s", namespace)
}
select {
case obj.input <- namespace:
case <-ctx.Done():
return nil, ctx.Err()
}
keyMap, err := obj.init.World.StrMapGet(ctx, namespace)
if err != nil {
return nil, errwrap.Wrapf(err, "channel read failed on `%s`", namespace)

445
lang/core/world/schedule.go
View File

@@ -27,24 +27,12 @@
// additional permission if he deems it necessary to achieve the goals of this
// additional permission.
// test with:
// time ./mgmt run --hostname h1 --tmp-prefix --no-pgp lang examples/lang/schedule0.mcl
// time ./mgmt run --hostname h2 --seeds=http://127.0.0.1:2379 --client-urls=http://127.0.0.1:2381 --server-urls=http://127.0.0.1:2382 --tmp-prefix --no-pgp lang examples/lang/schedule0.mcl
// time ./mgmt run --hostname h3 --seeds=http://127.0.0.1:2379 --client-urls=http://127.0.0.1:2383 --server-urls=http://127.0.0.1:2384 --tmp-prefix --no-pgp lang examples/lang/schedule0.mcl
// kill h2 (should see h1 and h3 pick [h1, h3] instead)
// restart h2 (should see [h1, h3] as before)
// kill h3 (should see h1 and h2 pick [h1, h2] instead)
// restart h3 (should see [h1, h2] as before)
// kill h3
// kill h2
// kill h1... all done!
package coreworld
import (
"context"
"fmt"
"sort"
"sync"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/etcd/scheduler" // XXX: abstract this if possible
@@ -58,48 +46,25 @@ const (
// ScheduleFuncName is the name this function is registered as.
ScheduleFuncName = "schedule"
// DefaultStrategy is the strategy to use if none has been specified.
DefaultStrategy = "rr"
// StrictScheduleOpts specifies whether the opts passed into the
// scheduler must be strictly what we're expecting, and nothing more.
// If this was false, then we'd allow an opts struct that had a field
// that wasn't used by the scheduler. This could be useful if we need to
// migrate to a newer version of the function. It's probably best to
// keep this strict.
StrictScheduleOpts = true
// arg names...
scheduleArgNameNamespace = "namespace"
scheduleArgNameOpts = "opts"
)
func init() {
funcs.ModuleRegister(ModuleName, ScheduleFuncName, func() interfaces.Func { return &ScheduleFunc{} })
}
var _ interfaces.BuildableFunc = &ScheduleFunc{} // ensure it meets this expectation
// ScheduleFunc is special function which determines where code should run in
// the cluster.
type ScheduleFunc struct {
Type *types.Type // this is the type of opts used if specified
built bool // was this function built yet?
init *interfaces.Init
args []types.Value
init *interfaces.Init
world engine.SchedulerWorld
namespace string
scheduler *scheduler.Result
input chan string // stream of inputs
namespace *string // the active namespace
last types.Value
result types.Value // last calculated output
watchChan chan *schedulerResult
mutex *sync.Mutex // guards value
value []string // list of hosts
}
// String returns a simple name for this function. This is needed so this struct
@@ -108,19 +73,9 @@ func (obj *ScheduleFunc) String() string {
return ScheduleFuncName
}
// validOpts returns the available mapping of valid opts fields to types.
func (obj *ScheduleFunc) validOpts() map[string]*types.Type {
return map[string]*types.Type{
"strategy": types.TypeStr,
"max": types.TypeInt,
"reuse": types.TypeBool,
"ttl": types.TypeInt,
}
}
// ArgGen returns the Nth arg name for this function.
func (obj *ScheduleFunc) ArgGen(index int) (string, error) {
seq := []string{scheduleArgNameNamespace, scheduleArgNameOpts} // 2nd arg is optional
seq := []string{scheduleArgNameNamespace}
if l := len(seq); index >= l {
return "", fmt.Errorf("index %d exceeds arg length of %d", index, l)
}
@@ -130,163 +85,24 @@ func (obj *ScheduleFunc) ArgGen(index int) (string, error) {
// helper
func (obj *ScheduleFunc) sig() *types.Type {
sig := types.NewType(fmt.Sprintf("func(%s str) []str", scheduleArgNameNamespace)) // simplest form
if obj.Type != nil {
sig = types.NewType(fmt.Sprintf("func(%s str, %s %s) []str", scheduleArgNameNamespace, scheduleArgNameOpts, obj.Type.String()))
}
return sig
}
// FuncInfer takes partial type and value information from the call site of this
// function so that it can build an appropriate type signature for it. The type
// signature may include unification variables.
func (obj *ScheduleFunc) FuncInfer(partialType *types.Type, partialValues []types.Value) (*types.Type, []*interfaces.UnificationInvariant, error) {
// func(namespace str) []str
// OR
// func(namespace str, opts ?1) []str
if l := len(partialValues); l < 1 || l > 2 {
return nil, nil, fmt.Errorf("must have at either one or two args")
}
var typ *types.Type
if len(partialValues) == 1 {
typ = types.NewType(fmt.Sprintf("func(%s str) []str", scheduleArgNameNamespace))
}
if len(partialValues) == 2 {
typ = types.NewType(fmt.Sprintf("func(%s str, %s ?1) []str", scheduleArgNameNamespace, scheduleArgNameOpts))
}
return typ, []*interfaces.UnificationInvariant{}, nil
}
// Build is run to turn the polymorphic, undetermined function, into the
// specific statically typed version. It is usually run after Unify completes,
// and must be run before Info() and any of the other Func interface methods are
// used. This function is idempotent, as long as the arg isn't changed between
// runs.
func (obj *ScheduleFunc) Build(typ *types.Type) (*types.Type, error) {
// typ is the KindFunc signature we're trying to build...
if typ.Kind != types.KindFunc {
return nil, fmt.Errorf("input type must be of kind func")
}
if len(typ.Ord) != 1 && len(typ.Ord) != 2 {
return nil, fmt.Errorf("the schedule function needs either one or two args")
}
if typ.Out == nil {
return nil, fmt.Errorf("return type of function must be specified")
}
if typ.Map == nil {
return nil, fmt.Errorf("invalid input type")
}
if err := typ.Out.Cmp(types.TypeListStr); err != nil {
return nil, errwrap.Wrapf(err, "return type must be a list of strings")
}
tNamespace, exists := typ.Map[typ.Ord[0]]
if !exists || tNamespace == nil {
return nil, fmt.Errorf("first arg must be specified")
}
if len(typ.Ord) == 1 {
obj.Type = nil
obj.built = true
return obj.sig(), nil // done early, 2nd arg is absent!
}
tOpts, exists := typ.Map[typ.Ord[1]]
if !exists || tOpts == nil {
return nil, fmt.Errorf("second argument was missing")
}
if tOpts.Kind != types.KindStruct {
return nil, fmt.Errorf("second argument must be of kind struct")
}
validOpts := obj.validOpts()
if StrictScheduleOpts {
// strict opts field checking!
for _, name := range tOpts.Ord {
t := tOpts.Map[name]
value, exists := validOpts[name]
if !exists {
return nil, fmt.Errorf("unexpected opts field: `%s`", name)
}
if err := t.Cmp(value); err != nil {
return nil, errwrap.Wrapf(err, "expected different type for opts field: `%s`", name)
}
}
} else {
// permissive field checking...
validOptsSorted := []string{}
for name := range validOpts {
validOptsSorted = append(validOptsSorted, name)
}
sort.Strings(validOptsSorted)
for _, name := range validOptsSorted {
value := validOpts[name] // type
t, exists := tOpts.Map[name]
if !exists {
continue // ignore it
}
// if it exists, check the type
if err := t.Cmp(value); err != nil {
return nil, errwrap.Wrapf(err, "expected different type for opts field: `%s`", name)
}
}
}
obj.Type = tOpts // type of opts struct, even an empty: `struct{}`
obj.built = true
return obj.sig(), nil
}
// Copy is implemented so that the type value is not lost if we copy this
// function.
func (obj *ScheduleFunc) Copy() interfaces.Func {
return &ScheduleFunc{
Type: obj.Type, // don't copy because we use this after unification
built: obj.built,
init: obj.init, // likely gets overwritten anyways
}
}
// Validate tells us if the input struct takes a valid form.
func (obj *ScheduleFunc) Validate() error {
if !obj.built {
return fmt.Errorf("function wasn't built yet")
}
// obj.Type can be nil if no 2nd arg is given, or a struct (even empty!)
if obj.Type != nil && obj.Type.Kind != types.KindStruct { // build must be run first
return fmt.Errorf("type must be nil or a struct")
}
return nil
}
// Info returns some static info about itself. Build must be called before this
// will return correct data.
func (obj *ScheduleFunc) Info() *interfaces.Info {
// Since this function implements FuncInfer we want sig to return nil to
// avoid an accidental return of unification variables when we should be
// getting them from FuncInfer, and not from here. (During unification!)
var sig *types.Type
if obj.built {
sig = obj.sig() // helper
}
return &interfaces.Info{
Pure: false, // definitely false
Memo: false,
Fast: false,
Spec: false,
// output is list of hostnames chosen
Sig: sig, // func kind
Sig: obj.sig(), // func kind
Err: obj.Validate(),
}
}
@@ -294,155 +110,58 @@ func (obj *ScheduleFunc) Info() *interfaces.Info {
// Init runs some startup code for this function.
func (obj *ScheduleFunc) Init(init *interfaces.Init) error {
obj.init = init
world, ok := obj.init.World.(engine.SchedulerWorld)
if !ok {
return fmt.Errorf("world backend does not support the SchedulerWorld interface")
}
obj.world = world
obj.watchChan = make(chan *schedulerResult)
obj.input = make(chan string)
obj.mutex = &sync.Mutex{}
obj.value = []string{} // empty
//obj.init.Debug = true // use this for local debugging
return nil
}
// Stream returns the changing values that this func has over time.
func (obj *ScheduleFunc) Stream(ctx context.Context) error {
defer close(obj.init.Output) // the sender closes
ctx, cancel := context.WithCancel(ctx)
defer cancel() // important so that we cleanup the watch when exiting
watchChan := make(chan *scheduler.ScheduledResult) // XXX: sender should close this, but did I implement that part yet???
for {
select {
// TODO: should this first chan be run as a priority channel to
// avoid some sort of glitch? is that even possible? can our
// hostname check with reality (below) fix that?
case input, ok := <-obj.init.Input:
case namespace, ok := <-obj.input:
if !ok {
obj.init.Input = nil // don't infinite loop back
continue // no more inputs, but don't return!
}
//if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil {
// return errwrap.Wrapf(err, "wrong function input")
//}
if obj.last != nil && input.Cmp(obj.last) == nil {
continue // value didn't change, skip it
}
obj.last = input // store for next
args, err := interfaces.StructToCallableArgs(input) // []types.Value, error)
if err != nil {
return err
}
obj.args = args
namespace := args[0].Str()
//namespace := input.Struct()[scheduleArgNameNamespace].Str()
if namespace == "" {
return fmt.Errorf("can't use an empty namespace")
}
opts := make(map[string]types.Value) // empty "struct"
if val, exists := input.Struct()[scheduleArgNameOpts]; exists {
opts = val.Struct()
}
if obj.init.Debug {
obj.init.Logf("namespace: %s", namespace)
}
schedulerOpts := []scheduler.Option{}
// don't add bad or zero-value options
defaultStrategy := true
if val, exists := opts["strategy"]; exists {
if strategy := val.Str(); strategy != "" {
if obj.init.Debug {
obj.init.Logf("opts: strategy: %s", strategy)
}
defaultStrategy = false
schedulerOpts = append(schedulerOpts, scheduler.StrategyKind(strategy))
}
}
if defaultStrategy { // we always need to add one!
schedulerOpts = append(schedulerOpts, scheduler.StrategyKind(DefaultStrategy))
}
if val, exists := opts["max"]; exists {
// TODO: check for overflow
if max := int(val.Int()); max > 0 {
if obj.init.Debug {
obj.init.Logf("opts: max: %d", max)
}
schedulerOpts = append(schedulerOpts, scheduler.MaxCount(max))
}
}
if val, exists := opts["reuse"]; exists {
reuse := val.Bool()
if obj.init.Debug {
obj.init.Logf("opts: reuse: %t", reuse)
}
schedulerOpts = append(schedulerOpts, scheduler.ReuseLease(reuse))
}
if val, exists := opts["ttl"]; exists {
// TODO: check for overflow
if ttl := int(val.Int()); ttl > 0 {
if obj.init.Debug {
obj.init.Logf("opts: ttl: %d", ttl)
}
schedulerOpts = append(schedulerOpts, scheduler.SessionTTL(ttl))
}
obj.input = nil // don't infinite loop back
return fmt.Errorf("unexpected close")
}
// TODO: support changing the namespace over time...
// TODO: possibly removing our stored value there first!
if obj.namespace == "" {
obj.namespace = namespace // store it
if obj.init.Debug {
obj.init.Logf("starting scheduler...")
}
if obj.namespace == nil {
obj.namespace = &namespace // store it
var err error
obj.scheduler, err = obj.world.Scheduler(obj.namespace, schedulerOpts...)
watchChan, err = obj.world.Scheduled(ctx, namespace) // watch for var changes
if err != nil {
return errwrap.Wrapf(err, "can't create scheduler")
return err
}
// process the stream of scheduling output...
go func() {
defer close(obj.watchChan)
// XXX: maybe we could share the parent
// ctx, but I have to work out the
// ordering logic first. For now this is
// just a port of what it was before.
newCtx, cancel := context.WithCancel(context.Background())
go func() {
defer cancel() // unblock Next()
defer obj.scheduler.Shutdown()
select {
case <-ctx.Done():
return
}
}()
for {
hosts, err := obj.scheduler.Next(newCtx)
select {
case obj.watchChan <- &schedulerResult{
hosts: hosts,
err: err,
}:
case <-ctx.Done():
return
}
}
}()
} else if obj.namespace != namespace {
return fmt.Errorf("can't change namespace, previously: `%s`", obj.namespace)
continue // we get values on the watch chan, not here!
}
continue // we send values on the watch chan, not here!
if *obj.namespace == namespace {
continue // skip duplicates
}
case schedulerResult, ok := <-obj.watchChan:
// *obj.namespace != namespace
return fmt.Errorf("can't change namespace, previously: `%s`", *obj.namespace)
case scheduledResult, ok := <-watchChan:
if !ok { // closed
// XXX: maybe etcd reconnected? (fix etcd implementation)
@@ -452,52 +171,76 @@ func (obj *ScheduleFunc) Stream(ctx context.Context) error {
// reconnect" signal, and the lang will restart?
return nil
}
if err := schedulerResult.err; err != nil {
if err == scheduler.ErrEndOfResults {
//return nil // TODO: we should probably fix the reconnect issue and use this here
return fmt.Errorf("scheduler shutdown, reconnect bug?") // XXX: fix etcd reconnects
}
return errwrap.Wrapf(err, "channel watch failed on `%s`", obj.namespace)
if scheduledResult == nil {
return fmt.Errorf("unexpected nil result")
}
if err := scheduledResult.Err; err != nil {
return errwrap.Wrapf(err, "scheduler result error")
}
if obj.init.Debug {
obj.init.Logf("got hosts: %+v", schedulerResult.hosts)
obj.init.Logf("got hosts: %+v", scheduledResult.Hosts)
}
obj.mutex.Lock()
obj.value = scheduledResult.Hosts // store it
obj.mutex.Unlock()
if err := obj.init.Event(ctx); err != nil { // send event
return err
}
var result types.Value
l := types.NewList(obj.Info().Sig.Out)
for _, val := range schedulerResult.hosts {
if err := l.Add(&types.StrValue{V: val}); err != nil {
return errwrap.Wrapf(err, "list could not add val: `%s`", val)
}
}
result = l // set list as result
if obj.init.Debug {
obj.init.Logf("result: %+v", result)
}
// if the result is still the same, skip sending an update...
if obj.result != nil && result.Cmp(obj.result) == nil {
continue // result didn't change
}
obj.result = result // store new result
case <-ctx.Done():
return nil
}
select {
case obj.init.Output <- obj.result: // send
// pass
case <-ctx.Done():
return nil
}
}
}
// schedulerResult combines our internal events into a single message packet.
type schedulerResult struct {
hosts []string
err error
// Call this function with the input args and return the value if it is possible
// to do so at this time.
func (obj *ScheduleFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) {
if len(args) < 1 {
return nil, fmt.Errorf("not enough args")
}
namespace := args[0].Str()
if namespace == "" {
return nil, fmt.Errorf("can't use an empty namespace")
}
// Check before we send to a chan where we'd need Stream to be running.
if obj.init == nil {
return nil, funcs.ErrCantSpeculate
}
if obj.init.Debug {
obj.init.Logf("namespace: %s", namespace)
}
// Tell the Stream what we're watching now... This doesn't block because
// Stream should always be ready to consume unless it's closing down...
// If it dies, then a ctx closure should come soon.
select {
case obj.input <- namespace:
case <-ctx.Done():
return nil, ctx.Err()
}
obj.mutex.Lock() // TODO: could be a read lock
value := obj.value // initially we might get an empty list
obj.mutex.Unlock()
var result types.Value
l := types.NewList(obj.Info().Sig.Out)
for _, val := range value {
if err := l.Add(&types.StrValue{V: val}); err != nil {
return nil, errwrap.Wrapf(err, "list could not add val: `%s`", val)
}
}
result = l // set list as result
if obj.init.Debug {
obj.init.Logf("result: %+v", result)
}
return result, nil
}