From 790b7199ca894a7af00b8f08742062773cba72a6 Mon Sep 17 00:00:00 2001 From: James Shubin Date: Tue, 9 Sep 2025 02:46:59 -0400 Subject: [PATCH] lang: New function engine This mega patch primarily introduces a new function engine. The main reasons for this new engine are: 1) Massively improved performance with lock-contended graphs. Certain large function graphs could have very high lock-contention which turned out to be much slower than I would have liked. This new algorithm happens to be basically lock-free, so that's another helpful improvement. 2) Glitch-free function graphs. The function graphs could "glitch" (an FRP term) which could be undesirable in theory. In practice this was never really an issue, and I've not explicitly guaranteed that the new graphs are provably glitch-free, but in practice things are a lot more consistent. 3) Simpler graph shape. The new graphs don't require the private channels. This makes understanding the graphs a lot easier. 4) Branched graphs only run half. Previously we would run two pure side of an if statement, and while this was mostly meant as an early experiment, it stayed in for far too long and now was the right time to remove this. This also means our graphs are much smaller and more efficient too. Note that this changed the function API slightly. Everything has been ported. It's possible that we introduce a new API in the future, but it is unexpected to cause removal of the two current APIs. In addition, we finally split out the "schedule" aspect from world.schedule(). The "pick me" aspects now happen in a separate resource, rather than as a yucky side-effect in the function. This also lets us more precisely choose when we're scheduled, and we can observe without being chosen too. As usual many thanks to Sam for helping through some of the algorithmic graph shape issues! --- docs/function-guide.md | 75 +- engine/graph/actions.go | 2 +- engine/resources/kv.go | 2 +- engine/resources/schedule.go | 349 +++ engine/world.go | 5 +- etcd/scheduler/scheduler.go | 197 +- etcd/world.go | 8 +- examples/lang/exchange0.mcl | 20 - examples/lang/history1.mcl | 4 +- examples/lang/hysteresis1.mcl | 4 +- examples/lang/map-iterator0.mcl | 14 +- examples/lang/schedule0.mcl | 38 +- examples/lang/system1.mcl | 11 + gapi/empty/empty.go | 18 +- lang/ast/structs.go | 140 +- lang/core/collect.go | 90 +- lang/core/core_test.go | 1012 --------- lang/core/datetime/now.go | 27 +- lang/core/datetime/str_now.go | 130 ++ lang/core/deploy/abspath.go | 58 +- lang/core/deploy/readfile.go | 62 - lang/core/deploy/readfileabs.go | 62 - lang/core/example/flipflop.go | 29 +- lang/core/example/vumeter.go | 73 +- lang/core/fmt/printf.go | 47 - lang/core/golang/template.go | 50 - lang/core/history.go | 240 +- lang/core/iter/filter.go | 312 +-- lang/core/iter/map.go | 247 +-- lang/core/iter/range.go | 44 - lang/core/local/pool.go | 43 +- lang/core/local/vardir.go | 42 - lang/core/lookup.go | 20 +- lang/core/lookup_default.go | 20 +- lang/core/os/modinfo.go | 63 +- lang/core/os/readfile.go | 115 +- lang/core/os/readfilewait.go | 117 +- lang/core/os/system.go | 114 +- lang/core/random1.go | 114 +- lang/core/struct_lookup.go | 56 - lang/core/struct_lookup_optional.go | 65 - lang/core/sys/cpucount.go | 37 +- lang/core/sys/cpucount_test.go | 41 - lang/core/sys/hostname.go | 27 +- lang/core/sys/load.go | 26 +- lang/core/sys/uptime.go | 31 +- lang/core/test/fastcount.go | 34 +- lang/core/test/oneinstance.go | 33 - lang/core/value/get.go | 93 +- lang/core/world/collect/res.go | 89 +- lang/core/world/exchange.go | 214 -- lang/core/world/getval.go | 98 +- lang/core/world/kvlookup.go | 106 +- lang/core/world/schedule.go | 445 +--- lang/funcs/dage/dage.go | 1959 ++++++----------- lang/funcs/dage/dage_test.go | 807 ------- lang/funcs/operators/operators.go | 64 - lang/funcs/structs/call.go | 149 +- lang/funcs/structs/channel_based_sink.go | 156 -- lang/funcs/structs/channel_based_source.go | 125 -- lang/funcs/structs/composite.go | 69 - lang/funcs/structs/const.go | 16 - lang/funcs/structs/exprif.go | 100 +- lang/funcs/structs/for.go | 140 +- lang/funcs/structs/forkv.go | 144 +- lang/funcs/structs/output.go | 42 +- lang/funcs/structs/stmtif.go | 163 ++ lang/funcs/structs/util.go | 29 +- lang/funcs/txn/txn.go | 26 +- lang/funcs/txn/txn_test.go | 13 +- lang/funcs/wrapped/wrapped.go | 63 - lang/gapi/gapi.go | 23 +- lang/interfaces/error.go | 5 + lang/interfaces/func.go | 118 +- lang/interpret/interpret.go | 2 + lang/interpret_test.go | 328 ++- .../TestAstFunc1/changing-func.txtar | 22 +- .../interpret_test/TestAstFunc1/graph10.txtar | 2 + .../interpret_test/TestAstFunc1/graph11.txtar | 4 +- .../interpret_test/TestAstFunc1/graph12.txtar | 4 +- lang/interpret_test/TestAstFunc1/graph4.txtar | 2 + lang/interpret_test/TestAstFunc1/graph7.txtar | 10 +- .../TestAstFunc1/importscope0.txtar | 8 +- .../TestAstFunc1/importscope2.txtar | 8 +- .../TestAstFunc1/returned-func.txtar | 4 + .../TestAstFunc1/returned-lambda.txtar | 4 + .../TestAstFunc1/shadowing1.txtar | 2 + .../TestAstFunc1/shadowing2.txtar | 3 +- lang/interpret_test/TestAstFunc1/shape0.txtar | 1 + lang/interpret_test/TestAstFunc1/shape5.txtar | 8 +- lang/interpret_test/TestAstFunc1/shape6.txtar | 18 +- lang/interpret_test/TestAstFunc1/shape7.txtar | 18 +- .../TestAstFunc1/simple-func1.txtar | 6 +- .../TestAstFunc1/simple-func2.txtar | 16 +- .../TestAstFunc1/simple-lambda1.txtar | 6 +- .../TestAstFunc1/simple-lambda2.txtar | 15 +- .../TestAstFunc1/slow_unification0.txtar | 33 +- .../TestAstFunc1/static-function0.txtar | 16 +- .../class-include-as-vars-simple.txtar | 14 + .../TestAstFunc2/consistency0.txtar | 13 + .../TestAstFunc2/map-ignore-arg.txtar | 2 + lang/lang.go | 197 +- lang/lang_test.go | 15 +- lang/types/full/full.go | 6 +- lib/main.go | 3 + puppet/gapi.go | 19 +- test/shell/exchange.sh | 68 - test/shell/exchange0.mcl | 20 - yamlgraph/gapi.go | 15 +- 109 files changed, 3632 insertions(+), 6904 deletions(-) create mode 100644 engine/resources/schedule.go delete mode 100644 examples/lang/exchange0.mcl create mode 100644 examples/lang/system1.mcl delete mode 100644 lang/core/core_test.go create mode 100644 lang/core/datetime/str_now.go delete mode 100644 lang/core/world/exchange.go delete mode 100644 lang/funcs/dage/dage_test.go delete mode 100644 lang/funcs/structs/channel_based_sink.go delete mode 100644 lang/funcs/structs/channel_based_source.go create mode 100644 lang/funcs/structs/stmtif.go create mode 100644 lang/interpret_test/TestAstFunc2/class-include-as-vars-simple.txtar create mode 100644 lang/interpret_test/TestAstFunc2/consistency0.txtar delete mode 100755 test/shell/exchange.sh delete mode 100644 test/shell/exchange0.mcl diff --git a/docs/function-guide.md b/docs/function-guide.md index 39c0a3cd..d7d49c58 100644 --- a/docs/function-guide.md +++ b/docs/function-guide.md @@ -177,66 +177,69 @@ func (obj *FooFunc) Init(init *interfaces.Init) error { } ``` +### Call + +Call is run when you want to return a new value from the function. It takes the +input arguments to the function. + +#### Example + +```golang +func (obj *FooFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) { + return &types.StrValue{ // Our type system "str" (string) value. + V: strconv.FormatInt(args[0].Int(), 10), // a golang string + }, nil +} +``` + ### Stream ```golang Stream(context.Context) error ``` -`Stream` is where the real _work_ is done. This method is started by the -language function engine. It will run this function while simultaneously sending -it values on the `Input` channel. It will only send a complete set of input -values. You should send a value to the output channel when you have decided that -one should be produced. Make sure to only use input values of the expected type -as declared in the `Info` struct, and send values of the similarly declared -appropriate return type. Failure to do so will may result in a panic and -sadness. You must shutdown if the input context cancels. You must close the -`Output` channel if you are done generating new values and/or when you shutdown. +`Stream` is where any evented work is done. This method is started by the +function engine. It will run this function once. It should call the +`obj.init.Event()` method when it believes the function engine should run +`Call()` again. + +Implementing this is not required if you don't have events. + +If the `ctx` closes, you must shutdown as soon as possible. #### Example ```golang -// Stream returns the single value that was generated and then closes. +// Stream starts a mainloop and runs Event when it's time to Call() again. func (obj *FooFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes - var result string + + ticker := time.NewTicker(time.Duration(1) * time.Second) + defer ticker.Stop() + + // streams must generate an initial event on startup + // even though ticker will send one, we want to be faster to first event + startChan := make(chan struct{}) // start signal + close(startChan) // kick it off! + for { select { - case input, ok := <-obj.init.Input: - if !ok { - return nil // can't output any more - } + case <-startChan: + startChan = nil // disable - ix := input.Struct()["a"].Int() - if ix < 0 { - return fmt.Errorf("we can't deal with negatives") - } - - result = fmt.Sprintf("the input is: %d", ix) + case <-ticker.C: // received the timer event + // pass case <-ctx.Done(): return nil } - select { - case obj.init.Output <- &types.StrValue{ - V: result, - }: - - case <-ctx.Done(): - return nil + if err := obj.init.Event(ctx); err != nil { + return err } } } ``` -As you can see, we read our inputs from the `input` channel, and write to the -`output` channel. Our code is careful to never block or deadlock, and can always -exit if a close signal is requested. It also cleans up after itself by closing -the `output` channel when it is done using it. This is done easily with `defer`. -If it notices that the `input` channel closes, then it knows that no more input -values are coming and it can consider shutting down early. - ## Further considerations There is some additional information that any function author will need to know. diff --git a/engine/graph/actions.go b/engine/graph/actions.go index 99c82302..818612ad 100644 --- a/engine/graph/actions.go +++ b/engine/graph/actions.go @@ -437,7 +437,7 @@ func (obj *Engine) Worker(vertex pgraph.Vertex) error { //defer close(state.stopped) // done signal - state.cuid = obj.Converger.Register() // XXX RACE READ + state.cuid = obj.Converger.Register() state.tuid = obj.Converger.Register() // must wait for all users of the cuid to finish *before* we unregister! // as a result, this defer happens *before* the below wait group Wait... diff --git a/engine/resources/kv.go b/engine/resources/kv.go index 971e0d89..7777604e 100644 --- a/engine/resources/kv.go +++ b/engine/resources/kv.go @@ -268,7 +268,7 @@ func (obj *KVRes) lessThanCheck(value string) (bool, error) { return false, nil } -// CheckApply method for Password resource. Does nothing, returns happy! +// CheckApply method for resource. Does nothing, returns happy! func (obj *KVRes) CheckApply(ctx context.Context, apply bool) (bool, error) { wg := &sync.WaitGroup{} defer wg.Wait() // this must be above the defer cancel() call diff --git a/engine/resources/schedule.go b/engine/resources/schedule.go new file mode 100644 index 00000000..2f5b14c5 --- /dev/null +++ b/engine/resources/schedule.go @@ -0,0 +1,349 @@ +// Mgmt +// Copyright (C) James Shubin and the project contributors +// Written by James Shubin and the project contributors +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . +// +// Additional permission under GNU GPL version 3 section 7 +// +// If you modify this program, or any covered work, by linking or combining it +// with embedded mcl code and modules (and that the embedded mcl code and +// modules which link with this program, contain a copy of their source code in +// the authoritative form) containing parts covered by the terms of any other +// license, the licensors of this program grant you additional permission to +// convey the resulting work. Furthermore, the licensors of this program grant +// the original author, James Shubin, additional permission to update this +// additional permission if he deems it necessary to achieve the goals of this +// additional permission. + +package resources + +import ( + "context" + "fmt" + "sync" + + "github.com/purpleidea/mgmt/engine" + "github.com/purpleidea/mgmt/engine/traits" + "github.com/purpleidea/mgmt/etcd/scheduler" // XXX: abstract this if possible + "github.com/purpleidea/mgmt/util/errwrap" +) + +func init() { + engine.RegisterResource("schedule", func() engine.Res { return &ScheduleRes{} }) +} + +// ScheduleRes is a resource which starts up a "distributed scheduler". All +// nodes of the same namespace will be part of the same scheduling pool. The +// scheduling result can be determined by using the "schedule" function. If the +// options specified are different among peers in the same namespace, then it is +// undefined which options if any will get chosen. +type ScheduleRes struct { + traits.Base // add the base methods without re-implementation + + init *engine.Init + + world engine.SchedulerWorld + + // Namespace represents the namespace key to use. If it is not + // specified, the Name value is used instead. + Namespace string `lang:"namespace" yaml:"namespace"` + + // Strategy is the scheduling strategy to use. If this value is nil or, + // undefined, then a default will be chosen automatically. + Strategy *string `lang:"strategy" yaml:"strategy"` + + // Max is the max number of hosts to elect. If this is unspecified, then + // a default of 1 is used. + Max *int `lang:"max" yaml:"max"` + + // Reuse specifies that we reuse the client lease on reconnect. If reuse + // is false, then on host disconnect, that hosts entry will immediately + // expire, and the scheduler will react instantly and remove that host + // entry from the list. If this is true, or if the host closes without a + // clean shutdown, it will take the TTL number of seconds to remove the + // entry. + Reuse *bool `lang:"reuse" yaml:"reuse"` + + // TTL is the time to live for added scheduling "votes". If this value + // is nil or, undefined, then a default value is used. See the `Reuse` + // entry for more information. + TTL *int `lang:"ttl" yaml:"ttl"` + + // once is the startup signal for the scheduler + once chan struct{} +} + +// getNamespace returns the namespace key to be used for this resource. If the +// Namespace field is specified, it will use that, otherwise it uses the Name. +func (obj *ScheduleRes) getNamespace() string { + if obj.Namespace != "" { + return obj.Namespace + } + return obj.Name() +} + +func (obj *ScheduleRes) getOpts() []scheduler.Option { + + schedulerOpts := []scheduler.Option{} + // don't add bad or zero-value options + + defaultStrategy := true + if obj.Strategy != nil && *obj.Strategy != "" { + strategy := *obj.Strategy + if obj.init.Debug { + obj.init.Logf("opts: strategy: %s", strategy) + } + defaultStrategy = false + schedulerOpts = append(schedulerOpts, scheduler.StrategyKind(strategy)) + } + if defaultStrategy { // we always need to add one! + schedulerOpts = append(schedulerOpts, scheduler.StrategyKind(scheduler.DefaultStrategy)) + } + + if obj.Max != nil && *obj.Max > 0 { + max := *obj.Max + // TODO: check for overflow + if obj.init.Debug { + obj.init.Logf("opts: max: %d", max) + } + schedulerOpts = append(schedulerOpts, scheduler.MaxCount(max)) + } + + if obj.Reuse != nil { + reuse := *obj.Reuse + if obj.init.Debug { + obj.init.Logf("opts: reuse: %t", reuse) + } + schedulerOpts = append(schedulerOpts, scheduler.ReuseLease(reuse)) + } + + if obj.TTL != nil && *obj.TTL > 0 { + ttl := *obj.TTL + // TODO: check for overflow + if obj.init.Debug { + obj.init.Logf("opts: ttl: %d", ttl) + } + schedulerOpts = append(schedulerOpts, scheduler.SessionTTL(ttl)) + } + + return schedulerOpts +} + +// Default returns some sensible defaults for this resource. +func (obj *ScheduleRes) Default() engine.Res { + return &ScheduleRes{} +} + +// Validate if the params passed in are valid data. +func (obj *ScheduleRes) Validate() error { + if obj.getNamespace() == "" { + return fmt.Errorf("the Namespace must not be empty") + } + return nil +} + +// Init initializes the resource. +func (obj *ScheduleRes) Init(init *engine.Init) error { + obj.init = init // save for later + + world, ok := obj.init.World.(engine.SchedulerWorld) + if !ok { + return fmt.Errorf("world backend does not support the SchedulerWorld interface") + } + obj.world = world + + obj.once = make(chan struct{}, 1) // buffered! + + return nil +} + +// Cleanup is run by the engine to clean up after the resource is done. +func (obj *ScheduleRes) Cleanup() error { + return nil +} + +// Watch is the primary listener for this resource and it outputs events. +func (obj *ScheduleRes) Watch(ctx context.Context) error { + wg := &sync.WaitGroup{} + defer wg.Wait() + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + obj.init.Running() // when started, notify engine that we're running + + select { + case <-obj.once: + // pass + case <-ctx.Done(): + return ctx.Err() + } + + if obj.init.Debug { + obj.init.Logf("starting scheduler...") + } + + sched, err := obj.world.Scheduler(obj.getNamespace(), obj.getOpts()...) + if err != nil { + return errwrap.Wrapf(err, "can't create scheduler") + } + + watchChan := make(chan *scheduler.ScheduledResult) + + wg.Add(1) + go func() { + defer wg.Done() + defer sched.Shutdown() + select { + case <-ctx.Done(): + return + } + }() + + // process the stream of scheduling output... + wg.Add(1) + go func() { + defer wg.Done() + defer close(watchChan) + for { + hosts, err := sched.Next(ctx) + select { + case watchChan <- &scheduler.ScheduledResult{ + Hosts: hosts, + Err: err, + }: + + case <-ctx.Done(): + return + } + } + }() + + for { + select { + case result, ok := <-watchChan: + if !ok { // channel shutdown + return nil + } + if result == nil { + return fmt.Errorf("unexpected nil result") + } + if err := result.Err; err != nil { + if err == scheduler.ErrEndOfResults { + //return nil // TODO: we should probably fix the reconnect issue and use this here + return fmt.Errorf("scheduler shutdown, reconnect bug?") // XXX: fix etcd reconnects + } + return errwrap.Wrapf(err, "channel watch failed on `%s`", obj.getNamespace()) + } + + if obj.init.Debug { + obj.init.Logf("event!") + } + + case <-ctx.Done(): // closed by the engine to signal shutdown + return nil + } + + obj.init.Event() // notify engine of an event (this can block) + } +} + +// CheckApply method for resource. +func (obj *ScheduleRes) CheckApply(ctx context.Context, apply bool) (bool, error) { + // For maximum correctness, don't start scheduling myself until this + // CheckApply runs at least once. Effectively this unblocks Watch() once + // it has run. If we didn't do this, then illogical graphs could happen + // where we have an edge like Foo["whatever"] -> Schedule["bar"] and if + // Foo failed, we'd still be scheduling, which is not what we want. + + select { + case obj.once <- struct{}{}: + default: // if buffer is full + } + + // FIXME: If we wanted to be really fancy, we could wait until the write + // to the scheduler (etcd) finished before we returned true. + return true, nil +} + +// Cmp compares two resources and returns an error if they are not equivalent. +func (obj *ScheduleRes) Cmp(r engine.Res) error { + // we can only compare ScheduleRes to others of the same resource kind + res, ok := r.(*ScheduleRes) + if !ok { + return fmt.Errorf("not a %s", obj.Kind()) + } + + if obj.getNamespace() != res.getNamespace() { + return fmt.Errorf("the Namespace differs") + } + + if (obj.Strategy == nil) != (res.Strategy == nil) { // xor + return fmt.Errorf("the Strategy differs") + } + if obj.Strategy != nil && res.Strategy != nil { + if *obj.Strategy != *res.Strategy { // compare the values + return fmt.Errorf("the contents of Strategy differs") + } + } + + if (obj.Max == nil) != (res.Max == nil) { // xor + return fmt.Errorf("the Max differs") + } + if obj.Max != nil && res.Max != nil { + if *obj.Max != *res.Max { // compare the values + return fmt.Errorf("the contents of Max differs") + } + } + + if (obj.Reuse == nil) != (res.Reuse == nil) { // xor + return fmt.Errorf("the Reuse differs") + } + if obj.Reuse != nil && res.Reuse != nil { + if *obj.Reuse != *res.Reuse { // compare the values + return fmt.Errorf("the contents of Reuse differs") + } + } + + if (obj.TTL == nil) != (res.TTL == nil) { // xor + return fmt.Errorf("the TTL differs") + } + if obj.TTL != nil && res.TTL != nil { + if *obj.TTL != *res.TTL { // compare the values + return fmt.Errorf("the contents of TTL differs") + } + } + + return nil +} + +// UnmarshalYAML is the custom unmarshal handler for this struct. It is +// primarily useful for setting the defaults. +func (obj *ScheduleRes) UnmarshalYAML(unmarshal func(interface{}) error) error { + type rawRes ScheduleRes // indirection to avoid infinite recursion + + def := obj.Default() // get the default + res, ok := def.(*ScheduleRes) // put in the right format + if !ok { + return fmt.Errorf("could not convert to ScheduleRes") + } + raw := rawRes(*res) // convert; the defaults go here + + if err := unmarshal(&raw); err != nil { + return err + } + + *obj = ScheduleRes(raw) // restore from indirection with type conversion! + return nil +} diff --git a/engine/world.go b/engine/world.go index 527fbf97..4f786880 100644 --- a/engine/world.go +++ b/engine/world.go @@ -34,7 +34,7 @@ import ( "fmt" "github.com/purpleidea/mgmt/etcd/interfaces" - "github.com/purpleidea/mgmt/etcd/scheduler" + "github.com/purpleidea/mgmt/etcd/scheduler" // XXX: abstract this if possible ) // WorldInit is some data passed in when starting the World interface. @@ -236,6 +236,9 @@ type ResDelete struct { type SchedulerWorld interface { // Scheduler runs a distributed scheduler. Scheduler(namespace string, opts ...scheduler.Option) (*scheduler.Result, error) + + // Scheduled gets the scheduled results without participating. + Scheduled(ctx context.Context, namespace string) (chan *scheduler.ScheduledResult, error) } // EtcdWorld is a world interface that should be implemented if the world diff --git a/etcd/scheduler/scheduler.go b/etcd/scheduler/scheduler.go index ab5131f7..2bcc0b50 100644 --- a/etcd/scheduler/scheduler.go +++ b/etcd/scheduler/scheduler.go @@ -39,6 +39,7 @@ import ( "sync" "sync/atomic" + "github.com/purpleidea/mgmt/etcd/interfaces" "github.com/purpleidea/mgmt/util/errwrap" etcd "go.etcd.io/etcd/client/v3" @@ -46,6 +47,9 @@ import ( ) const ( + // DefaultStrategy is the strategy to use if none has been specified. + DefaultStrategy = "rr" + // DefaultSessionTTL is the number of seconds to wait before a dead or // unresponsive host is removed from the scheduled pool. DefaultSessionTTL = 10 // seconds @@ -62,10 +66,10 @@ var ErrEndOfResults = errors.New("scheduler: end of results") var schedulerLeases = make(map[string]etcd.LeaseID) // process lifetime in-memory lease store -// schedulerResult represents output from the scheduler. -type schedulerResult struct { - hosts []string - err error +// ScheduledResult represents output from the scheduler. +type ScheduledResult struct { + Hosts []string + Err error } // Result is what is returned when you request a scheduler. You can call methods @@ -73,7 +77,7 @@ type schedulerResult struct { // these is produced, the scheduler has already kicked off running for you // automatically. type Result struct { - results chan *schedulerResult + results chan *ScheduledResult closeFunc func() // run this when you're done with the scheduler // TODO: replace with an input `context` } @@ -87,7 +91,7 @@ func (obj *Result) Next(ctx context.Context) ([]string, error) { if !ok { return nil, ErrEndOfResults } - return val.hosts, val.err + return val.Hosts, val.Err case <-ctx.Done(): return nil, ctx.Err() @@ -108,7 +112,9 @@ func (obj *Result) Shutdown() { // can be passed in to customize the behaviour. Hostname represents the unique // identifier for the caller. The behaviour is undefined if this is run more // than once with the same path and hostname simultaneously. -func Schedule(client *etcd.Client, path string, hostname string, opts ...Option) (*Result, error) { +func Schedule(client interfaces.Client, path string, hostname string, opts ...Option) (*Result, error) { + c := client.GetClient() + if strings.HasSuffix(path, "/") { return nil, fmt.Errorf("scheduler: path must not end with the slash char") } @@ -160,7 +166,7 @@ func Schedule(client *etcd.Client, path string, hostname string, opts ...Option) } //options.debug = true // use this for local debugging - session, err := concurrency.NewSession(client, sessionOptions...) + session, err := concurrency.NewSession(c, sessionOptions...) if err != nil { return nil, errwrap.Wrapf(err, "scheduler: could not create session") } @@ -175,29 +181,43 @@ func Schedule(client *etcd.Client, path string, hostname string, opts ...Option) // stored scheduler results scheduledPath := fmt.Sprintf("%s/scheduled", path) - scheduledChan := client.Watcher.Watch(ctx, scheduledPath) + //scheduledPath := fmt.Sprintf("%s%s/scheduled", client.GetNamespace(), path) + //scheduledChan := client.Watcher.Watch(ctx, scheduledPath) + scheduledChan, err := client.Watcher(ctx, scheduledPath) + if err != nil { + cancel() + return nil, errwrap.Wrapf(err, "scheduler: could not watch scheduled path") + } // exchange hostname, and attach it to session (leaseID) so it expires // (gets deleted) when we disconnect... exchangePath := fmt.Sprintf("%s/exchange", path) + //exchangePath := fmt.Sprintf("%s%s/exchange", client.GetNamespace(), path) exchangePathHost := fmt.Sprintf("%s/%s", exchangePath, hostname) exchangePathPrefix := fmt.Sprintf("%s/", exchangePath) // open the watch *before* we set our key so that we can see the change! - watchChan := client.Watcher.Watch(ctx, exchangePathPrefix, etcd.WithPrefix()) + //watchChan := client.Watcher.Watch(ctx, exchangePathPrefix, etcd.WithPrefix()) + watchChan, err := client.Watcher(ctx, exchangePathPrefix, etcd.WithPrefix()) + if err != nil { + cancel() + return nil, errwrap.Wrapf(err, "scheduler: could not watch exchange path") + } data := "TODO" // XXX: no data to exchange alongside hostnames yet ifops := []etcd.Cmp{ etcd.Compare(etcd.Value(exchangePathHost), "=", data), etcd.Compare(etcd.LeaseValue(exchangePathHost), "=", leaseID), } - elsop := etcd.OpPut(exchangePathHost, data, etcd.WithLease(leaseID)) + elsop := []etcd.Op{ + etcd.OpPut(exchangePathHost, data, etcd.WithLease(leaseID)), + } // it's important to do this in one transaction, and atomically, because // this way, we only generate one watch event, and only when it's needed // updating leaseID, or key expiry (deletion) both generate watch events // XXX: context!!! - if txn, err := client.KV.Txn(context.TODO()).If(ifops...).Then([]etcd.Op{}...).Else(elsop).Commit(); err != nil { + if txn, err := client.Txn(context.TODO(), ifops, nil, elsop); err != nil { defer cancel() // cancel to avoid leaks if we exit early... return nil, errwrap.Wrapf(err, "could not exchange in `%s`", path) } else if txn.Succeeded { @@ -207,19 +227,19 @@ func Schedule(client *etcd.Client, path string, hostname string, opts ...Option) } // create an election object - electionPath := fmt.Sprintf("%s/election", path) + electionPath := fmt.Sprintf("%s%s/election", client.GetNamespace(), path) election := concurrency.NewElection(session, electionPath) electionChan := election.Observe(ctx) elected := "" // who we "assume" is elected wg := &sync.WaitGroup{} - ch := make(chan *schedulerResult) + ch := make(chan *ScheduledResult) closeChan := make(chan struct{}) send := func(hosts []string, err error) bool { // helper function for sending select { - case ch <- &schedulerResult{ // send - hosts: hosts, - err: err, + case ch <- &ScheduledResult{ // send + Hosts: hosts, + Err: err, }: return true case <-closeChan: // unblock @@ -403,24 +423,22 @@ func Schedule(client *etcd.Client, path string, hostname string, opts ...Option) continue } - err := watchResp.Err() - if watchResp.Canceled || err == context.Canceled { + //err := watchResp.Err() + err := watchResp + if err == context.Canceled { // channel get closed shortly... continue } - if watchResp.Header.Revision == 0 { // by inspection - // received empty message ? - // switched client connection ? - // FIXME: what should we do here ? - continue - } + //if watchResp.Header.Revision == 0 { // by inspection + // // received empty message ? + // // switched client connection ? + // // FIXME: what should we do here ? + // continue + //} if err != nil { send(nil, errwrap.Wrapf(err, "scheduler: exchange watcher failed")) continue } - if len(watchResp.Events) == 0 { // nothing interesting - continue - } options.logf("running exchange values get...") resp, err := client.Get(ctx, exchangePathPrefix, etcd.WithPrefix(), etcd.WithSort(etcd.SortByKey, etcd.SortAscend)) @@ -437,13 +455,12 @@ func Schedule(client *etcd.Client, path string, hostname string, opts ...Option) // specific information which is used for some // purpose, eg: seconds active, and other data? hostnames = make(map[string]string) // reset - for _, x := range resp.Kvs { - k := string(x.Key) + for k, v := range resp { if !strings.HasPrefix(k, exchangePathPrefix) { continue } k = k[len(exchangePathPrefix):] // strip - hostnames[k] = string(x.Value) + hostnames[k] = v } if options.debug { options.logf("available hostnames: %+v", hostnames) @@ -490,20 +507,25 @@ func Schedule(client *etcd.Client, path string, hostname string, opts ...Option) if elected != hostname { options.logf("i am not the leader, running scheduling result get...") resp, err := client.Get(ctx, scheduledPath) - if err != nil || resp == nil || len(resp.Kvs) != 1 { + if err != nil || resp == nil || len(resp) != 1 { if err != nil { send(nil, errwrap.Wrapf(err, "scheduler: could not get scheduling result in `%s`", path)) } else if resp == nil { send(nil, fmt.Errorf("scheduler: could not get scheduling result in `%s`, resp is nil", path)) - } else if len(resp.Kvs) > 1 { - send(nil, fmt.Errorf("scheduler: could not get scheduling result in `%s`, resp kvs: %+v", path, resp.Kvs)) + } else if len(resp) > 1 { + send(nil, fmt.Errorf("scheduler: could not get scheduling result in `%s`, resp kvs: %+v", path, resp)) } - // if len(resp.Kvs) == 0, we shouldn't error + // if len(resp) == 0, we shouldn't error // in that situation it's just too early... continue } - result := string(resp.Kvs[0].Value) + var result string + for _, v := range resp { + result = v + break // get one value + } + //result := string(resp.Kvs[0].Value) hosts := strings.Split(result, hostnameJoinChar) if options.debug { @@ -530,17 +552,20 @@ func Schedule(client *etcd.Client, path string, hostname string, opts ...Option) sort.Strings(hosts) // for consistency options.logf("storing scheduling result...") + data := strings.Join(hosts, hostnameJoinChar) ifops := []etcd.Cmp{ etcd.Compare(etcd.Value(scheduledPath), "=", data), } - elsop := etcd.OpPut(scheduledPath, data) + elsop := []etcd.Op{ + etcd.OpPut(scheduledPath, data), + } // it's important to do this in one transaction, and atomically, because // this way, we only generate one watch event, and only when it's needed // updating leaseID, or key expiry (deletion) both generate watch events // XXX: context!!! - if _, err := client.KV.Txn(context.TODO()).If(ifops...).Then([]etcd.Op{}...).Else(elsop).Commit(); err != nil { + if _, err := client.Txn(context.TODO(), ifops, nil, elsop); err != nil { send(nil, errwrap.Wrapf(err, "scheduler: could not set scheduling result in `%s`", path)) continue } @@ -576,3 +601,99 @@ func Schedule(client *etcd.Client, path string, hostname string, opts ...Option) return result, nil } + +// Scheduled gets the scheduled results without participating. +func Scheduled(ctx context.Context, client interfaces.Client, path string) (chan *ScheduledResult, error) { + if strings.HasSuffix(path, "/") { + return nil, fmt.Errorf("scheduled: path must not end with the slash char") + } + if !strings.HasPrefix(path, "/") { + return nil, fmt.Errorf("scheduled: path must start with the slash char") + } + + // key structure is $path/election = ??? + // key structure is $path/exchange/$hostname = ??? + // key structure is $path/scheduled = ??? + + // stored scheduler results + scheduledPath := fmt.Sprintf("%s/scheduled", path) + ch, err := client.Watcher(ctx, scheduledPath) + if err != nil { + return nil, err + } + + // XXX: What about waitgroups? Does the caller need to know? + result := make(chan *ScheduledResult) + go func() { + defer close(result) + for { + var err error + var hosts []string + + select { + case event, ok := <-ch: + if !ok { + // XXX: should this be an error? + err = nil // we shut down I guess + } else { + err = event // it may be an error + } + + case <-ctx.Done(): + return + } + + // We had an event! + + // Get data to send... + if err == nil { // did we error above? + hosts, err = getScheduled(ctx, client, scheduledPath) + } + + // Send off that data... + select { + case result <- &ScheduledResult{ + Hosts: hosts, + Err: err, + }: + + case <-ctx.Done(): + return + } + + if err != nil { + return + } + } + }() + + return result, nil +} + +// getScheduled gets the list of hosts which are scheduled for a given namespace +// in etcd. +func getScheduled(ctx context.Context, client interfaces.Client, path string) ([]string, error) { + keyMap, err := client.Get(ctx, path) + if err != nil { + return nil, err + } + + if len(keyMap) == 0 { + // nobody scheduled yet + //return nil, interfaces.ErrNotExist + return []string{}, nil + } + + if count := len(keyMap); count != 1 { + return nil, fmt.Errorf("returned %d entries", count) + } + + val, exists := keyMap[path] + if !exists { + return nil, fmt.Errorf("path `%s` is missing", path) + } + + hosts := strings.Split(val, hostnameJoinChar) + + return hosts, nil +} diff --git a/etcd/world.go b/etcd/world.go index f364c48c..1f7f2e99 100644 --- a/etcd/world.go +++ b/etcd/world.go @@ -301,7 +301,13 @@ func (obj *World) Scheduler(namespace string, opts ...scheduler.Option) (*schedu modifiedOpts = append(modifiedOpts, scheduler.Logf(obj.init.Logf)) path := fmt.Sprintf(schedulerPathFmt, namespace) - return scheduler.Schedule(obj.client.GetClient(), path, obj.init.Hostname, modifiedOpts...) + return scheduler.Schedule(obj.client, path, obj.init.Hostname, modifiedOpts...) +} + +// Scheduled gets the scheduled results without participating. +func (obj *World) Scheduled(ctx context.Context, namespace string) (chan *scheduler.ScheduledResult, error) { + path := fmt.Sprintf(schedulerPathFmt, namespace) + return scheduler.Scheduled(ctx, obj.client, path) } // URI returns the current FS URI. diff --git a/examples/lang/exchange0.mcl b/examples/lang/exchange0.mcl deleted file mode 100644 index 3151d88e..00000000 --- a/examples/lang/exchange0.mcl +++ /dev/null @@ -1,20 +0,0 @@ -# run this example with these commands -# watch -n 0.1 'tail *' # run this in /tmp/mgmt/ -# time ./mgmt run --hostname h1 --tmp-prefix --no-pgp empty -# time ./mgmt run --hostname h2 --seeds=http://127.0.0.1:2379 --client-urls=http://127.0.0.1:2381 --server-urls=http://127.0.0.1:2382 --tmp-prefix --no-pgp empty -# time ./mgmt run --hostname h3 --seeds=http://127.0.0.1:2379 --client-urls=http://127.0.0.1:2383 --server-urls=http://127.0.0.1:2384 --tmp-prefix --no-pgp empty -# time ./mgmt run --hostname h4 --seeds=http://127.0.0.1:2379 --client-urls=http://127.0.0.1:2385 --server-urls=http://127.0.0.1:2386 --tmp-prefix --no-pgp empty -# time ./mgmt deploy --no-git --seeds=http://127.0.0.1:2379 lang examples/lang/exchange0.mcl - -import "golang" -import "sys" -import "world" - -$rand = random1(8) -$exchanged = world.exchange("keyns", $rand) - -$host = sys.hostname() -file "/tmp/mgmt/exchange-${host}" { - state => $const.res.file.state.exists, - content => golang.template("Found: {{ . }}\n", $exchanged), -} diff --git a/examples/lang/history1.mcl b/examples/lang/history1.mcl index 6b97e68c..5cc53b64 100644 --- a/examples/lang/history1.mcl +++ b/examples/lang/history1.mcl @@ -3,9 +3,9 @@ import "golang" $dt = datetime.now() -$hystvalues = {"ix0" => $dt, "ix1" => history($dt, 1), "ix2" => history($dt, 2), "ix3" => history($dt, 3),} +$hystvalues = {"now" => $dt, "ix0" => history($dt, 0), "ix1" => history($dt, 1000), "ix2" => history($dt, 2000), "ix3" => history($dt, 3000),} file "/tmp/mgmt/history" { state => $const.res.file.state.exists, - content => golang.template("Index(0) {{.ix0}}: {{ datetime_print .ix0 }}\nIndex(1) {{.ix1}}: {{ datetime_print .ix1 }}\nIndex(2) {{.ix2}}: {{ datetime_print .ix2 }}\nIndex(3) {{.ix3}}: {{ datetime_print .ix3 }}\n", $hystvalues), + content => golang.template("Index($) {{.ix0}}: {{ datetime_print .now }}\nIndex(0) {{.ix0}}: {{ datetime_print .ix0 }}\nIndex(1) {{.ix1}}: {{ datetime_print .ix1 }}\nIndex(2) {{.ix2}}: {{ datetime_print .ix2 }}\nIndex(3) {{.ix3}}: {{ datetime_print .ix3 }}\n", $hystvalues), } diff --git a/examples/lang/hysteresis1.mcl b/examples/lang/hysteresis1.mcl index 238d8144..b1e7d3b6 100644 --- a/examples/lang/hysteresis1.mcl +++ b/examples/lang/hysteresis1.mcl @@ -13,8 +13,8 @@ $threshold = 1.5 # change me if you like # simple hysteresis implementation $h1 = $theload > $threshold -$h2 = history($theload, 1) > $threshold -$h3 = history($theload, 2) > $threshold +$h2 = history($theload, 1000) > $threshold +$h3 = history($theload, 2000) > $threshold $unload = $h1 or $h2 or $h3 virt "mgmt1" { diff --git a/examples/lang/map-iterator0.mcl b/examples/lang/map-iterator0.mcl index a5b7c711..d9c7a1c4 100644 --- a/examples/lang/map-iterator0.mcl +++ b/examples/lang/map-iterator0.mcl @@ -1,14 +1,22 @@ import "golang" import "iter" +#import "fmt" +#import "datetime" +#$str_now = fmt.printf("%d", datetime.now()) + $fn = func($x) { # notable because concrete type is fn(t1) t2, where t1 != t2 - len($x) + #len($x) + #$x + $str_now + #$x + fmt.printf("%d:%d", len($x), datetime.now()) + $x } $in1 = ["a", "bb", "ccc", "dddd", "eeeee",] $out1 = iter.map($in1, $fn) -$t1 = golang.template("out1: {{ . }}", $out1) +#$t1 = golang.template("out1: {{ . }}", $out1) -test [$t1,] {} +print $out1 {} +#print [$t1,] {} diff --git a/examples/lang/schedule0.mcl b/examples/lang/schedule0.mcl index ee86762c..b20dfad6 100644 --- a/examples/lang/schedule0.mcl +++ b/examples/lang/schedule0.mcl @@ -1,24 +1,32 @@ +# test with: +# time ./mgmt run --hostname h1 --tmp-prefix --no-pgp lang examples/lang/schedule0.mcl +# time ./mgmt run --hostname h2 --seeds=http://127.0.0.1:2379 --no-server --no-magic --tmp-prefix --no-pgp lang examples/lang/schedule0.mcl +# time ./mgmt run --hostname h3 --seeds=http://127.0.0.1:2379 --no-server --no-magic --tmp-prefix --no-pgp lang examples/lang/schedule0.mcl +# kill h2 (should see h1 and h3 pick [h1, h3] instead) +# restart h2 (should see [h1, h3] as before) +# kill h3 (should see h1 and h2 pick [h1, h2] instead) +# restart h3 (should see [h1, h2] as before) +# kill h3 +# kill h2 +# kill h1... all done! + import "golang" -import "sys" import "world" -# here are all the possible options: -#$opts = struct{strategy => "rr", max => 3, reuse => false, ttl => 10,} +$ns = "xsched" # some unique string -# although an empty struct is valid too: -#$opts = struct{} +# This node wishes to be included in the scheduled set. These are the options... +schedule "${ns}" { + strategy => "rr", + max => 2, + #reuse => false, + ttl => 10, +} -# we'll just use a smaller subset today: -$opts = struct{strategy => "rr", max => 2, ttl => 10,} +# See the scheduled selection for a particular namespace: +$set = world.schedule($ns) -# schedule in a particular namespace with options: -$set = world.schedule("xsched", $opts) - -# and if you want, you can omit the options entirely: -#$set = world.schedule("xsched") - -$host = sys.hostname() -file "/tmp/mgmt/scheduled-${host}" { +file "/tmp/mgmt/scheduled-${hostname}" { state => $const.res.file.state.exists, content => golang.template("set: {{ . }}\n", $set), } diff --git a/examples/lang/system1.mcl b/examples/lang/system1.mcl new file mode 100644 index 00000000..a9c44d21 --- /dev/null +++ b/examples/lang/system1.mcl @@ -0,0 +1,11 @@ +import "fmt" +import "os" + +$i = os.system(fmt.printf("for x in `seq 3`; do echo \"hello \$x\"; sleep 1s; done; echo done; echo double done")) + +# The msg field is updated several times in quick succession, the resource is +# only guaranteed to be ran for the last value. Thus, it is likely that the +# single "done" values will not be printed. +print "out" { + msg => $i, +} diff --git a/gapi/empty/empty.go b/gapi/empty/empty.go index 47c31c4e..c1bcbd82 100644 --- a/gapi/empty/empty.go +++ b/gapi/empty/empty.go @@ -37,6 +37,7 @@ import ( cliUtil "github.com/purpleidea/mgmt/cli/util" "github.com/purpleidea/mgmt/gapi" "github.com/purpleidea/mgmt/pgraph" + "github.com/purpleidea/mgmt/util/errwrap" ) const ( @@ -65,6 +66,7 @@ type GAPI struct { initialized bool wg *sync.WaitGroup // sync group for tunnel go routines err error + errMutex *sync.Mutex // guards err } // Cli takes an *Info struct, and returns our deploy if activated, and if there @@ -93,6 +95,7 @@ func (obj *GAPI) Init(data *gapi.Data) error { } obj.data = data // store for later obj.wg = &sync.WaitGroup{} + obj.errMutex = &sync.Mutex{} obj.initialized = true return nil } @@ -120,17 +123,17 @@ func (obj *GAPI) Next(ctx context.Context) chan gapi.Next { select { case ch <- next: case <-ctx.Done(): - obj.err = ctx.Err() + obj.errAppend(ctx.Err()) return } - obj.err = err + obj.errAppend(err) return } obj.data.Logf("generating empty graph...") g, err := pgraph.NewGraph("empty") if err != nil { - obj.err = err + obj.errAppend(err) return } @@ -146,7 +149,7 @@ func (obj *GAPI) Next(ctx context.Context) chan gapi.Next { // unblock if we exit while waiting to send! case <-ctx.Done(): - obj.err = ctx.Err() + obj.errAppend(ctx.Err()) return } }() @@ -159,3 +162,10 @@ func (obj *GAPI) Err() error { obj.wg.Wait() return obj.err } + +// errAppend is a simple helper function. +func (obj *GAPI) errAppend(err error) { + obj.errMutex.Lock() + obj.err = errwrap.Append(obj.err, err) + obj.errMutex.Unlock() +} diff --git a/lang/ast/structs.go b/lang/ast/structs.go index 314a41dd..958c92f0 100644 --- a/lang/ast/structs.go +++ b/lang/ast/structs.go @@ -132,7 +132,7 @@ const ( // time. It's mostly recommended that this is set to true, but it is // sometimes made false when debugging scenarios containing the extended // graph shape using CallFunc and other similar function call nodes. - AllowSpeculation = true + AllowSpeculation = true // set to false to test more graph shapes // RequireStrictModulePath can be set to true if you wish to ignore any // of the metadata parent path searching. By default that is allowed, @@ -3505,17 +3505,12 @@ func (obj *StmtIf) TypeCheck() ([]*interfaces.UnificationInvariant, error) { } // Graph returns the reactive function graph which is expressed by this node. It -// includes any vertices produced by this node, and the appropriate edges to any -// vertices that are produced by its children. Nodes which fulfill the Expr -// interface directly produce vertices (and possible children) where as nodes -// that fulfill the Stmt interface do not produces vertices, where as their -// children might. This particular if statement doesn't do anything clever here -// other than adding in both branches of the graph. Since we're functional, this -// shouldn't have any ill effects. -// XXX: is this completely true if we're running technically impure, but safe -// built-in functions on both branches? Can we turn off half of this? +// includes the condition produced by this node, and the appropriate edges of +// that. The then or else side of the graph is added at runtime based on the +// value of the condition. +// TODO: If we know the condition is static, generate only that side statically. func (obj *StmtIf) Graph(env *interfaces.Env) (*pgraph.Graph, error) { - graph, err := pgraph.NewGraph("if") + graph, err := pgraph.NewGraph("stmtif") if err != nil { return nil, err } @@ -3527,16 +3522,39 @@ func (obj *StmtIf) Graph(env *interfaces.Env) (*pgraph.Graph, error) { graph.AddGraph(g) obj.conditionPtr = f - for _, x := range []interfaces.Stmt{obj.ThenBranch, obj.ElseBranch} { - if x == nil { - continue - } - g, err := x.Graph(env) - if err != nil { - return nil, err - } - graph.AddGraph(g) + // We need to call Graph during runtime. We can't do it here. + //var thenGraph *pgraph.Graph + //var elseGraph *pgraph.Graph + //if obj.ThenBranch != nil { + // g, err := obj.ThenBranch.Graph(env) + // if err != nil { + // return nil, err + // } + // thenGraph = g + //} + //if obj.ElseBranch != nil { + // g, err := obj.ElseBranch.Graph(env) + // if err != nil { + // return nil, err + // } + // elseGraph = g + //} + + // Add a vertex for the if statement itself. + edgeName := structs.StmtIfFuncArgNameCondition + stmtIfFunc := &structs.StmtIfFunc{ + EdgeName: edgeName, + // It's unusual to pass in env here, but not unprecedented. It's + // the first Stmt/Expr to require it, but it seems to work and + // Sam has approved this approach! + Env: env, + Then: obj.ThenBranch, + Else: obj.ElseBranch, } + graph.AddVertex(stmtIfFunc) + graph.AddEdge(obj.conditionPtr, stmtIfFunc, &interfaces.FuncEdge{ + Args: []string{edgeName}, + }) return graph, nil } @@ -4044,6 +4062,9 @@ func (obj *StmtFor) Graph(env *interfaces.Env) (*pgraph.Graph, error) { obj.iterBody = []interfaces.Stmt{} mutex.Unlock() }, + + ArgVertices: []interfaces.Func{f}, + //OutputVertex: ???, } graph.AddVertex(forFunc) graph.AddEdge(f, forFunc, &interfaces.FuncEdge{ @@ -4546,6 +4567,9 @@ func (obj *StmtForKV) Graph(env *interfaces.Env) (*pgraph.Graph, error) { obj.iterBody = map[types.Value]interfaces.Stmt{} mutex.Unlock() }, + + ArgVertices: []interfaces.Func{f}, + //OutputVertex: ???, } graph.AddVertex(forKVFunc) graph.AddEdge(f, forKVFunc, &interfaces.FuncEdge{ @@ -9931,7 +9955,7 @@ func (obj *ExprFunc) Graph(env *interfaces.Env) (*pgraph.Graph, interfaces.Func, //return nil, funcs.ErrCantSpeculate return nil, fmt.Errorf("not implemented") } - v := func(innerTxn interfaces.Txn, args []interfaces.Func) (interfaces.Func, error) { + v := func(innerTxn interfaces.Txn, args []interfaces.Func, out interfaces.Func) (interfaces.Func, error) { // Extend the environment with the arguments. extendedEnv := env.Copy() // TODO: Should we copy? for i := range obj.Args { @@ -9965,8 +9989,15 @@ func (obj *ExprFunc) Graph(env *interfaces.Env) (*pgraph.Graph, interfaces.Func, innerTxn.AddGraph(subgraph) + // XXX: do we need to use the `out` arg here? + // XXX: eg: via .SetShape(args, out) + //if shapelyFunc, ok := bodyFunc.(interfaces.ShapelyFunc); ok { + // shapelyFunc.SetShape(args, out) + //} + return bodyFunc, nil } + funcValueFunc = structs.FuncValueToConstFunc(&full.FuncValue{ V: v, F: f, @@ -9975,18 +10006,19 @@ func (obj *ExprFunc) Graph(env *interfaces.Env) (*pgraph.Graph, interfaces.Func, } else if obj.Function != nil { // Build this "callable" version in case it's available and we // can use that directly. We don't need to copy it because we - // expect anything that is Callable to be stateless, and so it + // expect anything that isn't Streamable to be stateless, and it // can use the same function call for every instantiation of it. - var f interfaces.FuncSig - callableFunc, ok := obj.function.(interfaces.CallableFunc) - if ok { - // XXX: this might be dead code, how do we exercise it? - // If the function is callable then the surrounding - // ExprCall will produce a graph containing this func - // instead of calling ExprFunc.Graph(). - f = callableFunc.Call - } - v := func(txn interfaces.Txn, args []interfaces.Func) (interfaces.Func, error) { + // This used to test if it was a CallableFunc. Now they all are, + // so we remove the streamable funcs which are more complicated. + var f interfaces.FuncSig = obj.function.Call + //if _, ok := obj.function.(interfaces.StreamableFunc); !ok { // XXX: is this what we want now? + // // XXX: this might be dead code, how do we exercise it? + // // If the function is callable then the surrounding + // // ExprCall will produce a graph containing this func + // // instead of calling ExprFunc.Graph(). + // f = obj.function.Call + //} + v := func(txn interfaces.Txn, args []interfaces.Func, out interfaces.Func) (interfaces.Func, error) { // Copy obj.function so that the underlying ExprFunc.function gets // refreshed with a new ExprFunc.Function() call. Otherwise, multiple // calls to this function will share the same Func. @@ -10007,12 +10039,19 @@ func (obj *ExprFunc) Graph(env *interfaces.Env) (*pgraph.Graph, interfaces.Func, Args: []string{argName}, }) } + + // XXX: is this the best way to pass this stuff in? + if shapelyFunc, ok := valueTransformingFunc.(interfaces.ShapelyFunc); ok { + shapelyFunc.SetShape(args, out) + } + return valueTransformingFunc, nil } // obj.function is a node which transforms input values into // an output value, but we need to construct a node which takes no // inputs and produces a FuncValue, so we need to wrap it. + // XXX: yes, this (obj.typ) is the type of iter.map and others! funcValueFunc = structs.FuncValueToConstFunc(&full.FuncValue{ V: v, F: f, @@ -10088,7 +10127,7 @@ func (obj *ExprFunc) Value() (types.Value, error) { //return nil, fmt.Errorf("not implemented") return nil, funcs.ErrCantSpeculate } - v := func(innerTxn interfaces.Txn, args []interfaces.Func) (interfaces.Func, error) { + v := func(innerTxn interfaces.Txn, args []interfaces.Func, out interfaces.Func) (interfaces.Func, error) { // There are no ExprParams, so we start with the empty environment. // Extend that environment with the arguments. extendedEnv := interfaces.EmptyEnv() @@ -10131,6 +10170,12 @@ func (obj *ExprFunc) Value() (types.Value, error) { innerTxn.AddGraph(subgraph) + // XXX: do we need to use the `out` arg here? + // XXX: eg: via .SetShape(args, out) + //if shapelyFunc, ok := bodyFunc.(interfaces.ShapelyFunc); ok { + // shapelyFunc.SetShape(args, out) + //} + return bodyFunc, nil } @@ -10961,6 +11006,13 @@ func (obj *ExprCall) Graph(env *interfaces.Env) (*pgraph.Graph, interfaces.Func, argFuncs = append(argFuncs, argFunc) } + callSubgraphOutput := &structs.OutputFunc{ // the new graph shape thing! + Textarea: obj.Textarea, + Name: "callSubgraphOutput", + Type: obj.typ, + EdgeName: structs.OutputFuncArgName, + } + // Speculate early, in an attempt to get a simpler graph shape. //exprFunc, ok := obj.expr.(*ExprFunc) // XXX: Does this need to be .Pure for it to be allowed? @@ -10976,13 +11028,14 @@ func (obj *ExprCall) Graph(env *interfaces.Env) (*pgraph.Graph, interfaces.Func, obj.data.Logf(format, v...) }, }).Init(), + Post: func() error { return nil }, Lock: func() {}, Unlock: func() {}, RefCount: (&ref.Count{}).Init(), }).Init() txn.AddGraph(graph) // add all of the graphs so far... - outputFunc, err := exprFuncValue.CallWithFuncs(txn, argFuncs) + outputFunc, err := exprFuncValue.CallWithFuncs(txn, argFuncs, callSubgraphOutput) // XXX: callSubgraphOutput as the last arg? if err != nil { return nil, nil, errwrap.Wrapf(err, "could not construct the static graph for a function call") } @@ -10992,7 +11045,11 @@ func (obj *ExprCall) Graph(env *interfaces.Env) (*pgraph.Graph, interfaces.Func, return nil, nil, err } - return txn.Graph(), outputFunc, nil + // XXX: do I need to build a callSubgraphOutput and return it + // here for the special cases that need it like iter.map ? + if outputFunc.Info().Spec { // otherwise, don't speculate + return txn.Graph(), outputFunc, nil + } } else if err != nil && ok && canSpeculate && err != funcs.ErrCantSpeculate { // This is a permanent error, not a temporary speculation error. //return nil, nil, err // XXX: Consider adding this... @@ -11009,14 +11066,13 @@ func (obj *ExprCall) Graph(env *interfaces.Env) (*pgraph.Graph, interfaces.Func, edgeName := structs.CallFuncArgNameFunction edgeNameDummy := structs.OutputFuncDummyArgName - callSubgraphOutput := &structs.OutputFunc{ // the new graph shape thing! - Textarea: obj.Textarea, - Name: "callSubgraphOutput", - Type: obj.typ, - EdgeName: structs.OutputFuncArgName, - } graph.AddVertex(callSubgraphOutput) + // XXX: is this the best way to pass this stuff in? + if shapelyFunc, ok := funcValueFunc.(interfaces.ShapelyFunc); ok { + shapelyFunc.SetShape(argFuncs, callSubgraphOutput) + } + callFunc := &structs.CallFunc{ Textarea: obj.Textarea, @@ -12774,6 +12830,8 @@ func (obj *ExprIf) Graph(env *interfaces.Env) (*pgraph.Graph, interfaces.Func, e EdgeName: edgeName, + // XXX: Do we need to pass in `env` and the Then/Else Expr's, + // the way we do with StmtIf, instead of doing it like this? ThenGraph: thenGraph, ElseGraph: elseGraph, diff --git a/lang/core/collect.go b/lang/core/collect.go index ac679cb7..5165da3b 100644 --- a/lang/core/collect.go +++ b/lang/core/collect.go @@ -100,10 +100,8 @@ type CollectFunc struct { init *interfaces.Init - last types.Value // last value received to use for diff - args []types.Value - kind string - result types.Value // last calculated output + input chan string // stream of inputs + kind *string // the active kind watchChan chan error } @@ -292,13 +290,13 @@ func (obj *CollectFunc) Info() *interfaces.Info { // Init runs some startup code for this function. func (obj *CollectFunc) Init(init *interfaces.Init) error { obj.init = init + obj.input = make(chan string) obj.watchChan = make(chan error) // XXX: sender should close this, but did I implement that part yet??? return nil } // Stream returns the changing values that this func has over time. func (obj *CollectFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes ctx, cancel := context.WithCancel(ctx) defer cancel() // important so that we cleanup the watch when exiting for { @@ -306,50 +304,35 @@ func (obj *CollectFunc) Stream(ctx context.Context) error { // TODO: should this first chan be run as a priority channel to // avoid some sort of glitch? is that even possible? can our // hostname check with reality (below) fix that? - case input, ok := <-obj.init.Input: + case kind, ok := <-obj.input: if !ok { - obj.init.Input = nil // don't infinite loop back - continue // no more inputs, but don't return! + obj.input = nil // don't infinite loop back + return fmt.Errorf("unexpected close") } - //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { - // return errwrap.Wrapf(err, "wrong function input") - //} - if obj.last != nil && input.Cmp(obj.last) == nil { - continue // value didn't change, skip it - } - obj.last = input // store for next - - args, err := interfaces.StructToCallableArgs(input) // []types.Value, error) - if err != nil { - return err - } - obj.args = args - - kind := args[0].Str() - if kind == "" { - return fmt.Errorf("can't use an empty kind") - } - if obj.init.Debug { - obj.init.Logf("kind: %s", kind) + if obj.kind != nil && *obj.kind == kind { + continue // nothing changed } // TODO: support changing the key over time? - if obj.kind == "" { - obj.kind = kind // store it + if obj.kind == nil { + obj.kind = &kind // store var err error // Don't send a value right away, wait for the // first Watch startup event to get one! - obj.watchChan, err = obj.init.World.ResWatch(ctx, obj.kind) // watch for var changes + obj.watchChan, err = obj.init.World.ResWatch(ctx, kind) // watch for var changes if err != nil { return err } - - } else if obj.kind != kind { - return fmt.Errorf("can't change kind, previously: `%s`", obj.kind) + continue // we get values on the watch chan, not here! } - continue // we get values on the watch chan, not here! + if *obj.kind == kind { + continue // skip duplicates + } + + // *obj.kind != kind + return fmt.Errorf("can't change kind, previously: `%s`", *obj.kind) case err, ok := <-obj.watchChan: if !ok { // closed @@ -360,27 +343,13 @@ func (obj *CollectFunc) Stream(ctx context.Context) error { return nil } if err != nil { - return errwrap.Wrapf(err, "channel watch failed on `%s`", obj.kind) + return errwrap.Wrapf(err, "channel watch failed on `%s`", *obj.kind) } - result, err := obj.Call(ctx, obj.args) // get the value... - if err != nil { + if err := obj.init.Event(ctx); err != nil { // send event return err } - // if the result is still the same, skip sending an update... - if obj.result != nil && result.Cmp(obj.result) == nil { - continue // result didn't change - } - obj.result = result // store new result - - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- obj.result: // send - // pass case <-ctx.Done(): return nil } @@ -402,6 +371,21 @@ func (obj *CollectFunc) Call(ctx context.Context, args []types.Value) (types.Val return nil, fmt.Errorf("invalid resource kind: %s", kind) } + // Check before we send to a chan where we'd need Stream to be running. + if obj.init == nil { + return nil, funcs.ErrCantSpeculate + } + + if obj.init.Debug { + obj.init.Logf("kind: %s", kind) + } + + select { + case obj.input <- kind: + case <-ctx.Done(): + return nil, ctx.Err() + } + filters := []*engine.ResFilter{} arg := args[1] @@ -453,10 +437,6 @@ func (obj *CollectFunc) Call(ctx context.Context, args []types.Value) (types.Val } } - if obj.init == nil { - return nil, funcs.ErrCantSpeculate - } - list := types.NewList(obj.Info().Sig.Out) // collectFuncOutType if len(filters) == 0 { diff --git a/lang/core/core_test.go b/lang/core/core_test.go deleted file mode 100644 index 78abd12a..00000000 --- a/lang/core/core_test.go +++ /dev/null @@ -1,1012 +0,0 @@ -// Mgmt -// Copyright (C) James Shubin and the project contributors -// Written by James Shubin and the project contributors -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// Additional permission under GNU GPL version 3 section 7 -// -// If you modify this program, or any covered work, by linking or combining it -// with embedded mcl code and modules (and that the embedded mcl code and -// modules which link with this program, contain a copy of their source code in -// the authoritative form) containing parts covered by the terms of any other -// license, the licensors of this program grant you additional permission to -// convey the resulting work. Furthermore, the licensors of this program grant -// the original author, James Shubin, additional permission to update this -// additional permission if he deems it necessary to achieve the goals of this -// additional permission. - -//go:build !root - -package core - -import ( - "context" - "fmt" - "os" - "reflect" - "sync" - "testing" - - "github.com/purpleidea/mgmt/lang/funcs" - "github.com/purpleidea/mgmt/lang/interfaces" - "github.com/purpleidea/mgmt/lang/types" - "github.com/purpleidea/mgmt/util" - "github.com/purpleidea/mgmt/util/errwrap" - - "github.com/davecgh/go-spew/spew" - "github.com/kylelemons/godebug/pretty" -) - -// PureFuncExec is only used for tests. -func PureFuncExec(handle interfaces.Func, args []types.Value) (types.Value, error) { - hostname := "" // XXX: add to interface - debug := false // XXX: add to interface - logf := func(format string, v ...interface{}) {} // XXX: add to interface - ctx, cancel := context.WithCancel(context.TODO()) - defer cancel() - - info := handle.Info() - if !info.Pure { - return nil, fmt.Errorf("func is not pure") - } - // if function is expensive to run, we won't run it provisionally - if !info.Fast { - return nil, fmt.Errorf("func is not fast") - } - - sig := handle.Info().Sig - if sig.Kind != types.KindFunc { - return nil, fmt.Errorf("must be kind func") - } - if sig.HasUni() { - return nil, fmt.Errorf("func contains unification vars") - } - - if buildableFunc, ok := handle.(interfaces.BuildableFunc); ok { - if _, err := buildableFunc.Build(sig); err != nil { - return nil, fmt.Errorf("can't build function: %v", err) - } - } - - if err := handle.Validate(); err != nil { - return nil, errwrap.Wrapf(err, "could not validate func") - } - - ord := handle.Info().Sig.Ord - if i, j := len(ord), len(args); i != j { - return nil, fmt.Errorf("expected %d args, got %d", i, j) - } - - wg := &sync.WaitGroup{} - defer wg.Wait() - - errch := make(chan error) - input := make(chan types.Value) // we close this when we're done - output := make(chan types.Value) // we create it, func closes it - - init := &interfaces.Init{ - Hostname: hostname, - Input: input, - Output: output, - World: nil, // should not be used for pure functions - Debug: debug, - Logf: func(format string, v ...interface{}) { - logf("func: "+format, v...) - }, - } - - if err := handle.Init(init); err != nil { - return nil, errwrap.Wrapf(err, "could not init func") - } - - close1 := make(chan struct{}) - close2 := make(chan struct{}) - wg.Add(1) - go func() { - defer wg.Done() - defer close(errch) // last one turns out the lights - select { - case <-close1: - } - select { - case <-close2: - } - }() - - wg.Add(1) - go func() { - defer wg.Done() - defer close(close1) - if debug { - logf("Running func") - } - err := handle.Stream(ctx) // sends to output chan - if debug { - logf("Exiting func") - } - if err == nil { - return - } - // we closed with an error... - select { - case errch <- errwrap.Wrapf(err, "problem streaming func"): - } - }() - - wg.Add(1) - go func() { - defer wg.Done() - defer close(close2) - defer close(input) // we only send one value - if len(args) == 0 { - return - } - si := &types.Type{ - // input to functions are structs - Kind: types.KindStruct, - Map: handle.Info().Sig.Map, - Ord: handle.Info().Sig.Ord, - } - st := types.NewStruct(si) - - for i, arg := range args { - name := handle.Info().Sig.Ord[i] - if err := st.Set(name, arg); err != nil { // populate struct - select { - case errch <- errwrap.Wrapf(err, "struct set failure"): - } - return - } - } - - select { - case input <- st: // send to function (must not block) - case <-close1: // unblock the input send in case stream closed - select { - case errch <- fmt.Errorf("stream closed early"): - } - } - }() - - once := false - var result types.Value - var reterr error -Loop: - for { - select { - case value, ok := <-output: // read from channel - if !ok { - output = nil - continue Loop // only exit via errch closing! - } - if once { - reterr = fmt.Errorf("got more than one value") - continue // only exit via errch closing! - } - once = true - result = value // save value - - case err, ok := <-errch: // handle possible errors - if !ok { - break Loop - } - if err == nil { - // programming error - err = fmt.Errorf("error was missing") - } - e := errwrap.Wrapf(err, "problem streaming func") - reterr = errwrap.Append(reterr, e) - } - } - - cancel() - - if result == nil && reterr == nil { - // programming error - // XXX: i think this can happen when we exit without error, but - // before we send one output message... not sure how this happens - // XXX: iow, we never send on output, and errch closes... - // XXX: this could happen if we send zero input args, and Stream exits without error - return nil, fmt.Errorf("function exited with nil result and nil error") - } - return result, reterr -} - -func TestPureFuncExec0(t *testing.T) { - type test struct { // an individual test - name string - funcname string - args []types.Value - fail bool - expect types.Value - } - testCases := []test{} - - //{ - // testCases = append(testCases, test{ - // name: "", - // funcname: "", - // args: []types.Value{ - // }, - // fail: false, - // expect: nil, - // }) - //} - { - testCases = append(testCases, test{ - name: "strings.to_lower 0", - funcname: "strings.to_lower", - args: []types.Value{ - &types.StrValue{ - V: "HELLO", - }, - }, - fail: false, - expect: &types.StrValue{ - V: "hello", - }, - }) - } - { - testCases = append(testCases, test{ - name: "datetime.now fail", - funcname: "datetime.now", - args: nil, - fail: true, - expect: nil, - }) - } - // TODO: run unification in PureFuncExec if it makes sense to do so... - //{ - // testCases = append(testCases, test{ - // name: "len 0", - // funcname: "len", - // args: []types.Value{ - // &types.StrValue{ - // V: "Hello, world!", - // }, - // }, - // fail: false, - // expect: &types.IntValue{ - // V: 13, - // }, - // }) - //} - - names := []string{} - for index, tc := range testCases { // run all the tests - if tc.name == "" { - t.Errorf("test #%d: not named", index) - continue - } - if util.StrInList(tc.name, names) { - t.Errorf("test #%d: duplicate sub test name of: %s", index, tc.name) - continue - } - names = append(names, tc.name) - - //if index != 3 { // hack to run a subset (useful for debugging) - //if (index != 20 && index != 21) { - //if tc.name != "nil" { - // continue - //} - - t.Run(fmt.Sprintf("test #%d (%s)", index, tc.name), func(t *testing.T) { - name, funcname, args, fail, expect := tc.name, tc.funcname, tc.args, tc.fail, tc.expect - - t.Logf("\n\ntest #%d (%s) ----------------\n\n", index, name) - - f, err := funcs.Lookup(funcname) - if err != nil { - t.Errorf("test #%d: func lookup failed with: %+v", index, err) - return - } - - result, err := PureFuncExec(f, args) - - if !fail && err != nil { - t.Errorf("test #%d: func failed with: %+v", index, err) - return - } - if fail && err == nil { - t.Errorf("test #%d: func passed, expected fail", index) - return - } - if !fail && result == nil { - t.Errorf("test #%d: func output was nil", index) - return - } - - if reflect.DeepEqual(result, expect) { - return - } - - // double check because DeepEqual is different since the func exists - diff := pretty.Compare(result, expect) - if diff == "" { // bonus - return - } - t.Errorf("test #%d: result did not match expected", index) - // TODO: consider making our own recursive print function - t.Logf("test #%d: actual: \n\n%s\n", index, spew.Sdump(result)) - t.Logf("test #%d: expected: \n\n%s", index, spew.Sdump(expect)) - - // more details, for tricky cases: - diffable := &pretty.Config{ - Diffable: true, - IncludeUnexported: true, - //PrintStringers: false, - //PrintTextMarshalers: false, - //SkipZeroFields: false, - } - t.Logf("test #%d: actual: \n\n%s\n", index, diffable.Sprint(result)) - t.Logf("test #%d: expected: \n\n%s", index, diffable.Sprint(expect)) - t.Logf("test #%d: diff:\n%s", index, diff) - }) - } -} - -// Step is used for the timeline in tests. -type Step interface { - Action() error - Expect() error -} - -type manualStep struct { - action func() error - expect func() error - - exit chan struct{} // exit signal, set by test harness - argch chan []types.Value // send new inputs, set by test harness - valueptrch chan int // incoming values, set by test harness - results []types.Value // all values, set by test harness -} - -func (obj *manualStep) Action() error { - return obj.action() -} -func (obj *manualStep) Expect() error { - return obj.expect() -} - -// NewManualStep creates a new manual step with an action and an expect test. -func NewManualStep(action, expect func() error) Step { - return &manualStep{ - action: action, - expect: expect, - } -} - -// NewSendInputs sends a list of inputs to the running function to populate it. -// If you send the wrong input signature, then you'll cause a failure. Testing -// this kind of failure is not a goal of these tests, since the unification code -// is meant to guarantee we always send the correct type signature. -func NewSendInputs(inputs []types.Value) Step { - return &sendInputsStep{ - inputs: inputs, - } -} - -type sendInputsStep struct { - inputs []types.Value - - exit chan struct{} // exit signal, set by test harness - argch chan []types.Value // send new inputs, set by test harness - //valueptrch chan int // incoming values, set by test harness - //results []types.Value // all values, set by test harness -} - -func (obj *sendInputsStep) Action() error { - select { - case obj.argch <- obj.inputs: - return nil - case <-obj.exit: - return fmt.Errorf("exit called") - } -} - -func (obj *sendInputsStep) Expect() error { return nil } - -// NewWaitForNSeconds waits this many seconds for new values from the stream. It -// can timeout if it gets bored of waiting. -func NewWaitForNSeconds(number int, timeout int) Step { - return &waitAmountStep{ - timer: number, // timer seconds - timeout: timeout, - } -} - -// NewWaitForNValues waits for this many values from the stream. It can timeout -// if it gets bored of waiting. If you request more values than can be produced, -// then it will block indefinitely if there's no timeout. -func NewWaitForNValues(number int, timeout int) Step { - return &waitAmountStep{ - count: number, // count values - timeout: timeout, - } -} - -// waitAmountStep waits for either a count of N values, or a timer of N seconds, -// or both. It also accepts a timeout which will cause it to error. -// TODO: have the timeout timer be overall instead of per step! -type waitAmountStep struct { - count int // nth count (set to a negative value to disable) - timer int // seconds (set to a negative value to disable) - timeout int // seconds to fail after - - exit chan struct{} // exit signal, set by test harness - //argch chan []types.Value // send new inputs, set by test harness - valueptrch chan int // incoming values, set by test harness - results []types.Value // all values, set by test harness -} - -func (obj *waitAmountStep) Action() error { - count := 0 - ticked := false // did we get the timer event? - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - ticker := util.TimeAfterOrBlockCtx(ctx, obj.timer) - if obj.timer < 0 { // disable timer - ticked = true - } - - for { - if count >= obj.count { // got everything we wanted - if ticked { - break - } - } - select { - case <-obj.exit: - return fmt.Errorf("exit called") - - case <-util.TimeAfterOrBlock(obj.timeout): - // TODO: make this overall instead of re-running it each time - return fmt.Errorf("waited too long for a value") - - case n, ok := <-obj.valueptrch: // read output - if !ok { - return fmt.Errorf("unexpected close") - } - count++ - _ = n // this is the index of the value we're at - - case <-ticker: // received the timer event - ticker = nil - ticked = true - if obj.count > -1 { - break - } - } - } - return nil -} -func (obj *waitAmountStep) Expect() error { return nil } - -// NewRangeExpect passes in an expect function which will receive the entire -// range of values ever received. This stream (list) of values can be matched on -// however you like. -func NewRangeExpect(fn func([]types.Value) error) Step { - return &rangeExpectStep{ - fn: fn, - } -} - -type rangeExpectStep struct { - fn func([]types.Value) error - - // TODO: we could pass exit to the expect fn if we wanted in the future - exit chan struct{} // exit signal, set by test harness - //argch chan []types.Value // send new inputs, set by test harness - //valueptrch chan int // incoming values, set by test harness - results []types.Value // all values, set by test harness -} - -func (obj *rangeExpectStep) Action() error { return nil } - -func (obj *rangeExpectStep) Expect() error { - results := []types.Value{} - for _, v := range obj.results { // copy - value := v.Copy() - results = append(results, value) - } - return obj.fn(results) // run with a copy -} - -// vog is a helper function to produce mcl values from golang equivalents that -// is only safe in tests because it panics on error. -func vog(i interface{}) types.Value { - v, err := types.ValueOfGolang(i) - if err != nil { - panic(fmt.Sprintf("unexpected error in vog: %+v", err)) - } - return v -} - -// rcopy is a helper to copy a list of types.Value structs. -func rcopy(input []types.Value) []types.Value { - result := []types.Value{} - for i := range input { - x := input[i].Copy() - result = append(result, x) - } - return result -} - -// TestLiveFuncExec0 runs a live execution timeline on a function stream. It is -// very useful for testing function streams. -// FIXME: if the function returns a different type than what is specified by its -// signature, we might block instead of returning a useful error. -func TestLiveFuncExec0(t *testing.T) { - t.Skip("Skipping this test because it's currently broken and we might change the API.") - - type args struct { - argv []types.Value - next func() // specifies we're ready for the next set of inputs - } - - type test struct { // an individual test - name string - hostname string // in case we want to simulate a hostname - funcname string - - // TODO: this could be a generator that keeps pushing out steps until it's done! - timeline []Step - expect func() error // function to check for expected state - startup func() error // function to run as startup - cleanup func() error // function to run as cleanup - } - - timeout := -1 // default timeout (block) if not specified elsewhere - testCases := []test{} - { - count := 5 - timeline := []Step{ - NewWaitForNValues(count, timeout), // get 5 values - // pass in a custom validation function - NewRangeExpect(func(args []types.Value) error { - //fmt.Printf("range: %+v\n", args) // debugging - if len(args) < count { - return fmt.Errorf("no args found") - } - // check for increasing ints (ideal delta == 1) - x := args[0].Int() - for i := 1; i < count; i++ { - if args[i].Int()-x < 1 { - return fmt.Errorf("range jumps: %+v", args) - } - if args[i].Int()-x != 1 { - // if this fails, travis is just slow - return fmt.Errorf("timing error: %+v", args) - } - x = args[i].Int() - } - return nil - }), - //NewWaitForNSeconds(5, timeout), // not needed - } - - testCases = append(testCases, test{ - name: "simple func", - hostname: "", // not needed for this func - funcname: "datetime.now", - timeline: timeline, - expect: func() error { return nil }, - startup: func() error { return nil }, - cleanup: func() error { return nil }, - }) - } - { - timeline := []Step{ - NewSendInputs([]types.Value{ - vog("helloXworld"), - vog("X"), // split by this - }), - - NewWaitForNValues(1, timeout), // more than 1 blocks here - - // pass in a custom validation function - NewRangeExpect(func(args []types.Value) error { - //fmt.Printf("range: %+v\n", args) // debugging - if c := len(args); c != 1 { - return fmt.Errorf("wrong args count, got: %d", c) - } - if args[0].Type().Kind != types.KindList { - return fmt.Errorf("expected list, got: %+v", args[0]) - } - if err := vog([]string{"hello", "world"}).Cmp(args[0]); err != nil { - return errwrap.Wrapf(err, "got different expected value: %+v", args[0]) - } - return nil - }), - } - - testCases = append(testCases, test{ - name: "simple pure func", - hostname: "", // not needed for this func - funcname: "strings.split", - timeline: timeline, - expect: func() error { return nil }, - startup: func() error { return nil }, - cleanup: func() error { return nil }, - }) - } - { - p := "/tmp/somefiletoread" - content := "hello world!\n" - timeline := []Step{ - NewSendInputs([]types.Value{ - vog(p), - }), - - NewWaitForNValues(1, timeout), // more than 1 blocks here - NewWaitForNSeconds(5, 10), // wait longer just to be sure - - // pass in a custom validation function - NewRangeExpect(func(args []types.Value) error { - //fmt.Printf("range: %+v\n", args) // debugging - if c := len(args); c != 1 { - return fmt.Errorf("wrong args count, got: %d", c) - } - if args[0].Type().Kind != types.KindStr { - return fmt.Errorf("expected str, got: %+v", args[0]) - } - if err := vog(content).Cmp(args[0]); err != nil { - return errwrap.Wrapf(err, "got different expected value: %+v", args[0]) - } - return nil - }), - } - - testCases = append(testCases, test{ - name: "readfile", - hostname: "", // not needed for this func - funcname: "os.readfile", - timeline: timeline, - expect: func() error { return nil }, - startup: func() error { return os.WriteFile(p, []byte(content), 0666) }, - cleanup: func() error { return os.Remove(p) }, - }) - } - names := []string{} - for index, tc := range testCases { // run all the tests - if tc.name == "" { - t.Errorf("test #%d: not named", index) - continue - } - if util.StrInList(tc.name, names) { - t.Errorf("test #%d: duplicate sub test name of: %s", index, tc.name) - continue - } - names = append(names, tc.name) - t.Run(fmt.Sprintf("test #%d (%s)", index, tc.name), func(t *testing.T) { - hostname, funcname, timeline, expect, startup, cleanup := tc.hostname, tc.funcname, tc.timeline, tc.expect, tc.startup, tc.cleanup - - t.Logf("\n\ntest #%d: func: %+v\n", index, funcname) - defer t.Logf("test #%d: done!", index) - - handle, err := funcs.Lookup(funcname) // get function... - if err != nil { - t.Errorf("test #%d: func lookup failed with: %+v", index, err) - return - } - sig := handle.Info().Sig - if sig.Kind != types.KindFunc { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: must be kind func", index) - return - } - if sig.HasUni() { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: func contains unification vars", index) - return - } - - if buildableFunc, ok := handle.(interfaces.BuildableFunc); ok { - if _, err := buildableFunc.Build(sig); err != nil { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: can't build function: %v", index, err) - return - } - } - - // run validate! - if err := handle.Validate(); err != nil { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: could not validate Func: %+v", index, err) - return - } - - input := make(chan types.Value) // we close this when we're done - output := make(chan types.Value) // we create it, func closes it - - debug := testing.Verbose() // set via the -test.v flag to `go test` - logf := func(format string, v ...interface{}) { - t.Logf(fmt.Sprintf("test #%d: func: ", index)+format, v...) - } - init := &interfaces.Init{ - Hostname: hostname, - Input: input, - Output: output, - World: nil, // TODO: add me somehow! - Debug: debug, - Logf: logf, - } - - t.Logf("test #%d: running startup()", index) - if err := startup(); err != nil { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: could not startup: %+v", index, err) - return - } - - // run init - t.Logf("test #%d: running Init", index) - if err := handle.Init(init); err != nil { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: could not init func: %+v", index, err) - return - } - defer func() { - t.Logf("test #%d: running cleanup()", index) - if err := cleanup(); err != nil { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: could not cleanup: %+v", index, err) - } - }() - - wg := &sync.WaitGroup{} - defer wg.Wait() // if we return early - - argch := make(chan []types.Value) - errch := make(chan error) - close1 := make(chan struct{}) - close2 := make(chan struct{}) - kill1 := make(chan struct{}) - kill2 := make(chan struct{}) - //kill3 := make(chan struct{}) // future use - exit := make(chan struct{}) - - mutex := &sync.RWMutex{} - results := []types.Value{} // all values received so far - valueptrch := make(chan int) // which Nth value are we at? - killTimeline := make(chan struct{}) // ask timeline to exit - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - // wait for close signals - wg.Add(1) - go func() { - defer wg.Done() - defer close(errch) // last one turns out the lights - select { - case <-close1: - } - select { - case <-close2: - } - }() - - // wait for kill signals - wg.Add(1) - go func() { - defer wg.Done() - select { - case <-exit: - case <-kill1: - case <-kill2: - //case <-kill3: // future use - } - close(killTimeline) // main kill signal for tl - }() - - // run the stream - wg.Add(1) - go func() { - defer wg.Done() - defer close(close1) - if debug { - logf("Running func") - } - err := handle.Stream(ctx) // sends to output chan - t.Logf("test #%d: stream exited with: %+v", index, err) - if debug { - logf("Exiting func") - } - if err == nil { - return - } - // we closed with an error... - select { - case errch <- errwrap.Wrapf(err, "problem streaming func"): - } - }() - - // read from incoming args and send to input channel - wg.Add(1) - go func() { - defer wg.Done() - defer close(close2) - defer close(input) // close input when done - if argch == nil { // no args - return - } - si := &types.Type{ - // input to functions are structs - Kind: types.KindStruct, - Map: handle.Info().Sig.Map, - Ord: handle.Info().Sig.Ord, - } - t.Logf("test #%d: func has sig: %s", index, si) - - // TODO: should this be a select with an exit signal? - for args := range argch { // chan - st := types.NewStruct(si) - count := 0 - for i, arg := range args { - //name := util.NumToAlpha(i) // assume (incorrectly) for now... - name := handle.Info().Sig.Ord[i] // better - if err := st.Set(name, arg); err != nil { // populate struct - select { - case errch <- errwrap.Wrapf(err, "struct set failure"): - } - close(kill1) // unblock tl and cause fail - return - } - count++ - } - if count != len(si.Map) { // expect this number - select { - case errch <- fmt.Errorf("struct field count is wrong"): - } - close(kill1) // unblock tl and cause fail - return - } - - t.Logf("test #%d: send to func: %s", index, args) - select { - case input <- st: // send to function (must not block) - case <-close1: // unblock the input send in case stream closed - select { - case errch <- fmt.Errorf("stream closed early"): - } - } - } - }() - - // run timeline - wg.Add(1) - go func() { - t.Logf("test #%d: executing timeline", index) - defer wg.Done() - Timeline: - for ix, step := range timeline { - select { - case <-killTimeline: - break Timeline - default: - // pass - } - - mutex.RLock() - // magic setting of important values... - if s, ok := step.(*manualStep); ok { - s.exit = killTimeline // kill signal - s.argch = argch // send inputs here - s.valueptrch = valueptrch // receive value ptr - s.results = rcopy(results) // all results as array - } - if s, ok := step.(*sendInputsStep); ok { - s.exit = killTimeline - s.argch = argch - //s.valueptrch = valueptrch - //s.results = rcopy(results) - } - if s, ok := step.(*waitAmountStep); ok { - s.exit = killTimeline - //s.argch = argch - s.valueptrch = valueptrch - s.results = rcopy(results) - } - if s, ok := step.(*rangeExpectStep); ok { - s.exit = killTimeline - //s.argch = argch - //s.valueptrch = valueptrch - s.results = rcopy(results) - } - mutex.RUnlock() - - t.Logf("test #%d: step(%d)...", index, ix) - if err := step.Action(); err != nil { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: step(%d) action failed: %s", index, ix, err.Error()) - break - } - if err := step.Expect(); err != nil { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: step(%d) expect failed: %s", index, ix, err.Error()) - break - } - } - t.Logf("test #%d: timeline finished", index) - close(argch) - - t.Logf("test #%d: running cancel", index) - cancel() - }() - - // read everything - counter := 0 - Loop: - for { - select { - case value, ok := <-output: // read from channel - if !ok { - output = nil - continue Loop // only exit via errch closing! - } - t.Logf("test #%d: got from func: %s", index, value) - // check return type - if err := handle.Info().Sig.Out.Cmp(value.Type()); err != nil { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: unexpected return type from func: %+v", index, err) - close(kill2) - continue Loop // only exit via errch closing! - } - - mutex.Lock() - results = append(results, value) // save value - mutex.Unlock() - counter++ - - case err, ok := <-errch: // handle possible errors - if !ok { - break Loop - } - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: error: %+v", index, err) - continue Loop // only exit via errch closing! - } - - // send events to our timeline - select { - case valueptrch <- counter: // TODO: send value? - - // TODO: add this sort of thing, but don't block everyone who doesn't read - //case <-time.After(time.Duration(globalStepReadTimeout) * time.Second): - // t.Errorf("test #%d: FAIL", index) - // t.Errorf("test #%d: timeline receiver was too slow for value", index) - // t.Errorf("test #%d: got(%d): %+v", index, counter, results[counter]) - // close(kill3) // shut everything down - // continue Loop // only exit via errch closing! - } - } - - t.Logf("test #%d: waiting for shutdown", index) - close(exit) - wg.Wait() - - if err := expect(); err != nil { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: expect failed: %s", index, err.Error()) - return - } - - // all done! - }) - } -} diff --git a/lang/core/datetime/now.go b/lang/core/datetime/now.go index c488de50..d26b18fa 100644 --- a/lang/core/datetime/now.go +++ b/lang/core/datetime/now.go @@ -31,7 +31,6 @@ package coredatetime import ( "context" - "fmt" "time" "github.com/purpleidea/mgmt/lang/funcs" @@ -82,22 +81,8 @@ func (obj *Now) Init(init *interfaces.Init) error { return nil } -// Stream returns the changing values that this fact has over time. +// Stream starts a mainloop and runs Event when it's time to Call() again. func (obj *Now) Stream(ctx context.Context) error { - defer close(obj.init.Output) // always signal when we're done - - // We always wait for our initial event to start. - select { - case _, ok := <-obj.init.Input: - if ok { - return fmt.Errorf("unexpected input") - } - obj.init.Input = nil - - case <-ctx.Done(): - return nil - } - // XXX: this might be an interesting fact to write because: // 1) will the sleeps from the ticker be in sync with the second ticker? // 2) if we care about a less precise interval (eg: minute changes) can @@ -124,17 +109,9 @@ func (obj *Now) Stream(ctx context.Context) error { return nil } - result, err := obj.Call(ctx, nil) - if err != nil { + if err := obj.init.Event(ctx); err != nil { return err } - - select { - case obj.init.Output <- result: - - case <-ctx.Done(): - return nil - } } } diff --git a/lang/core/datetime/str_now.go b/lang/core/datetime/str_now.go new file mode 100644 index 00000000..ef569c29 --- /dev/null +++ b/lang/core/datetime/str_now.go @@ -0,0 +1,130 @@ +// Mgmt +// Copyright (C) James Shubin and the project contributors +// Written by James Shubin and the project contributors +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . +// +// Additional permission under GNU GPL version 3 section 7 +// +// If you modify this program, or any covered work, by linking or combining it +// with embedded mcl code and modules (and that the embedded mcl code and +// modules which link with this program, contain a copy of their source code in +// the authoritative form) containing parts covered by the terms of any other +// license, the licensors of this program grant you additional permission to +// convey the resulting work. Furthermore, the licensors of this program grant +// the original author, James Shubin, additional permission to update this +// additional permission if he deems it necessary to achieve the goals of this +// additional permission. + +package coredatetime + +import ( + "context" + "strconv" + "time" + + "github.com/purpleidea/mgmt/lang/funcs" + "github.com/purpleidea/mgmt/lang/interfaces" + "github.com/purpleidea/mgmt/lang/types" +) + +const ( + // StrNowFuncName is the name this fact is registered as. It's still a + // Func Name because this is the name space the fact is actually using. + StrNowFuncName = "str_now" +) + +func init() { + funcs.ModuleRegister(ModuleName, StrNowFuncName, func() interfaces.Func { return &StrNow{} }) // must register the fact and name +} + +// StrNow is a fact which returns the current date and time. +type StrNow struct { + init *interfaces.Init +} + +// String returns a simple name for this fact. This is needed so this struct can +// satisfy the pgraph.Vertex interface. +func (obj *StrNow) String() string { + return NowFuncName +} + +// Validate makes sure we've built our struct properly. +func (obj *StrNow) Validate() error { + return nil +} + +// Info returns some static info about itself. +func (obj *StrNow) Info() *interfaces.Info { + return &interfaces.Info{ + Pure: false, // non-constant facts can't be pure! + Memo: false, + Fast: false, + Spec: false, + Sig: types.NewType("func() str"), + } +} + +// Init runs some startup code for this fact. +func (obj *StrNow) Init(init *interfaces.Init) error { + obj.init = init + return nil +} + +// Stream returns the changing values that this fact has over time. +func (obj *StrNow) Stream(ctx context.Context) error { + // XXX: this might be an interesting fact to write because: + // 1) will the sleeps from the ticker be in sync with the second ticker? + // 2) if we care about a less precise interval (eg: minute changes) can + // we set this up so it doesn't tick as often? -- Yes (make this a function or create a limit function to wrap this) + // 3) is it best to have a delta timer that wakes up before it's needed + // and calculates how much longer to sleep for? + ticker := time.NewTicker(time.Duration(1) * time.Second) + //ticker := time.NewTicker(time.Duration(1) * time.Millisecond) + //ticker := time.NewTicker(time.Duration(1) * time.Microsecond) + //ticker := time.NewTicker(time.Duration(1) * time.Nanosecond) + defer ticker.Stop() + + // streams must generate an initial event on startup + // even though ticker will send one, we want to be faster to first event + startChan := make(chan struct{}) // start signal + close(startChan) // kick it off! + + for { + select { + case <-startChan: + startChan = nil // disable + + case <-ticker.C: // received the timer event + // pass + + case <-ctx.Done(): + return nil + } + + if err := obj.init.Event(ctx); err != nil { + return err + } + } +} + +// Call this fact and return the value if it is possible to do so at this time. +func (obj *StrNow) Call(ctx context.Context, args []types.Value) (types.Value, error) { + return &types.StrValue{ // seconds since 1970 as a string... + V: strconv.FormatInt(time.Now().Unix(), 10), // .UTC() not necessary + //V: strconv.FormatInt(time.Now().UnixMilli(), 10), // .UTC() not necessary + //V: strconv.FormatInt(time.Now().UnixMicro(), 10), // .UTC() not necessary + //V: strconv.FormatInt(time.Now().UnixNano(), 10), // .UTC() not necessary + }, nil +} diff --git a/lang/core/deploy/abspath.go b/lang/core/deploy/abspath.go index 6f0f308d..699332f0 100644 --- a/lang/core/deploy/abspath.go +++ b/lang/core/deploy/abspath.go @@ -114,62 +114,6 @@ func (obj *AbsPathFunc) Init(init *interfaces.Init) error { return nil } -// Stream returns the changing values that this func has over time. -func (obj *AbsPathFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes - for { - select { - case input, ok := <-obj.init.Input: - if !ok { - obj.init.Input = nil // don't infinite loop back - continue // no more inputs, but don't return! - } - //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { - // return errwrap.Wrapf(err, "wrong function input") - //} - - if obj.last != nil && input.Cmp(obj.last) == nil { - continue // value didn't change, skip it - } - obj.last = input // store for next - - args, err := interfaces.StructToCallableArgs(input) // []types.Value, error) - if err != nil { - return err - } - obj.args = args - - path := args[0].Str() - // TODO: add validation for absolute path? - if obj.path != nil && *obj.path == path { - continue // nothing changed - } - obj.path = &path - - result, err := obj.Call(ctx, obj.args) - if err != nil { - return err - } - - // if the result is still the same, skip sending an update... - if obj.result != nil && result.Cmp(obj.result) == nil { - continue // result didn't change - } - obj.result = result // store new result - - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- obj.result: // send - // pass - case <-ctx.Done(): - return nil - } - } -} - // Copy is implemented so that the obj.built value is not lost if we copy this // function. func (obj *AbsPathFunc) Copy() interfaces.Func { @@ -187,6 +131,8 @@ func (obj *AbsPathFunc) Call(ctx context.Context, args []types.Value) (types.Val } path := args[0].Str() + // TODO: add validation for absolute path? + if obj.data == nil { return nil, funcs.ErrCantSpeculate } diff --git a/lang/core/deploy/readfile.go b/lang/core/deploy/readfile.go index 515ae689..d802d3c7 100644 --- a/lang/core/deploy/readfile.go +++ b/lang/core/deploy/readfile.go @@ -61,11 +61,6 @@ var _ interfaces.DataFunc = &ReadFileFunc{} type ReadFileFunc struct { init *interfaces.Init data *interfaces.FuncData - last types.Value // last value received to use for diff - - args []types.Value - filename *string // the active filename - result types.Value // last calculated output } // String returns a simple name for this function. This is needed so this struct @@ -115,63 +110,6 @@ func (obj *ReadFileFunc) Init(init *interfaces.Init) error { return nil } -// Stream returns the changing values that this func has over time. -func (obj *ReadFileFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes - for { - select { - case input, ok := <-obj.init.Input: - if !ok { - obj.init.Input = nil // don't infinite loop back - continue // no more inputs, but don't return! - } - //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { - // return errwrap.Wrapf(err, "wrong function input") - //} - - if obj.last != nil && input.Cmp(obj.last) == nil { - continue // value didn't change, skip it - } - obj.last = input // store for next - - args, err := interfaces.StructToCallableArgs(input) // []types.Value, error) - if err != nil { - return err - } - obj.args = args - - filename := args[0].Str() - // TODO: add validation for absolute path? - // TODO: add check for empty string - if obj.filename != nil && *obj.filename == filename { - continue // nothing changed - } - obj.filename = &filename - - result, err := obj.Call(ctx, obj.args) - if err != nil { - return err - } - - // if the result is still the same, skip sending an update... - if obj.result != nil && result.Cmp(obj.result) == nil { - continue // result didn't change - } - obj.result = result // store new result - - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- obj.result: // send - // pass - case <-ctx.Done(): - return nil - } - } -} - // Copy is implemented so that the obj.built value is not lost if we copy this // function. func (obj *ReadFileFunc) Copy() interfaces.Func { diff --git a/lang/core/deploy/readfileabs.go b/lang/core/deploy/readfileabs.go index e8facab1..e38ef346 100644 --- a/lang/core/deploy/readfileabs.go +++ b/lang/core/deploy/readfileabs.go @@ -61,11 +61,6 @@ var _ interfaces.DataFunc = &ReadFileAbsFunc{} type ReadFileAbsFunc struct { init *interfaces.Init data *interfaces.FuncData - last types.Value // last value received to use for diff - - args []types.Value - filename *string // the active filename - result types.Value // last calculated output } // String returns a simple name for this function. This is needed so this struct @@ -115,63 +110,6 @@ func (obj *ReadFileAbsFunc) Init(init *interfaces.Init) error { return nil } -// Stream returns the changing values that this func has over time. -func (obj *ReadFileAbsFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes - for { - select { - case input, ok := <-obj.init.Input: - if !ok { - obj.init.Input = nil // don't infinite loop back - continue // no more inputs, but don't return! - } - //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { - // return errwrap.Wrapf(err, "wrong function input") - //} - - if obj.last != nil && input.Cmp(obj.last) == nil { - continue // value didn't change, skip it - } - obj.last = input // store for next - - args, err := interfaces.StructToCallableArgs(input) // []types.Value, error) - if err != nil { - return err - } - obj.args = args - - filename := args[0].Str() - // TODO: add validation for absolute path? - // TODO: add check for empty string - if obj.filename != nil && *obj.filename == filename { - continue // nothing changed - } - obj.filename = &filename - - result, err := obj.Call(ctx, obj.args) - if err != nil { - return err - } - - // if the result is still the same, skip sending an update... - if obj.result != nil && result.Cmp(obj.result) == nil { - continue // result didn't change - } - obj.result = result // store new result - - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- obj.result: // send - // pass - case <-ctx.Done(): - return nil - } - } -} - // Copy is implemented so that the obj.built value is not lost if we copy this // function. func (obj *ReadFileAbsFunc) Copy() interfaces.Func { diff --git a/lang/core/example/flipflop.go b/lang/core/example/flipflop.go index 81fb8589..402b1f4b 100644 --- a/lang/core/example/flipflop.go +++ b/lang/core/example/flipflop.go @@ -31,7 +31,6 @@ package coreexample import ( "context" - "fmt" "sync" "time" @@ -88,22 +87,8 @@ func (obj *FlipFlop) Init(init *interfaces.Init) error { return nil } -// Stream returns the changing values that this fact has over time. +// Stream starts a mainloop and runs Event when it's time to Call() again. func (obj *FlipFlop) Stream(ctx context.Context) error { - defer close(obj.init.Output) // always signal when we're done - - // We always wait for our initial event to start. - select { - case _, ok := <-obj.init.Input: - if ok { - return fmt.Errorf("unexpected input") - } - obj.init.Input = nil - - case <-ctx.Done(): - return nil - } - // TODO: don't hard code 5 sec interval ticker := time.NewTicker(time.Duration(5) * time.Second) defer ticker.Stop() @@ -125,20 +110,12 @@ func (obj *FlipFlop) Stream(ctx context.Context) error { return nil } - result, err := obj.Call(ctx, nil) - if err != nil { - return err - } - obj.mutex.Lock() obj.value = !obj.value // flip it obj.mutex.Unlock() - select { - case obj.init.Output <- result: - - case <-ctx.Done(): - return nil + if err := obj.init.Event(ctx); err != nil { + return err } } } diff --git a/lang/core/example/vumeter.go b/lang/core/example/vumeter.go index 436d5eff..92c98c1c 100644 --- a/lang/core/example/vumeter.go +++ b/lang/core/example/vumeter.go @@ -37,7 +37,6 @@ import ( "path/filepath" "strconv" "strings" - "sync" "syscall" "time" @@ -147,66 +146,36 @@ func (obj *VUMeterFunc) Init(init *interfaces.Init) error { // Stream returns the changing values that this func has over time. func (obj *VUMeterFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes - ticker := newTicker() + ticker := time.NewTicker(time.Duration(1) * time.Second) defer ticker.Stop() - // FIXME: this goChan seems to work better than the ticker :) - // this is because we have a ~1sec delay in capturing the value in exec - goChan := make(chan struct{}) - once := &sync.Once{} - onceFunc := func() { close(goChan) } // only run once! + + // streams must generate an initial event on startup + // even though ticker will send one, we want to be faster to first event + startChan := make(chan struct{}) // start signal + close(startChan) // kick it off! + + // XXX: We have a back pressure problem! Call takes ~1sec to run. But + // since we moved to a buffered event channel, we now generate new + // events every second, rather than every second _after_ the previous + // event was consumed... This means, we're bombaring the engine with + // more events than it can ever process. Add a backpressure mechanism so + // that we're always draining the event channel. We could do that here + // per-function for rare cases like this, and/or we could try and fix it + // in the engine. for { select { - case input, ok := <-obj.init.Input: - if !ok { - obj.init.Input = nil // don't infinite loop back - continue // no more inputs, but don't return! - } - //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { - // return errwrap.Wrapf(err, "wrong function input") - //} + case <-startChan: + startChan = nil // disable - if obj.last != nil && input.Cmp(obj.last) == nil { - continue // value didn't change, skip it - } - obj.last = input // store for next - - args, err := interfaces.StructToCallableArgs(input) // []types.Value, error) - if err != nil { - return err - } - obj.args = args - - once.Do(onceFunc) - continue // we must wrap around and go in through goChan - - //case <-ticker.C: // received the timer event - case <-goChan: // triggers constantly - - if obj.last == nil { - continue // still waiting for input values - } - - result, err := obj.Call(ctx, obj.args) - if err != nil { - return err - } - - // if the result is still the same, skip sending an update... - if obj.result != nil && result.Cmp(obj.result) == nil { - continue // result didn't change - } - obj.result = result // store new result + case <-ticker.C: // received the timer event + // pass case <-ctx.Done(): return nil } - select { - case obj.init.Output <- obj.result: // send - // pass - case <-ctx.Done(): - return nil + if err := obj.init.Event(ctx); err != nil { + return err } } } diff --git a/lang/core/fmt/printf.go b/lang/core/fmt/printf.go index 11fe7261..1d3b18c7 100644 --- a/lang/core/fmt/printf.go +++ b/lang/core/fmt/printf.go @@ -315,53 +315,6 @@ func (obj *PrintfFunc) Init(init *interfaces.Init) error { return nil } -// Stream returns the changing values that this func has over time. -func (obj *PrintfFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes - for { - select { - case input, ok := <-obj.init.Input: - if !ok { - return nil // can't output any more - } - //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { - // return errwrap.Wrapf(err, "wrong function input") - //} - - if obj.last != nil && input.Cmp(obj.last) == nil { - continue // value didn't change, skip it - } - obj.last = input // store for next - - args, err := interfaces.StructToCallableArgs(input) // []types.Value, error) - if err != nil { - return err - } - - result, err := obj.Call(ctx, args) - if err != nil { - return err - } - - // if the result is still the same, skip sending an update... - if obj.result != nil && result.Cmp(obj.result) == nil { - continue // result didn't change - } - obj.result = result // store new result - - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- obj.result: // send - // pass - case <-ctx.Done(): - return nil - } - } -} - // Copy is implemented so that the obj.Type value is not lost if we copy this // function. func (obj *PrintfFunc) Copy() interfaces.Func { diff --git a/lang/core/golang/template.go b/lang/core/golang/template.go index c583c814..df50aa96 100644 --- a/lang/core/golang/template.go +++ b/lang/core/golang/template.go @@ -84,9 +84,6 @@ type TemplateFunc struct { built bool // was this function built yet? init *interfaces.Init - last types.Value // last value received to use for diff - - result types.Value // last calculated output } // String returns a simple name for this function. This is needed so this struct @@ -348,53 +345,6 @@ Loop: return buf.String(), nil } -// Stream returns the changing values that this func has over time. -func (obj *TemplateFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes - for { - select { - case input, ok := <-obj.init.Input: - if !ok { - return nil // can't output any more - } - //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { - // return errwrap.Wrapf(err, "wrong function input") - //} - - if obj.last != nil && input.Cmp(obj.last) == nil { - continue // value didn't change, skip it - } - obj.last = input // store for next - - args, err := interfaces.StructToCallableArgs(input) // []types.Value, error) - if err != nil { - return err - } - - result, err := obj.Call(ctx, args) - if err != nil { - return err - } - - // if the result is still the same, skip sending an update... - if obj.result != nil && result.Cmp(obj.result) == nil { - continue // result didn't change - } - obj.result = result // store new result - - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- obj.result: // send - // pass - case <-ctx.Done(): - return nil - } - } -} - // Copy is implemented so that the obj.built value is not lost if we copy this // function. func (obj *TemplateFunc) Copy() interfaces.Func { diff --git a/lang/core/history.go b/lang/core/history.go index ff04db75..7ca2b231 100644 --- a/lang/core/history.go +++ b/lang/core/history.go @@ -32,6 +32,8 @@ package core // TODO: should this be in its own individual package? import ( "context" "fmt" + "sync" + "time" "github.com/purpleidea/mgmt/lang/funcs" "github.com/purpleidea/mgmt/lang/interfaces" @@ -46,6 +48,9 @@ const ( // arg names... historyArgNameValue = "value" historyArgNameIndex = "index" + + // factor helps us sample much faster for precision reasons. + factor = 10 ) func init() { @@ -54,25 +59,39 @@ func init() { var _ interfaces.BuildableFunc = &HistoryFunc{} // ensure it meets this expectation -// HistoryFunc is special function which returns the Nth oldest value seen. It -// must store up incoming values until it gets enough to return the desired one. -// A restart of the program, will expunge the stored state. This obviously takes -// more memory, the further back you wish to index. A change in the index var is -// generally not useful, but it is permitted. Moving it to a smaller value will -// cause older index values to be expunged. If this is undesirable, a max count -// could be added. This was not implemented with efficiency in mind. Since some -// functions might not send out un-changed values, it might also make sense to -// implement a *time* based hysteresis, since this only looks at the last N -// changed values. A time based hysteresis would tick every precision-width, and -// store whatever the latest value at that time is. +// HistoryFunc is special function which returns the value N milliseconds ago. +// It must store up incoming values until it gets enough to return the desired +// one. If it doesn't yet have a value, it will initially return the oldest +// value it can. A restart of the program, will expunge the stored state. This +// obviously takes more memory, the further back you wish to index. A change in +// the index var is generally not useful, but it is permitted. Moving it to a +// smaller value will cause older index values to be expunged. If this is +// undesirable, a max count could be added. This was not implemented with +// efficiency in mind. This implements a *time* based hysteresis, since +// previously this only looked at the last N changed values. Since some +// functions might not send out un-changed values, it might make more sense this +// way. This time based hysteresis should tick every precision-width, and store +// whatever the latest value at that time is. This is implemented wrong, because +// we can't guarantee the sampling interval is constant, and it's also wasteful. +// We should implement a better version that keeps track of the time, so that we +// can pick the closest one and also not need to store duplicates. +// XXX: This function needs another look. We likely we to snapshot everytime we +// get a new value in obj.Call instead of having a ticker. type HistoryFunc struct { Type *types.Type // type of input value (same as output type) init *interfaces.Init - history []types.Value // goes from newest (index->0) to oldest (len()-1) + input chan int64 + delay *int64 - result types.Value // last calculated output + value types.Value // last value + buffer []*valueWithTimestamp + interval int + retention int + + ticker *time.Ticker + mutex *sync.Mutex // don't need an rwmutex since only one reader } // String returns a simple name for this function. This is needed so this struct @@ -168,68 +187,165 @@ func (obj *HistoryFunc) Info() *interfaces.Info { // Init runs some startup code for this function. func (obj *HistoryFunc) Init(init *interfaces.Init) error { obj.init = init + obj.input = make(chan int64) + obj.mutex = &sync.Mutex{} return nil } // Stream returns the changing values that this func has over time. func (obj *HistoryFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes + obj.ticker = time.NewTicker(1) // build it however (non-zero to avoid panic!) + defer obj.ticker.Stop() // double stop is safe + obj.ticker.Stop() // begin with a stopped ticker + select { + case <-obj.ticker.C: // drain if needed + default: + } + for { select { - case input, ok := <-obj.init.Input: + case delay, ok := <-obj.input: if !ok { - return nil // can't output any more + obj.input = nil // don't infinite loop back + return fmt.Errorf("unexpected close") } - //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { - // return errwrap.Wrapf(err, "wrong function input") + + // obj.delay is only used here for duplicate detection, + // and while similar to obj.interval, we don't reuse it + // because we don't want a race condition reading delay + if obj.delay != nil && *obj.delay == delay { + continue // nothing changed + } + obj.delay = &delay + + obj.reinit(int(delay)) // starts ticker! + + case <-obj.ticker.C: // received the timer event + obj.store() + // XXX: We deadlock here if the select{} in obj.Call + // runs at the same time and the event obj.ag is + // unbuffered. Should the engine buffer? + + // XXX: If we send events, we basically infinite loop :/ + // XXX: Didn't look into the feedback mechanism yet. + //if err := obj.init.Event(ctx); err != nil { + // return err //} - //if obj.last != nil && input.Cmp(obj.last) == nil { - // continue // value didn't change, skip it - //} - //obj.last = input // store for next - - index := int(input.Struct()[historyArgNameIndex].Int()) - value := input.Struct()[historyArgNameValue] - var result types.Value - - if index < 0 { - return fmt.Errorf("can't use a negative index of %d", index) - } - - // 1) truncate history so length equals index - if len(obj.history) > index { - // remove all but first N elements, where N == index - obj.history = obj.history[:index] - } - - // 2) (un)shift (add our new value to the beginning) - obj.history = append([]types.Value{value}, obj.history...) - - // 3) are we ready to output a sufficiently old value? - if index >= len(obj.history) { - continue // not enough history is stored yet... - } - - // 4) read one off the back - result = obj.history[len(obj.history)-1] - - // TODO: do we want to do this? - // if the result is still the same, skip sending an update... - if obj.result != nil && result.Cmp(obj.result) == nil { - continue // result didn't change - } - obj.result = result // store new result - - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- obj.result: // send - // pass case <-ctx.Done(): return nil } } } + +func (obj *HistoryFunc) reinit(delay int) { + obj.mutex.Lock() + defer obj.mutex.Unlock() + + if obj.buffer == nil { + } + + obj.interval = delay + obj.retention = delay + 10000 // XXX: arbitrary + obj.buffer = []*valueWithTimestamp{} + + duration := delay / factor // XXX: sample more often than delay? + + // Start sampler... + if duration == 0 { // can't be zero or ticker will panic + duration = 100 // XXX: 1ms is probably too fast + } + obj.ticker.Reset(time.Duration(duration) * time.Millisecond) +} + +func (obj *HistoryFunc) store() { + obj.mutex.Lock() + defer obj.mutex.Unlock() + + val := obj.value.Copy() // copy + + now := time.Now() + v := &valueWithTimestamp{ + Timestamp: now, + Value: val, + } + obj.buffer = append(obj.buffer, v) // newer values go at the end + + retention := time.Duration(obj.retention) * time.Millisecond + + // clean up old entries + cutoff := now.Add(-retention) + i := 0 + for ; i < len(obj.buffer); i++ { + if obj.buffer[i].Timestamp.After(cutoff) { + break + } + } + obj.buffer = obj.buffer[i:] +} + +func (obj *HistoryFunc) peekAgo(ms int) types.Value { + obj.mutex.Lock() + defer obj.mutex.Unlock() + + if obj.buffer == nil { // haven't started yet + return nil + } + if len(obj.buffer) == 0 { // no data exists yet + return nil + } + + target := time.Now().Add(-time.Duration(ms) * time.Millisecond) + + for i := len(obj.buffer) - 1; i >= 0; i-- { + if !obj.buffer[i].Timestamp.After(target) { + return obj.buffer[i].Value + } + } + + // If no value found, return the oldest one. + return obj.buffer[0].Value +} + +// Call this function with the input args and return the value if it is possible +// to do so at this time. +func (obj *HistoryFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) { + if len(args) < 2 { + return nil, fmt.Errorf("not enough args") + } + value := args[0] + interval := args[1].Int() // ms (used to be index) + + if interval < 0 { + return nil, fmt.Errorf("can't use a negative interval of %d", interval) + } + + // Check before we send to a chan where we'd need Stream to be running. + if obj.init == nil { + return nil, funcs.ErrCantSpeculate + } + + obj.mutex.Lock() + obj.value = value // store a copy + obj.mutex.Unlock() + + // XXX: we deadlock here if obj.init.Event also runs at the same time! + // XXX: ...only if it's unbuffered of course. Should the engine buffer? + select { + case obj.input <- interval: // inform the delay interval + case <-ctx.Done(): + return nil, ctx.Err() + } + + val := obj.peekAgo(int(interval)) // contains mutex + if val == nil { // don't have a value yet, return self... + return obj.value, nil + } + return val, nil +} + +// valueWithTimestamp stores a value alongside the time it was recorded. +type valueWithTimestamp struct { + Timestamp time.Time + Value types.Value +} diff --git a/lang/core/iter/filter.go b/lang/core/iter/filter.go index 6e81662a..a92589a7 100644 --- a/lang/core/iter/filter.go +++ b/lang/core/iter/filter.go @@ -77,10 +77,8 @@ type FilterFunc struct { listType *types.Type - // outputChan is an initially-nil channel from which we receive output - // lists from the subgraph. This channel is reset when the subgraph is - // recreated. - outputChan chan types.Value + argFuncs []interfaces.Func + outputFunc interfaces.Func } // String returns a simple name for this function. This is needed so this struct @@ -99,6 +97,11 @@ func (obj *FilterFunc) ArgGen(index int) (string, error) { } // helper +// +// NOTE: The expression signature is shown here, but the actual "signature" of +// this in the function graph returns the "dummy" value because we do the same +// this that we do with ExprCall for example. That means that this function is +// one of very few where the actual expr signature is different from the func! func (obj *FilterFunc) sig() *types.Type { // func(inputs []?1, function func(?1) bool) []?1 typ := "?1" @@ -121,6 +124,60 @@ func (obj *FilterFunc) sig() *types.Type { // runs. func (obj *FilterFunc) Build(typ *types.Type) (*types.Type, error) { // typ is the KindFunc signature we're trying to build... + if typ.Kind != types.KindFunc { + return nil, fmt.Errorf("input type must be of kind func") + } + + if len(typ.Ord) != 2 { + return nil, fmt.Errorf("the map needs exactly two args") + } + if typ.Map == nil { + return nil, fmt.Errorf("the map is nil") + } + + tInputs, exists := typ.Map[typ.Ord[0]] + if !exists || tInputs == nil { + return nil, fmt.Errorf("first argument was missing") + } + tFunction, exists := typ.Map[typ.Ord[1]] + if !exists || tFunction == nil { + return nil, fmt.Errorf("second argument was missing") + } + + if tInputs.Kind != types.KindList { + return nil, fmt.Errorf("first argument must be of kind list") + } + if tFunction.Kind != types.KindFunc { + return nil, fmt.Errorf("second argument must be of kind func") + } + + if typ.Out == nil { + return nil, fmt.Errorf("return type must be specified") + } + if typ.Out.Kind != types.KindList { + return nil, fmt.Errorf("return argument must be a list") + } + + if len(tFunction.Ord) != 1 { + return nil, fmt.Errorf("the functions map needs exactly one arg") + } + if tFunction.Map == nil { + return nil, fmt.Errorf("the functions map is nil") + } + tArg, exists := tFunction.Map[tFunction.Ord[0]] + if !exists || tArg == nil { + return nil, fmt.Errorf("the functions first argument was missing") + } + if err := tArg.Cmp(tInputs.Val); err != nil { + return nil, errwrap.Wrapf(err, "the functions arg type must match the input list contents type") + } + + if tFunction.Out == nil { + return nil, fmt.Errorf("return type of function must be specified") + } + if err := tFunction.Out.Cmp(types.TypeBool); err != nil { + return nil, errwrap.Wrapf(err, "return type of function must be a bool") + } // TODO: Do we need to be extra careful and check that this matches? // unificationUtil.UnifyCmp(typ, obj.sig()) != nil {} @@ -130,11 +187,22 @@ func (obj *FilterFunc) Build(typ *types.Type) (*types.Type, error) { return obj.sig(), nil } +// SetShape tells the function about some special graph engine pointers. +func (obj *FilterFunc) SetShape(argFuncs []interfaces.Func, outputFunc interfaces.Func) { + obj.argFuncs = argFuncs + obj.outputFunc = outputFunc +} + // Validate tells us if the input struct takes a valid form. func (obj *FilterFunc) Validate() error { if obj.Type == nil { return fmt.Errorf("type is not yet known") } + + if obj.argFuncs == nil || obj.outputFunc == nil { + return fmt.Errorf("function did not receive shape information") + } + return nil } @@ -145,7 +213,7 @@ func (obj *FilterFunc) Info() *interfaces.Info { Pure: false, // XXX: what if the input function isn't pure? Memo: false, Fast: false, - Spec: false, + Spec: false, // must be false with the current graph shape code Sig: obj.sig(), // helper Err: obj.Validate(), } @@ -162,125 +230,6 @@ func (obj *FilterFunc) Init(init *interfaces.Init) error { return nil } -// Stream returns the changing values that this func has over time. -func (obj *FilterFunc) Stream(ctx context.Context) error { - // Every time the FuncValue or the length of the list changes, recreate - // the subgraph, by calling the FuncValue N times on N nodes, each of - // which extracts one of the N values in the list. - - defer close(obj.init.Output) // the sender closes - - // A Func to send input lists to the subgraph. The Txn.Erase() call - // ensures that this Func is not removed when the subgraph is recreated, - // so that the function graph can propagate the last list we received to - // the subgraph. - inputChan := make(chan types.Value) - subgraphInput := &structs.ChannelBasedSourceFunc{ - Name: "subgraphInput", - Source: obj, - Chan: inputChan, - Type: obj.listType, - } - obj.init.Txn.AddVertex(subgraphInput) - if err := obj.init.Txn.Commit(); err != nil { - return errwrap.Wrapf(err, "commit error in Stream") - } - obj.init.Txn.Erase() // prevent the next Reverse() from removing subgraphInput - defer func() { - close(inputChan) - obj.init.Txn.Reverse() - obj.init.Txn.DeleteVertex(subgraphInput) - obj.init.Txn.Commit() - }() - - obj.outputChan = nil - - canReceiveMoreFuncValuesOrInputLists := true - canReceiveMoreOutputLists := true - for { - - if !canReceiveMoreFuncValuesOrInputLists && !canReceiveMoreOutputLists { - //break - return nil - } - - select { - case input, ok := <-obj.init.Input: - if !ok { - obj.init.Input = nil // block looping back here - canReceiveMoreFuncValuesOrInputLists = false - continue - } - - if obj.last != nil && input.Cmp(obj.last) == nil { - continue // value didn't change, skip it - } - obj.last = input // store for next - - value, exists := input.Struct()[filterArgNameFunction] - if !exists { - return fmt.Errorf("programming error, can't find edge") - } - - newFuncValue, ok := value.(*full.FuncValue) - if !ok { - return fmt.Errorf("programming error, can't convert to *FuncValue") - } - - newInputList, exists := input.Struct()[filterArgNameInputs] - if !exists { - return fmt.Errorf("programming error, can't find edge") - } - - // If we have a new function or the length of the input - // list has changed, then we need to replace the - // subgraph with a new one that uses the new function - // the correct number of times. - - // It's important to have this compare step to avoid - // redundant graph replacements which slow things down, - // but also cause the engine to lock, which can preempt - // the process scheduler, which can cause duplicate or - // unnecessary re-sending of values here, which causes - // the whole process to repeat ad-nauseum. - n := len(newInputList.List()) - if newFuncValue != obj.lastFuncValue || n != obj.lastInputListLength { - obj.lastFuncValue = newFuncValue - obj.lastInputListLength = n - // replaceSubGraph uses the above two values - if err := obj.replaceSubGraph(subgraphInput); err != nil { - return errwrap.Wrapf(err, "could not replace subgraph") - } - canReceiveMoreOutputLists = true - } - - // send the new input list to the subgraph - select { - case inputChan <- newInputList: - case <-ctx.Done(): - return nil - } - - case outputList, ok := <-obj.outputChan: - // send the new output list downstream - if !ok { - obj.outputChan = nil - canReceiveMoreOutputLists = false - continue - } - - select { - case obj.init.Output <- outputList: - case <-ctx.Done(): - return nil - } - - case <-ctx.Done(): - return nil - } - } -} - func (obj *FilterFunc) replaceSubGraph(subgraphInput interfaces.Func) error { // replaceSubGraph creates a subgraph which first splits the input list // into 'n' nodes. Then it applies 'newFuncValue' to each, and sends @@ -314,16 +263,9 @@ func (obj *FilterFunc) replaceSubGraph(subgraphInput interfaces.Func) error { // "filterCombineList0" -> "outputListFunc" // "filterCombineList1" -> "outputListFunc" // "filterCombineList2" -> "outputListFunc" - // "outputListFunc" -> "filterSubgraphOutput" + // + // "outputListFunc" -> "funcSubgraphOutput" // } - const channelBasedSinkFuncArgNameEdgeName = structs.ChannelBasedSinkFuncArgName // XXX: not sure if the specific name matters. - - // We pack the value pairs into structs that look like this... - structType := types.NewType(fmt.Sprintf("struct{v %s; b bool}", obj.Type.String())) - getArgName := func(i int) string { - return fmt.Sprintf("outputElem%d", i) - } - argNameInputList := "inputList" // delete the old subgraph if err := obj.init.Txn.Reverse(); err != nil { @@ -331,15 +273,26 @@ func (obj *FilterFunc) replaceSubGraph(subgraphInput interfaces.Func) error { } // create the new subgraph - obj.outputChan = make(chan types.Value) - subgraphOutput := &structs.ChannelBasedSinkFunc{ - Name: "filterSubgraphOutput", - Target: obj, - EdgeName: channelBasedSinkFuncArgNameEdgeName, - Chan: obj.outputChan, - Type: obj.listType, + + // XXX: Should we move creation of funcSubgraphOutput into Init() ? + funcSubgraphOutput := &structs.OutputFunc{ // the new graph shape thing! + //Textarea: obj.Textarea, + Name: "funcSubgraphOutput", + Type: obj.sig().Out, + EdgeName: structs.OutputFuncArgName, } - obj.init.Txn.AddVertex(subgraphOutput) + obj.init.Txn.AddVertex(funcSubgraphOutput) + obj.init.Txn.AddEdge(funcSubgraphOutput, obj.outputFunc, &interfaces.FuncEdge{Args: []string{structs.OutputFuncArgName}}) // "out" + + // XXX: hack add this edge that I thought would happen in call.go + obj.init.Txn.AddEdge(obj, funcSubgraphOutput, &interfaces.FuncEdge{Args: []string{structs.OutputFuncDummyArgName}}) // "dummy" + + // We pack the value pairs into structs that look like this... + structType := types.NewType(fmt.Sprintf("struct{v %s; b bool}", obj.Type.String())) + getArgName := func(i int) string { + return fmt.Sprintf("outputElem%d", i) + } + argNameInputList := "inputList" m := make(map[string]*types.Type) ord := []string{} @@ -385,10 +338,9 @@ func (obj *FilterFunc) replaceSubGraph(subgraphInput interfaces.Func) error { }, ) + edge := &interfaces.FuncEdge{Args: []string{structs.OutputFuncArgName}} // "out" obj.init.Txn.AddVertex(outputListFunc) - obj.init.Txn.AddEdge(outputListFunc, subgraphOutput, &interfaces.FuncEdge{ - Args: []string{channelBasedSinkFuncArgNameEdgeName}, - }) + obj.init.Txn.AddEdge(outputListFunc, funcSubgraphOutput, edge) for i := 0; i < obj.lastInputListLength; i++ { i := i @@ -414,7 +366,7 @@ func (obj *FilterFunc) replaceSubGraph(subgraphInput interfaces.Func) error { ) obj.init.Txn.AddVertex(inputElemFunc) - outputElemFunc, err := obj.lastFuncValue.CallWithFuncs(obj.init.Txn, []interfaces.Func{inputElemFunc}) + outputElemFunc, err := obj.lastFuncValue.CallWithFuncs(obj.init.Txn, []interfaces.Func{inputElemFunc}, funcSubgraphOutput) if err != nil { return errwrap.Wrapf(err, "could not call obj.lastFuncValue.CallWithFuncs()") } @@ -462,6 +414,76 @@ func (obj *FilterFunc) replaceSubGraph(subgraphInput interfaces.Func) error { return obj.init.Txn.Commit() } +// Call this function with the input args and return the value if it is possible +// to do so at this time. +func (obj *FilterFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) { + if len(args) < 2 { + return nil, fmt.Errorf("not enough args") + } + + // Check before we send to a chan where we'd need Stream to be running. + if obj.init == nil { + return nil, funcs.ErrCantSpeculate + } + + // Need this before we can *really* run this properly. + if len(obj.argFuncs) != 2 { + return nil, funcs.ErrCantSpeculate + //return nil, fmt.Errorf("unexpected input arg length") + } + + newInputList := args[0] + value := args[1] + newFuncValue, ok := value.(*full.FuncValue) + if !ok { + return nil, fmt.Errorf("programming error, can't convert to *FuncValue") + } + + a := obj.last != nil && newInputList.Cmp(obj.last) == nil + b := obj.lastFuncValue != nil && newFuncValue == obj.lastFuncValue + if a && b { + return types.NewNil(), nil // dummy value + } + obj.last = newInputList // store for next + obj.lastFuncValue = newFuncValue + + // Every time the FuncValue or the length of the list changes, recreate + // the subgraph, by calling the FuncValue N times on N nodes, each of + // which extracts one of the N values in the list. + + n := len(newInputList.List()) + + c := n == obj.lastInputListLength + if b && c { + return types.NewNil(), nil // dummy value + } + obj.lastInputListLength = n + + if b && !c { // different length list + return types.NewNil(), nil // dummy value + } + + // If we have a new function or the length of the input list has + // changed, then we need to replace the subgraph with a new one that + // uses the new function the correct number of times. + + subgraphInput := obj.argFuncs[0] + + // replaceSubGraph uses the above two values + if err := obj.replaceSubGraph(subgraphInput); err != nil { + return nil, errwrap.Wrapf(err, "could not replace subgraph") + } + + return nil, interfaces.ErrInterrupt +} + +// Cleanup runs after that function was removed from the graph. +func (obj *FilterFunc) Cleanup(ctx context.Context) error { + obj.init.Txn.Reverse() + //obj.init.Txn.DeleteVertex(subgraphInput) // XXX: should we delete it? + return obj.init.Txn.Commit() +} + // Copy is implemented so that the type value is not lost if we copy this // function. func (obj *FilterFunc) Copy() interfaces.Func { diff --git a/lang/core/iter/map.go b/lang/core/iter/map.go index a9f96f4c..e1fc6b78 100644 --- a/lang/core/iter/map.go +++ b/lang/core/iter/map.go @@ -81,10 +81,8 @@ type MapFunc struct { inputListType *types.Type outputListType *types.Type - // outputChan is an initially-nil channel from which we receive output - // lists from the subgraph. This channel is reset when the subgraph is - // recreated. - outputChan chan types.Value + argFuncs []interfaces.Func + outputFunc interfaces.Func } // String returns a simple name for this function. This is needed so this struct @@ -103,6 +101,11 @@ func (obj *MapFunc) ArgGen(index int) (string, error) { } // helper +// +// NOTE: The expression signature is shown here, but the actual "signature" of +// this in the function graph returns the "dummy" value because we do the same +// this that we do with ExprCall for example. That means that this function is +// one of very few where the actual expr signature is different from the func! func (obj *MapFunc) sig() *types.Type { // func(inputs []?1, function func(?1) ?2) []?2 tIi := "?1" @@ -186,17 +189,31 @@ func (obj *MapFunc) Build(typ *types.Type) (*types.Type, error) { return nil, errwrap.Wrapf(err, "return type of function must match returned list contents type") } + // TODO: Do we need to be extra careful and check that this matches? + // unificationUtil.UnifyCmp(typ, obj.sig()) != nil {} + obj.Type = tInputs.Val // or tArg obj.RType = tFunction.Out // or typ.Out.Val return obj.sig(), nil } +// SetShape tells the function about some special graph engine pointers. +func (obj *MapFunc) SetShape(argFuncs []interfaces.Func, outputFunc interfaces.Func) { + obj.argFuncs = argFuncs + obj.outputFunc = outputFunc +} + // Validate tells us if the input struct takes a valid form. func (obj *MapFunc) Validate() error { if obj.Type == nil || obj.RType == nil { return fmt.Errorf("type is not yet known") } + + if obj.argFuncs == nil || obj.outputFunc == nil { + return fmt.Errorf("function did not receive shape information") + } + return nil } @@ -207,7 +224,7 @@ func (obj *MapFunc) Info() *interfaces.Info { Pure: false, // XXX: what if the input function isn't pure? Memo: false, Fast: false, - Spec: false, + Spec: false, // must be false with the current graph shape code Sig: obj.sig(), // helper Err: obj.Validate(), } @@ -225,124 +242,6 @@ func (obj *MapFunc) Init(init *interfaces.Init) error { return nil } -// Stream returns the changing values that this func has over time. -func (obj *MapFunc) Stream(ctx context.Context) error { - // Every time the FuncValue or the length of the list changes, recreate the - // subgraph, by calling the FuncValue N times on N nodes, each of which - // extracts one of the N values in the list. - - defer close(obj.init.Output) // the sender closes - - // A Func to send input lists to the subgraph. The Txn.Erase() call ensures - // that this Func is not removed when the subgraph is recreated, so that the - // function graph can propagate the last list we received to the subgraph. - inputChan := make(chan types.Value) - subgraphInput := &structs.ChannelBasedSourceFunc{ - Name: "subgraphInput", - Source: obj, - Chan: inputChan, - Type: obj.inputListType, - } - obj.init.Txn.AddVertex(subgraphInput) - if err := obj.init.Txn.Commit(); err != nil { - return errwrap.Wrapf(err, "commit error in Stream") - } - obj.init.Txn.Erase() // prevent the next Reverse() from removing subgraphInput - defer func() { - close(inputChan) - obj.init.Txn.Reverse() - obj.init.Txn.DeleteVertex(subgraphInput) - obj.init.Txn.Commit() - }() - - obj.outputChan = nil - - canReceiveMoreFuncValuesOrInputLists := true - canReceiveMoreOutputLists := true - for { - - if !canReceiveMoreFuncValuesOrInputLists && !canReceiveMoreOutputLists { - //break - return nil - } - - select { - case input, ok := <-obj.init.Input: - if !ok { - obj.init.Input = nil // block looping back here - canReceiveMoreFuncValuesOrInputLists = false - continue - } - - if obj.last != nil && input.Cmp(obj.last) == nil { - continue // value didn't change, skip it - } - obj.last = input // store for next - - value, exists := input.Struct()[mapArgNameFunction] - if !exists { - return fmt.Errorf("programming error, can't find edge") - } - - newFuncValue, ok := value.(*full.FuncValue) - if !ok { - return fmt.Errorf("programming error, can't convert to *FuncValue") - } - - newInputList, exists := input.Struct()[mapArgNameInputs] - if !exists { - return fmt.Errorf("programming error, can't find edge") - } - - // If we have a new function or the length of the input - // list has changed, then we need to replace the - // subgraph with a new one that uses the new function - // the correct number of times. - - // It's important to have this compare step to avoid - // redundant graph replacements which slow things down, - // but also cause the engine to lock, which can preempt - // the process scheduler, which can cause duplicate or - // unnecessary re-sending of values here, which causes - // the whole process to repeat ad-nauseum. - n := len(newInputList.List()) - if newFuncValue != obj.lastFuncValue || n != obj.lastInputListLength { - obj.lastFuncValue = newFuncValue - obj.lastInputListLength = n - // replaceSubGraph uses the above two values - if err := obj.replaceSubGraph(subgraphInput); err != nil { - return errwrap.Wrapf(err, "could not replace subgraph") - } - canReceiveMoreOutputLists = true - } - - // send the new input list to the subgraph - select { - case inputChan <- newInputList: - case <-ctx.Done(): - return nil - } - - case outputList, ok := <-obj.outputChan: - // send the new output list downstream - if !ok { - obj.outputChan = nil - canReceiveMoreOutputLists = false - continue - } - - select { - case obj.init.Output <- outputList: - case <-ctx.Done(): - return nil - } - - case <-ctx.Done(): - return nil - } - } -} - func (obj *MapFunc) replaceSubGraph(subgraphInput interfaces.Func) error { // Create a subgraph which splits the input list into 'n' nodes, applies // 'newFuncValue' to each, then combines the 'n' outputs back into a @@ -363,11 +262,9 @@ func (obj *MapFunc) replaceSubGraph(subgraphInput interfaces.Func) error { // "outputElem1" -> "outputListFunc" // "outputElem2" -> "outputListFunc" // - // "outputListFunc" -> "mapSubgraphOutput" + // "outputListFunc" -> "funcSubgraphOutput" // } - const channelBasedSinkFuncArgNameEdgeName = structs.ChannelBasedSinkFuncArgName // XXX: not sure if the specific name matters. - // delete the old subgraph if err := obj.init.Txn.Reverse(); err != nil { return errwrap.Wrapf(err, "could not Reverse") @@ -375,15 +272,18 @@ func (obj *MapFunc) replaceSubGraph(subgraphInput interfaces.Func) error { // create the new subgraph - obj.outputChan = make(chan types.Value) - subgraphOutput := &structs.ChannelBasedSinkFunc{ - Name: "mapSubgraphOutput", - Target: obj, - EdgeName: channelBasedSinkFuncArgNameEdgeName, - Chan: obj.outputChan, - Type: obj.outputListType, + // XXX: Should we move creation of funcSubgraphOutput into Init() ? + funcSubgraphOutput := &structs.OutputFunc{ // the new graph shape thing! + //Textarea: obj.Textarea, + Name: "funcSubgraphOutput", + Type: obj.sig().Out, + EdgeName: structs.OutputFuncArgName, } - obj.init.Txn.AddVertex(subgraphOutput) + obj.init.Txn.AddVertex(funcSubgraphOutput) + obj.init.Txn.AddEdge(funcSubgraphOutput, obj.outputFunc, &interfaces.FuncEdge{Args: []string{structs.OutputFuncArgName}}) // "out" + + // XXX: hack add this edge that I thought would happen in call.go + obj.init.Txn.AddEdge(obj, funcSubgraphOutput, &interfaces.FuncEdge{Args: []string{structs.OutputFuncDummyArgName}}) // "dummy" m := make(map[string]*types.Type) ord := []string{} @@ -398,6 +298,7 @@ func (obj *MapFunc) replaceSubGraph(subgraphInput interfaces.Func) error { Ord: ord, Out: obj.outputListType, } + outputListFunc := structs.SimpleFnToDirectFunc( "mapOutputList", &types.FuncValue{ @@ -413,10 +314,9 @@ func (obj *MapFunc) replaceSubGraph(subgraphInput interfaces.Func) error { }, ) + edge := &interfaces.FuncEdge{Args: []string{structs.OutputFuncArgName}} // "out" obj.init.Txn.AddVertex(outputListFunc) - obj.init.Txn.AddEdge(outputListFunc, subgraphOutput, &interfaces.FuncEdge{ - Args: []string{channelBasedSinkFuncArgNameEdgeName}, - }) + obj.init.Txn.AddEdge(outputListFunc, funcSubgraphOutput, edge) for i := 0; i < obj.lastInputListLength; i++ { i := i @@ -434,6 +334,7 @@ func (obj *MapFunc) replaceSubGraph(subgraphInput interfaces.Func) error { return nil, fmt.Errorf("inputElemFunc: expected a ListValue argument") } + // Extract the correct list element. return list.List()[i], nil }, T: types.NewType(fmt.Sprintf("func(inputList %s) %s", obj.inputListType, obj.Type)), @@ -441,7 +342,7 @@ func (obj *MapFunc) replaceSubGraph(subgraphInput interfaces.Func) error { ) obj.init.Txn.AddVertex(inputElemFunc) - outputElemFunc, err := obj.lastFuncValue.CallWithFuncs(obj.init.Txn, []interfaces.Func{inputElemFunc}) + outputElemFunc, err := obj.lastFuncValue.CallWithFuncs(obj.init.Txn, []interfaces.Func{inputElemFunc}, funcSubgraphOutput) if err != nil { return errwrap.Wrapf(err, "could not call obj.lastFuncValue.CallWithFuncs()") } @@ -457,6 +358,76 @@ func (obj *MapFunc) replaceSubGraph(subgraphInput interfaces.Func) error { return obj.init.Txn.Commit() } +// Call this function with the input args and return the value if it is possible +// to do so at this time. +func (obj *MapFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) { + if len(args) < 2 { + return nil, fmt.Errorf("not enough args") + } + + // Check before we send to a chan where we'd need Stream to be running. + if obj.init == nil { + return nil, funcs.ErrCantSpeculate + } + + // Need this before we can *really* run this properly. + if len(obj.argFuncs) != 2 { + return nil, funcs.ErrCantSpeculate + //return nil, fmt.Errorf("unexpected input arg length") + } + + newInputList := args[0] + value := args[1] + newFuncValue, ok := value.(*full.FuncValue) + if !ok { + return nil, fmt.Errorf("programming error, can't convert to *FuncValue") + } + + a := obj.last != nil && newInputList.Cmp(obj.last) == nil + b := obj.lastFuncValue != nil && newFuncValue == obj.lastFuncValue + if a && b { + return types.NewNil(), nil // dummy value + } + obj.last = newInputList // store for next + obj.lastFuncValue = newFuncValue + + // Every time the FuncValue or the length of the list changes, recreate + // the subgraph, by calling the FuncValue N times on N nodes, each of + // which extracts one of the N values in the list. + + n := len(newInputList.List()) + + c := n == obj.lastInputListLength + if b && c { + return types.NewNil(), nil // dummy value + } + obj.lastInputListLength = n + + if b && !c { // different length list + return types.NewNil(), nil // dummy value + } + + // If we have a new function or the length of the input list has + // changed, then we need to replace the subgraph with a new one that + // uses the new function the correct number of times. + + subgraphInput := obj.argFuncs[0] + + // replaceSubGraph uses the above two values + if err := obj.replaceSubGraph(subgraphInput); err != nil { + return nil, errwrap.Wrapf(err, "could not replace subgraph") + } + + return nil, interfaces.ErrInterrupt +} + +// Cleanup runs after that function was removed from the graph. +func (obj *MapFunc) Cleanup(ctx context.Context) error { + obj.init.Txn.Reverse() + //obj.init.Txn.DeleteVertex(subgraphInput) // XXX: should we delete it? + return obj.init.Txn.Commit() +} + // Copy is implemented so that the type values are not lost if we copy this // function. func (obj *MapFunc) Copy() interfaces.Func { diff --git a/lang/core/iter/range.go b/lang/core/iter/range.go index 748f5989..39783549 100644 --- a/lang/core/iter/range.go +++ b/lang/core/iter/range.go @@ -47,7 +47,6 @@ const ( RangeFuncName = "range" ) -var _ interfaces.CallableFunc = &RangeFunc{} var _ interfaces.BuildableFunc = &RangeFunc{} // RangeFunc is a function that ranges over elements on a list according to @@ -175,49 +174,6 @@ func (obj *RangeFunc) Init(init *interfaces.Init) error { return nil } -// Stream returns the changing values that this func has over time. -func (obj *RangeFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // closing the sender - - for { - select { - case input, ok := <-obj.init.Input: - if !ok { - return nil // we don't have more inputs - } - - if obj.last != nil && input.Cmp(obj.last) == nil { - continue // nothing has changed, skip it - } - obj.last = input // storing the input for comparison - - args, err := interfaces.StructToCallableArgs(input) - if err != nil { - return err - } - - result, err := obj.Call(ctx, args) - if err != nil { - return err - } - - if obj.result != nil && result.Cmp(obj.result) == nil { - continue // if the result didn't change, we don't need to update - } - obj.result = result // store new result - - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- obj.result: // sending new result - case <-ctx.Done(): - return nil - } - } -} - // Call returns the result of this function. func (obj *RangeFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) { if len(args) == 1 { // we only have stop, assume start is 0 and step is 1 diff --git a/lang/core/local/pool.go b/lang/core/local/pool.go index 79bb4365..1e875be8 100644 --- a/lang/core/local/pool.go +++ b/lang/core/local/pool.go @@ -115,48 +115,6 @@ func (obj *PoolFunc) Init(init *interfaces.Init) error { return nil } -// Stream returns the changing values that this func has over time. -func (obj *PoolFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes - var value types.Value - for { - select { - case input, ok := <-obj.init.Input: - if !ok { - obj.init.Input = nil // don't infinite loop back - continue // no more inputs, but don't return! - } - //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { - // return errwrap.Wrapf(err, "wrong function input") - //} - - if obj.last != nil && input.Cmp(obj.last) == nil { - continue // value didn't change, skip it - } - obj.last = input // store for next - - args, err := interfaces.StructToCallableArgs(input) // []types.Value, error) - if err != nil { - return err - } - result, err := obj.Call(ctx, args) - if err != nil { - return err - } - value = result - - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- value: - case <-ctx.Done(): - return nil - } - } -} - // Copy is implemented so that the obj.built value is not lost if we copy this // function. func (obj *PoolFunc) Copy() interfaces.Func { @@ -181,6 +139,7 @@ func (obj *PoolFunc) Call(ctx context.Context, args []types.Value) (types.Value, if obj.init == nil { return nil, funcs.ErrCantSpeculate } + result, err := obj.init.Local.Pool(ctx, namespace, uid, nil) if err != nil { return nil, err diff --git a/lang/core/local/vardir.go b/lang/core/local/vardir.go index 675bb2ae..ed127e4c 100644 --- a/lang/core/local/vardir.go +++ b/lang/core/local/vardir.go @@ -122,48 +122,6 @@ func (obj *VarDirFunc) Init(init *interfaces.Init) error { return nil } -// Stream returns the changing values that this func has over time. -func (obj *VarDirFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes - var value types.Value - for { - select { - case input, ok := <-obj.init.Input: - if !ok { - obj.init.Input = nil // don't infinite loop back - continue // no more inputs, but don't return! - } - //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { - // return errwrap.Wrapf(err, "wrong function input") - //} - - if obj.last != nil && input.Cmp(obj.last) == nil { - continue // value didn't change, skip it - } - obj.last = input // store for next - - args, err := interfaces.StructToCallableArgs(input) // []types.Value, error) - if err != nil { - return err - } - result, err := obj.Call(ctx, args) - if err != nil { - return err - } - value = result - - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- value: - case <-ctx.Done(): - return nil - } - } -} - // Copy is implemented so that the obj.built value is not lost if we copy this // function. func (obj *VarDirFunc) Copy() interfaces.Func { diff --git a/lang/core/lookup.go b/lang/core/lookup.go index 3f63e43a..dd207b63 100644 --- a/lang/core/lookup.go +++ b/lang/core/lookup.go @@ -183,11 +183,6 @@ func (obj *LookupFunc) Build(typ *types.Type) (*types.Type, error) { return nil, err } - if _, ok := f.(interfaces.CallableFunc); !ok { - // programming error - return nil, fmt.Errorf("not a CallableFunc") - } - bf, ok := f.(interfaces.BuildableFunc) if !ok { // programming error @@ -248,23 +243,10 @@ func (obj *LookupFunc) Init(init *interfaces.Init) error { return obj.fn.Init(init) } -// Stream returns the changing values that this func has over time. -func (obj *LookupFunc) Stream(ctx context.Context) error { - if obj.fn == nil { - return fmt.Errorf("function not built correctly") - } - return obj.fn.Stream(ctx) -} - // Call returns the result of this function. func (obj *LookupFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) { if obj.fn == nil { return nil, funcs.ErrCantSpeculate } - cf, ok := obj.fn.(interfaces.CallableFunc) - if !ok { - // programming error - return nil, fmt.Errorf("not a CallableFunc") - } - return cf.Call(ctx, args) + return obj.fn.Call(ctx, args) } diff --git a/lang/core/lookup_default.go b/lang/core/lookup_default.go index 7a78eef1..05b5f8b2 100644 --- a/lang/core/lookup_default.go +++ b/lang/core/lookup_default.go @@ -128,11 +128,6 @@ func (obj *LookupDefaultFunc) Build(typ *types.Type) (*types.Type, error) { return nil, err } - if _, ok := f.(interfaces.CallableFunc); !ok { - // programming error - return nil, fmt.Errorf("not a CallableFunc") - } - bf, ok := f.(interfaces.BuildableFunc) if !ok { // programming error @@ -194,23 +189,10 @@ func (obj *LookupDefaultFunc) Init(init *interfaces.Init) error { return obj.fn.Init(init) } -// Stream returns the changing values that this func has over time. -func (obj *LookupDefaultFunc) Stream(ctx context.Context) error { - if obj.fn == nil { - return fmt.Errorf("function not built correctly") - } - return obj.fn.Stream(ctx) -} - // Call returns the result of this function. func (obj *LookupDefaultFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) { if obj.fn == nil { return nil, funcs.ErrCantSpeculate } - cf, ok := obj.fn.(interfaces.CallableFunc) - if !ok { - // programming error - return nil, fmt.Errorf("not a CallableFunc") - } - return cf.Call(ctx, args) + return obj.fn.Call(ctx, args) } diff --git a/lang/core/os/modinfo.go b/lang/core/os/modinfo.go index c9b89b48..d761ba66 100644 --- a/lang/core/os/modinfo.go +++ b/lang/core/os/modinfo.go @@ -64,8 +64,8 @@ type ModinfoLoadedFunc struct { init *interfaces.Init last types.Value // last value received to use for diff + input chan string // stream of inputs modulename *string // the active module name - result types.Value // last calculated output } // String returns a simple name for this function. This is needed so this struct @@ -103,13 +103,12 @@ func (obj *ModinfoLoadedFunc) Info() *interfaces.Info { // Init runs some startup code for this function. func (obj *ModinfoLoadedFunc) Init(init *interfaces.Init) error { obj.init = init + obj.input = make(chan string) return nil } // Stream returns the changing values that this func has over time. func (obj *ModinfoLoadedFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes - // create new watcher // XXX: does this file produce inotify events? recWatcher := &recwatch.RecWatcher{ @@ -127,22 +126,12 @@ func (obj *ModinfoLoadedFunc) Stream(ctx context.Context) error { for { select { - case input, ok := <-obj.init.Input: + case modulename, ok := <-obj.input: if !ok { - obj.init.Input = nil // don't infinite loop back - continue // no more inputs, but don't return! + obj.input = nil // don't infinite loop back + return fmt.Errorf("unexpected close") } - //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { - // return errwrap.Wrapf(err, "wrong function input") - //} - if obj.last != nil && input.Cmp(obj.last) == nil { - continue // value didn't change, skip it - } - obj.last = input // store for next - - modulename := input.Struct()[modinfoLoadedArgNameModule].Str() - // TODO: add check for empty string if obj.modulename != nil && *obj.modulename == modulename { continue // nothing changed } @@ -156,33 +145,13 @@ func (obj *ModinfoLoadedFunc) Stream(ctx context.Context) error { return errwrap.Wrapf(err, "error event received") } - if obj.last == nil { + if obj.modulename == nil { continue // still waiting for input values } - case <-ctx.Done(): - return nil - } - - args, err := interfaces.StructToCallableArgs(obj.last) // []types.Value, error) - if err != nil { - return err - } - - result, err := obj.Call(ctx, args) - if err != nil { - return err - } - - // if the result is still the same, skip sending an update... - if obj.result != nil && result.Cmp(obj.result) == nil { - continue // result didn't change - } - obj.result = result // store new result - - select { - case obj.init.Output <- obj.result: // send - // pass + if err := obj.init.Event(ctx); err != nil { // send event + return err + } case <-ctx.Done(): return nil @@ -198,6 +167,20 @@ func (obj *ModinfoLoadedFunc) Call(ctx context.Context, args []types.Value) (typ } modulename := args[0].Str() + // Check before we send to a chan where we'd need Stream to be running. + if obj.init == nil { + return nil, funcs.ErrCantSpeculate + } + + // Tell the Stream what we're watching now... This doesn't block because + // Stream should always be ready to consume unless it's closing down... + // If it dies, then a ctx closure should come soon. + select { + case obj.input <- modulename: + case <-ctx.Done(): + return nil, ctx.Err() + } + m, err := lsmod.LsMod() if err != nil { return nil, errwrap.Wrapf(err, "error reading modules") diff --git a/lang/core/os/readfile.go b/lang/core/os/readfile.go index 8bb06b47..68eb3560 100644 --- a/lang/core/os/readfile.go +++ b/lang/core/os/readfile.go @@ -60,15 +60,13 @@ func init() { // package. type ReadFileFunc struct { init *interfaces.Init - last types.Value // last value received to use for diff recWatcher *recwatch.RecWatcher events chan error // internal events wg *sync.WaitGroup - args []types.Value + input chan string // stream of inputs filename *string // the active filename - result types.Value // last calculated output } // String returns a simple name for this function. This is needed so this struct @@ -106,6 +104,7 @@ func (obj *ReadFileFunc) Info() *interfaces.Info { // Init runs some startup code for this function. func (obj *ReadFileFunc) Init(init *interfaces.Init) error { obj.init = init + obj.input = make(chan string) obj.events = make(chan error) obj.wg = &sync.WaitGroup{} return nil @@ -113,8 +112,8 @@ func (obj *ReadFileFunc) Init(init *interfaces.Init) error { // Stream returns the changing values that this func has over time. func (obj *ReadFileFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes - defer close(obj.events) // clean up for fun + //defer close(obj.input) // if we close, this is a race with the sender + defer close(obj.events) // clean up for fun defer obj.wg.Wait() defer func() { if obj.recWatcher != nil { @@ -124,24 +123,21 @@ func (obj *ReadFileFunc) Stream(ctx context.Context) error { }() for { select { - case input, ok := <-obj.init.Input: + case filename, ok := <-obj.input: if !ok { - obj.init.Input = nil // don't infinite loop back - continue // no more inputs, but don't return! + obj.input = nil // don't infinite loop back + return fmt.Errorf("unexpected close") } - //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { - // return errwrap.Wrapf(err, "wrong function input") - //} - if obj.last != nil && input.Cmp(obj.last) == nil { - continue // value didn't change, skip it - } - obj.last = input // store for next - - filename := input.Struct()[readFileArgNameFilename].Str() - // TODO: add validation for absolute path? - // TODO: add check for empty string if obj.filename != nil && *obj.filename == filename { + //select { + //case obj.ack <- struct{}{}: + //case <-ctx.Done(): + // // don't block here on shutdown + // return + //default: + // // pass, in case we didn't Call() + //} continue // nothing changed } obj.filename = &filename @@ -192,6 +188,17 @@ func (obj *ReadFileFunc) Stream(ctx context.Context) error { } } + // XXX: should we send an ACK event to + // Call() right here? This should ideally + // be from the Startup event of recWatcher + //select { + //case obj.ack <- struct{}{}: + //case <-ctx.Done(): + // // don't block here on shutdown + // return + //default: + // // pass, in case we didn't Call() + //} select { case obj.events <- err: // send event... @@ -213,37 +220,12 @@ func (obj *ReadFileFunc) Stream(ctx context.Context) error { return errwrap.Wrapf(err, "error event received") } - if obj.last == nil { - continue // still waiting for input values - } - - args, err := interfaces.StructToCallableArgs(obj.last) // []types.Value, error) - if err != nil { - return err - } - obj.args = args - - result, err := obj.Call(ctx, obj.args) - if err != nil { + if err := obj.init.Event(ctx); err != nil { // send event return err } - // if the result is still the same, skip sending an update... - if obj.result != nil && result.Cmp(obj.result) == nil { - continue // result didn't change - } - obj.result = result // store new result - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- obj.result: // send - // pass - - case <-ctx.Done(): - return nil + return ctx.Err() } } } @@ -256,6 +238,30 @@ func (obj *ReadFileFunc) Call(ctx context.Context, args []types.Value) (types.Va } filename := args[0].Str() + // TODO: add validation for absolute path? + // TODO: add check for empty string + + // Check before we send to a chan where we'd need Stream to be running. + if obj.init == nil { + return nil, funcs.ErrCantSpeculate + } + + // Tell the Stream what we're watching now... This doesn't block because + // Stream should always be ready to consume unless it's closing down... + // If it dies, then a ctx closure should come soon. + select { + case obj.input <- filename: + case <-ctx.Done(): + return nil, ctx.Err() + } + // XXX: Should we make sure the Stream is ready before we continue here? + //select { + //case <-obj.ack: + // // received + //case <-ctx.Done(): + // return nil, ctx.Err() + //} + // read file... content, err := os.ReadFile(filename) if err != nil { @@ -266,3 +272,20 @@ func (obj *ReadFileFunc) Call(ctx context.Context, args []types.Value) (types.Va V: string(content), // convert to string }, nil } + +// Cleanup runs after that function was removed from the graph. +func (obj *ReadFileFunc) Cleanup(ctx context.Context) error { + // Even if the filename stops changing, we never shutdown Stream because + // those file contents may change. Theoretically if someone sends us an + // empty string, and then it shuts down we could close. + //if obj.filename == "" { // we require obj.ack to not have a race here + // close(obj.exit) // add a channel into that Stream + //} + return nil +} + +// Done is a message from the engine to tell us that no more Call's are coming. +func (obj *ReadFileFunc) Done() error { + close(obj.input) // At this point we know obj.input won't be used. + return nil +} diff --git a/lang/core/os/readfilewait.go b/lang/core/os/readfilewait.go index 01809fc9..32729b0a 100644 --- a/lang/core/os/readfilewait.go +++ b/lang/core/os/readfilewait.go @@ -62,15 +62,13 @@ func init() { // will eventually be deprecated when the function graph error system is stable. type ReadFileWaitFunc struct { init *interfaces.Init - last types.Value // last value received to use for diff recWatcher *recwatch.RecWatcher events chan error // internal events wg *sync.WaitGroup - args []types.Value + input chan string // stream of inputs filename *string // the active filename - result types.Value // last calculated output } // String returns a simple name for this function. This is needed so this struct @@ -108,6 +106,7 @@ func (obj *ReadFileWaitFunc) Info() *interfaces.Info { // Init runs some startup code for this function. func (obj *ReadFileWaitFunc) Init(init *interfaces.Init) error { obj.init = init + obj.input = make(chan string) obj.events = make(chan error) obj.wg = &sync.WaitGroup{} return nil @@ -115,8 +114,8 @@ func (obj *ReadFileWaitFunc) Init(init *interfaces.Init) error { // Stream returns the changing values that this func has over time. func (obj *ReadFileWaitFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes - defer close(obj.events) // clean up for fun + //defer close(obj.input) // if we close, this is a race with the sender + defer close(obj.events) // clean up for fun defer obj.wg.Wait() defer func() { if obj.recWatcher != nil { @@ -126,24 +125,21 @@ func (obj *ReadFileWaitFunc) Stream(ctx context.Context) error { }() for { select { - case input, ok := <-obj.init.Input: + case filename, ok := <-obj.input: if !ok { - obj.init.Input = nil // don't infinite loop back - continue // no more inputs, but don't return! + obj.input = nil // don't infinite loop back + return fmt.Errorf("unexpected close") } - //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { - // return errwrap.Wrapf(err, "wrong function input") - //} - if obj.last != nil && input.Cmp(obj.last) == nil { - continue // value didn't change, skip it - } - obj.last = input // store for next - - filename := input.Struct()[readFileWaitArgNameFilename].Str() - // TODO: add validation for absolute path? - // TODO: add check for empty string if obj.filename != nil && *obj.filename == filename { + //select { + //case obj.ack <- struct{}{}: + //case <-ctx.Done(): + // // don't block here on shutdown + // return + //default: + // // pass, in case we didn't Call() + //} continue // nothing changed } obj.filename = &filename @@ -194,6 +190,17 @@ func (obj *ReadFileWaitFunc) Stream(ctx context.Context) error { } } + // XXX: should we send an ACK event to + // Call() right here? This should ideally + // be from the Startup event of recWatcher + //select { + //case obj.ack <- struct{}{}: + //case <-ctx.Done(): + // // don't block here on shutdown + // return + //default: + // // pass, in case we didn't Call() + //} select { case obj.events <- err: // send event... @@ -215,37 +222,12 @@ func (obj *ReadFileWaitFunc) Stream(ctx context.Context) error { return errwrap.Wrapf(err, "error event received") } - if obj.last == nil { - continue // still waiting for input values - } - - args, err := interfaces.StructToCallableArgs(obj.last) // []types.Value, error) - if err != nil { - return err - } - obj.args = args - - result, err := obj.Call(ctx, obj.args) - if err != nil { + if err := obj.init.Event(ctx); err != nil { // send event return err } - // if the result is still the same, skip sending an update... - if obj.result != nil && result.Cmp(obj.result) == nil { - continue // result didn't change - } - obj.result = result // store new result - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- obj.result: // send - // pass - - case <-ctx.Done(): - return nil + return ctx.Err() } } } @@ -258,6 +240,30 @@ func (obj *ReadFileWaitFunc) Call(ctx context.Context, args []types.Value) (type } filename := args[0].Str() + // TODO: add validation for absolute path? + // TODO: add check for empty string + + // Check before we send to a chan where we'd need Stream to be running. + if obj.init == nil { + return nil, funcs.ErrCantSpeculate + } + + // Tell the Stream what we're watching now... This doesn't block because + // Stream should always be ready to consume unless it's closing down... + // If it dies, then a ctx closure should come soon. + select { + case obj.input <- filename: + case <-ctx.Done(): + return nil, ctx.Err() + } + // XXX: Should we make sure the Stream is ready before we continue here? + //select { + //case <-obj.ack: + // // received + //case <-ctx.Done(): + // return nil, ctx.Err() + //} + // read file... content, err := os.ReadFile(filename) if err != nil && !os.IsNotExist(err) { // ignore file not found errors @@ -269,6 +275,23 @@ func (obj *ReadFileWaitFunc) Call(ctx context.Context, args []types.Value) (type //} return &types.StrValue{ - V: s, + V: s, // convert to string }, nil } + +// Cleanup runs after that function was removed from the graph. +func (obj *ReadFileWaitFunc) Cleanup(ctx context.Context) error { + // Even if the filename stops changing, we never shutdown Stream because + // those file contents may change. Theoretically if someone sends us an + // empty string, and then it shuts down we could close. + //if obj.filename == "" { // we require obj.ack to not have a race here + // close(obj.exit) // add a channel into that Stream + //} + return nil +} + +// Done is a message from the engine to tell us that no more Call's are coming. +func (obj *ReadFileWaitFunc) Done() error { + close(obj.input) // At this point we know obj.input won't be used. + return nil +} diff --git a/lang/core/os/system.go b/lang/core/os/system.go index 8d7a67f8..774982af 100644 --- a/lang/core/os/system.go +++ b/lang/core/os/system.go @@ -47,6 +47,10 @@ const ( // arg names... systemArgNameCmd = "cmd" + + // SystemFuncBufferLength is the number of lines we can buffer before we + // block. If you need a larger value, please let us know your use-case. + SystemFuncBufferLength = 1024 ) func init() { @@ -60,9 +64,25 @@ func init() { // Note that in the likely case in which the process emits several lines one // after the other, the downstream resources might not run for every line unless // the "Meta:realize" metaparam is set to true. +// +// Furthermore, there is no guarantee that every intermediate line will be seen, +// particularly if there is no delay between them. Only the last line is +// guaranteed. As a result, it is not recommend to use this for timing or +// coordination. If you are using this for an intermediate value, or a +// non-declarative system, then it's likely you are using this wrong. type SystemFunc struct { init *interfaces.Init cancel context.CancelFunc + + input chan string // stream of inputs + + last *string // the active command + output *string // the last output + + values chan string + + count int + mutex *sync.Mutex } // String returns a simple name for this function. This is needed so this struct @@ -101,18 +121,18 @@ func (obj *SystemFunc) Info() *interfaces.Info { // Init runs some startup code for this function. func (obj *SystemFunc) Init(init *interfaces.Init) error { obj.init = init + obj.input = make(chan string) + obj.values = make(chan string, SystemFuncBufferLength) + obj.mutex = &sync.Mutex{} return nil } // Stream returns the changing values that this func has over time. -func (obj *SystemFunc) Stream(ctx context.Context) error { +func (obj *SystemFunc) Stream(ctx context.Context) (reterr error) { // XXX: this implementation is a bit awkward especially with the port to // the Stream(context.Context) signature change. This is a straight port // but we could refactor this eventually. - // Close the output chan to signal that no more values are coming. - defer close(obj.init.Output) - // A channel which closes when the current process exits, on its own // or due to cancel(). The channel is only closed once all the pending // stdout and stderr lines have been processed. @@ -140,7 +160,7 @@ func (obj *SystemFunc) Stream(ctx context.Context) error { for { select { - case input, more := <-obj.init.Input: + case shellCommand, more := <-obj.input: if !more { // Wait until the current process exits and all of its // stdout is sent downstream. @@ -151,7 +171,11 @@ func (obj *SystemFunc) Stream(ctx context.Context) error { return nil } } - shellCommand := input.Struct()[systemArgNameCmd].Str() + + if obj.last != nil && *obj.last == shellCommand { + continue // nothing changed + } + obj.last = &shellCommand // Kill the previous command, if any. if obj.cancel != nil { @@ -193,8 +217,22 @@ func (obj *SystemFunc) Stream(ctx context.Context) error { stdoutScanner := bufio.NewScanner(stdoutReader) for stdoutScanner.Scan() { - outputValue := &types.StrValue{V: stdoutScanner.Text()} - obj.init.Output <- outputValue + s := stdoutScanner.Text() + obj.mutex.Lock() + obj.count++ + obj.mutex.Unlock() + select { + case obj.values <- s: // buffered + case <-ctx.Done(): + // don't block here on shutdown + reterr = ctx.Err() // return err + return + } + + if err := obj.init.Event(ctx); err != nil { // send event + reterr = err // return err + return + } } }() @@ -220,8 +258,66 @@ func (obj *SystemFunc) Stream(ctx context.Context) error { wg.Wait() close(processedChan) }() + case <-ctx.Done(): - return nil + return ctx.Err() } } } + +// Call this function with the input args and return the value if it is possible +// to do so at this time. +func (obj *SystemFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) { + if len(args) < 1 { + return nil, fmt.Errorf("not enough args") + } + cmd := args[0].Str() + + // Check before we send to a chan where we'd need Stream to be running. + if obj.init == nil { + return nil, funcs.ErrCantSpeculate + } + + // Tell the Stream what we're watching now... This doesn't block because + // Stream should always be ready to consume unless it's closing down... + // If it dies, then a ctx closure should come soon. + select { + case obj.input <- cmd: + case <-ctx.Done(): + return nil, ctx.Err() + } + + obj.mutex.Lock() + // If there are no values, and we've previously received a value then... + if obj.count == 0 && obj.output != nil { + s := *obj.output + obj.mutex.Unlock() + return &types.StrValue{ + V: s, + }, nil + } + obj.count-- // we might be briefly negative + obj.mutex.Unlock() + + // We know a value must be coming (or the command blocks) so we wait... + select { + case s, ok := <-obj.values: + if !ok { + return nil, fmt.Errorf("unexpected close") + } + obj.output = &s // store + + return &types.StrValue{ + V: s, + }, nil + + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +// Done is a message from the engine to tell us that no more Call's are coming. +func (obj *SystemFunc) Done() error { + close(obj.input) // At this point we know obj.input won't be used. + return nil +} diff --git a/lang/core/random1.go b/lang/core/random1.go index 6d156cc7..e4c7b309 100644 --- a/lang/core/random1.go +++ b/lang/core/random1.go @@ -33,6 +33,7 @@ import ( "context" "crypto/rand" "fmt" + "math" "math/big" "github.com/purpleidea/mgmt/lang/funcs" @@ -55,20 +56,22 @@ func init() { funcs.Register(Random1FuncName, func() interfaces.Func { return &Random1Func{} }) } -// Random1Func returns one random string of a certain length. -// XXX: return a stream instead, and combine this with a first(?) function which -// takes the first value and then puts backpressure on the stream. This should -// notify parent functions somehow that their values are no longer required so -// that they can shutdown if possible. Maybe it should be returning a stream of -// floats [0,1] as well, which someone can later map to the alphabet that they -// want. Should random() take an interval to know how often to spit out values? -// It could also just do it once per second, and we could filter for less. If we -// want something high precision, we could add that in the future... We could -// name that "random" and this one can be "random1" until we deprecate it. +// Random1Func returns one random string of a certain length. If you change the +// length, then it will produce a new random value. type Random1Func struct { + // XXX: To produce a stream of random values every N seconds, make a + // built-in function or use the dual <|> hack below? + // XXX: Maybe it should be returning a stream of floats [0,1] as well, + // which someone can later map to the alphabet that they want. Should + // random() take an interval to know how often to spit out values? It + // could also just do it once per second, and we could filter for less. + // If we want something high precision, we could add that in the future. + // We could name that "random" and this one can be "random1" until we + // deprecate it. init *interfaces.Init - finished bool // did we send the random string? + length uint16 // last length + result string // last random } // String returns a simple name for this function. This is needed so this struct @@ -136,49 +139,50 @@ func (obj *Random1Func) Init(init *interfaces.Init) error { return nil } -// Stream returns the single value that was generated and then closes. -func (obj *Random1Func) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes - var result string - for { - select { - case input, ok := <-obj.init.Input: - if !ok { - return nil // can't output any more - } - //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { - // return errwrap.Wrapf(err, "wrong function input") - //} - - if obj.finished { - // TODO: continue instead? - return fmt.Errorf("you can only pass a single input to random") - } - - length := input.Struct()[random1ArgNameLength].Int() - // TODO: if negative, randomly pick a length ? - if length < 0 { - return fmt.Errorf("can't generate a negative length") - } - - var err error - if result, err = generate(uint16(length)); err != nil { - return err // no errwrap needed b/c helper func - } - - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- &types.StrValue{ - V: result, - }: - // we only send one value, then wait for input to close - obj.finished = true - - case <-ctx.Done(): - return nil - } +// Call this function with the input args and return the value if it is possible +// to do so at this time. +func (obj *Random1Func) Call(ctx context.Context, args []types.Value) (types.Value, error) { + if len(args) < 1 { + return nil, fmt.Errorf("not enough args") } + length := args[0].Int() + + if length < 0 || length > math.MaxUint16 { + // On error, reset the cached values. This *may* be useful if we + // want to use the future "except" operator to produce an stream + // of random values-- we could flip flop between two "random1()" + // functions to successively get a val from one, while resetting + // the other one. Which happens right here... Here's an example: + // + // $now = datetime.now() + // $len = 8 # length of rand + // # alternate every second + // $out = if math.mod($now, 2) == 0 { + // random1($len) <|> random1(-1) + // } else { + // random1(-1) <|> random1($len) + // } + // + // Perhaps it's just better to have a core rand stream function? + obj.length = 0 + obj.result = "" + return nil, fmt.Errorf("can't generate an invalid length") + } + + if uint16(length) == obj.length { // same, so use cached value + return &types.StrValue{ + V: obj.result, + }, nil + } + obj.length = uint16(length) // cache + + result, err := generate(uint16(length)) + if err != nil { + return nil, err // no errwrap needed b/c helper func + } + obj.result = result // cache + + return &types.StrValue{ + V: result, + }, nil } diff --git a/lang/core/struct_lookup.go b/lang/core/struct_lookup.go index 96ece315..ad62f2c0 100644 --- a/lang/core/struct_lookup.go +++ b/lang/core/struct_lookup.go @@ -283,62 +283,6 @@ func (obj *StructLookupFunc) Init(init *interfaces.Init) error { return nil } -// Stream returns the changing values that this func has over time. -func (obj *StructLookupFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes - for { - select { - case input, ok := <-obj.init.Input: - if !ok { - return nil // can't output any more - } - //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { - // return errwrap.Wrapf(err, "wrong function input") - //} - - if obj.last != nil && input.Cmp(obj.last) == nil { - continue // value didn't change, skip it - } - obj.last = input // store for next - - st := (input.Struct()[structLookupArgNameStruct]).(*types.StructValue) - field := input.Struct()[structLookupArgNameField].Str() - - if field == "" { - return fmt.Errorf("received empty field") - } - if obj.field == "" { - // This can happen at compile time too. Bonus! - obj.field = field // store first field - } - if field != obj.field { - return fmt.Errorf("input field changed from: `%s`, to: `%s`", obj.field, field) - } - result, exists := st.Lookup(obj.field) - if !exists { - return fmt.Errorf("could not lookup field: `%s` in struct", field) - } - - // if previous input was `2 + 4`, but now it - // changed to `1 + 5`, the result is still the - // same, so we can skip sending an update... - if obj.result != nil && result.Cmp(obj.result) == nil { - continue // result didn't change - } - obj.result = result // store new result - - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- obj.result: // send - case <-ctx.Done(): - return nil - } - } -} - // Call returns the result of this function. func (obj *StructLookupFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) { if len(args) < 2 { diff --git a/lang/core/struct_lookup_optional.go b/lang/core/struct_lookup_optional.go index 9f02c725..7b642403 100644 --- a/lang/core/struct_lookup_optional.go +++ b/lang/core/struct_lookup_optional.go @@ -280,71 +280,6 @@ func (obj *StructLookupOptionalFunc) Init(init *interfaces.Init) error { return nil } -// Stream returns the changing values that this func has over time. -func (obj *StructLookupOptionalFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes - for { - select { - case input, ok := <-obj.init.Input: - if !ok { - return nil // can't output any more - } - //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { - // return errwrap.Wrapf(err, "wrong function input") - //} - - if obj.last != nil && input.Cmp(obj.last) == nil { - continue // value didn't change, skip it - } - obj.last = input // store for next - - st := (input.Struct()[structLookupOptionalArgNameStruct]).(*types.StructValue) - field := input.Struct()[structLookupOptionalArgNameField].Str() - optional := input.Struct()[structLookupOptionalArgNameOptional] - - if field == "" { - return fmt.Errorf("received empty field") - } - if obj.field == "" { - // This can happen at compile time too. Bonus! - obj.field = field // store first field - } - if field != obj.field { - return fmt.Errorf("input field changed from: `%s`, to: `%s`", obj.field, field) - } - - // We know the result of this lookup statically at - // compile time, but for simplicity we check each time - // here anyways. Maybe one day there will be a fancy - // reason why this might vary over time. - var result types.Value - val, exists := st.Lookup(obj.field) - if exists { - result = val - } else { - result = optional - } - - // if previous input was `2 + 4`, but now it - // changed to `1 + 5`, the result is still the - // same, so we can skip sending an update... - if obj.result != nil && result.Cmp(obj.result) == nil { - continue // result didn't change - } - obj.result = result // store new result - - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- obj.result: // send - case <-ctx.Done(): - return nil - } - } -} - // Call returns the result of this function. func (obj *StructLookupOptionalFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) { if len(args) < 3 { diff --git a/lang/core/sys/cpucount.go b/lang/core/sys/cpucount.go index 07dc4b43..2a04f270 100644 --- a/lang/core/sys/cpucount.go +++ b/lang/core/sys/cpucount.go @@ -33,7 +33,6 @@ package coresys import ( "context" - "fmt" "os" "regexp" "strconv" @@ -97,24 +96,10 @@ func (obj *CPUCount) Init(init *interfaces.Init) error { return nil } -// Stream returns the changing values that this fact has over time. It will -// first poll sysfs to get the initial cpu count, and then receives UEvents from -// the kernel as CPUs are added/removed. +// Stream starts a mainloop and runs Event when it's time to Call() again. It +// will first poll sysfs to get the initial cpu count, and then receives UEvents +// from the kernel as CPUs are added/removed. func (obj CPUCount) Stream(ctx context.Context) error { - defer close(obj.init.Output) // signal when we're done - - // We always wait for our initial event to start. - select { - case _, ok := <-obj.init.Input: - if ok { - return fmt.Errorf("unexpected input") - } - obj.init.Input = nil - - case <-ctx.Done(): - return nil - } - ss, err := socketset.NewSocketSet(rtmGrps, socketFile, unix.NETLINK_KOBJECT_UEVENT) if err != nil { return errwrap.Wrapf(err, "error creating socket set") @@ -182,23 +167,9 @@ func (obj CPUCount) Stream(ctx context.Context) error { return nil } - result, err := obj.Call(ctx, nil) - if err != nil { + if err := obj.init.Event(ctx); err != nil { return err } - - // if the result is still the same, skip sending an update... - if obj.result != nil && result.Cmp(obj.result) == nil { - continue // result didn't change - } - obj.result = result // store new result - - select { - case obj.init.Output <- result: - - case <-ctx.Done(): - return nil - } } } diff --git a/lang/core/sys/cpucount_test.go b/lang/core/sys/cpucount_test.go index 10ced3c4..128130dc 100644 --- a/lang/core/sys/cpucount_test.go +++ b/lang/core/sys/cpucount_test.go @@ -32,50 +32,9 @@ package coresys import ( - "context" "testing" - - "github.com/purpleidea/mgmt/lang/interfaces" - "github.com/purpleidea/mgmt/lang/types" ) -func TestSimple(t *testing.T) { - fact := &CPUCount{} - - input := make(chan types.Value) - close(input) // kick it off! - output := make(chan types.Value) - err := fact.Init(&interfaces.Init{ - Input: input, - Output: output, - Logf: func(format string, v ...interface{}) { - t.Logf("cpucount_test: "+format, v...) - }, - }) - if err != nil { - t.Errorf("could not init CPUCount") - return - } - - ctx, cancel := context.WithCancel(context.Background()) - go func() { - defer cancel() - Loop: - for { - select { - case cpus := <-output: - t.Logf("CPUS: %d\n", cpus.Int()) - break Loop - } - } - }() - - // now start the stream - if err := fact.Stream(ctx); err != nil { - t.Error(err) - } -} - func TestParseCPUList(t *testing.T) { var cpulistTests = []struct { desc string diff --git a/lang/core/sys/hostname.go b/lang/core/sys/hostname.go index 5419fe90..9626611b 100644 --- a/lang/core/sys/hostname.go +++ b/lang/core/sys/hostname.go @@ -94,20 +94,6 @@ func (obj *Hostname) Init(init *interfaces.Init) error { // Stream returns the single value that this fact has, and then closes. func (obj *Hostname) Stream(ctx context.Context) error { - defer close(obj.init.Output) // signal that we're done sending - - // We always wait for our initial event to start. - select { - case _, ok := <-obj.init.Input: - if ok { - return fmt.Errorf("unexpected input") - } - obj.init.Input = nil - - case <-ctx.Done(): - return nil - } - recurse := false // single file recWatcher, err := recwatch.NewRecWatcher("/etc/hostname", recurse) if err != nil { @@ -165,19 +151,9 @@ func (obj *Hostname) Stream(ctx context.Context) error { return nil } - // NOTE: We ask the actual machine instead of using obj.init.Hostname - value, err := obj.Call(ctx, nil) - if err != nil { + if err := obj.init.Event(ctx); err != nil { return err } - - select { - case obj.init.Output <- value: - // pass - - case <-ctx.Done(): - return nil - } } } @@ -191,6 +167,7 @@ func (obj *Hostname) Call(ctx context.Context, args []types.Value) (types.Value, hostnameObject := conn.Object(hostname1Iface, hostname1Path) + // NOTE: We ask the actual machine instead of using obj.init.Hostname h, err := obj.getHostnameProperty(hostnameObject, "Hostname") if err != nil { return nil, err diff --git a/lang/core/sys/load.go b/lang/core/sys/load.go index 46d0a743..27a18b9e 100644 --- a/lang/core/sys/load.go +++ b/lang/core/sys/load.go @@ -85,22 +85,8 @@ func (obj *Load) Init(init *interfaces.Init) error { return nil } -// Stream returns the changing values that this fact has over time. +// Stream starts a mainloop and runs Event when it's time to Call() again. func (obj *Load) Stream(ctx context.Context) error { - defer close(obj.init.Output) // always signal when we're done - - // We always wait for our initial event to start. - select { - case _, ok := <-obj.init.Input: - if ok { - return fmt.Errorf("unexpected input") - } - obj.init.Input = nil - - case <-ctx.Done(): - return nil - } - // it seems the different values only update once every 5 // seconds, so that's as often as we need to refresh this! // TODO: lookup this value if it's something configurable @@ -124,17 +110,9 @@ func (obj *Load) Stream(ctx context.Context) error { return nil } - result, err := obj.Call(ctx, nil) - if err != nil { + if err := obj.init.Event(ctx); err != nil { return err } - - select { - case obj.init.Output <- result: - - case <-ctx.Done(): - return nil - } } } diff --git a/lang/core/sys/uptime.go b/lang/core/sys/uptime.go index fd03c83e..c590860a 100644 --- a/lang/core/sys/uptime.go +++ b/lang/core/sys/uptime.go @@ -31,7 +31,6 @@ package coresys import ( "context" - "fmt" "time" "github.com/purpleidea/mgmt/lang/funcs" @@ -83,22 +82,8 @@ func (obj *Uptime) Init(init *interfaces.Init) error { return nil } -// Stream returns the changing values that this fact has over time. +// Stream starts a mainloop and runs Event when it's time to Call() again. func (obj *Uptime) Stream(ctx context.Context) error { - defer close(obj.init.Output) - - // We always wait for our initial event to start. - select { - case _, ok := <-obj.init.Input: - if ok { - return fmt.Errorf("unexpected input") - } - obj.init.Input = nil - - case <-ctx.Done(): - return nil - } - ticker := time.NewTicker(time.Duration(1) * time.Second) defer ticker.Stop() @@ -112,24 +97,16 @@ func (obj *Uptime) Stream(ctx context.Context) error { case <-startChan: startChan = nil // disable - case <-ticker.C: - // send + case <-ticker.C: // received the timer event + // pass case <-ctx.Done(): return nil } - result, err := obj.Call(ctx, nil) - if err != nil { + if err := obj.init.Event(ctx); err != nil { return err } - - select { - case obj.init.Output <- result: - - case <-ctx.Done(): - return nil - } } } diff --git a/lang/core/test/fastcount.go b/lang/core/test/fastcount.go index 50d46413..49b0fc08 100644 --- a/lang/core/test/fastcount.go +++ b/lang/core/test/fastcount.go @@ -31,7 +31,6 @@ package coretest import ( "context" - "fmt" "sync" "github.com/purpleidea/mgmt/lang/funcs" @@ -87,38 +86,23 @@ func (obj *FastCount) Init(init *interfaces.Init) error { return nil } -// Stream returns the changing values that this fact has over time. +// Stream starts a mainloop and runs Event when it's time to Call() again. func (obj *FastCount) Stream(ctx context.Context) error { - defer close(obj.init.Output) // always signal when we're done - - // We always wait for our initial event to start. - select { - case _, ok := <-obj.init.Input: - if ok { - return fmt.Errorf("unexpected input") - } - obj.init.Input = nil - - case <-ctx.Done(): - return nil - } - - // streams must generate an initial event on startup for { - result, err := obj.Call(ctx, nil) - if err != nil { - return err + select { + case <-ctx.Done(): + return nil + + default: + // run free } obj.mutex.Lock() obj.count++ obj.mutex.Unlock() - select { - case obj.init.Output <- result: - - case <-ctx.Done(): - return nil + if err := obj.init.Event(ctx); err != nil { + return err } } } diff --git a/lang/core/test/oneinstance.go b/lang/core/test/oneinstance.go index b017670f..49eaff18 100644 --- a/lang/core/test/oneinstance.go +++ b/lang/core/test/oneinstance.go @@ -31,7 +31,6 @@ package coretest import ( "context" - "fmt" "sync" "github.com/purpleidea/mgmt/lang/funcs" @@ -268,38 +267,6 @@ func (obj *OneInstance) Init(init *interfaces.Init) error { return nil } -// Stream returns the changing values that this fact has over time. -func (obj *OneInstance) Stream(ctx context.Context) error { - obj.init.Logf("Stream of `%s` @ %p", obj.Name, obj) - defer close(obj.init.Output) // always signal when we're done - - // We always wait for our initial event to start. - select { - case _, ok := <-obj.init.Input: - if ok { - return fmt.Errorf("unexpected input") - } - obj.init.Input = nil - - case <-ctx.Done(): - return nil - } - - result, err := obj.Call(ctx, nil) - if err != nil { - return err - } - - select { - case obj.init.Output <- result: - - case <-ctx.Done(): - return nil - } - - return nil -} - // Call this fact and return the value if it is possible to do so at this time. func (obj *OneInstance) Call(ctx context.Context, args []types.Value) (types.Value, error) { return &types.StrValue{ diff --git a/lang/core/value/get.go b/lang/core/value/get.go index 608f4bfb..16617726 100644 --- a/lang/core/value/get.go +++ b/lang/core/value/get.go @@ -77,7 +77,7 @@ func init() { funcs.ModuleRegister(ModuleName, GetFloatFuncName, func() interfaces.Func { return &GetFunc{Type: types.TypeFloat} }) } -var _ interfaces.CallableFunc = &GetFunc{} +var _ interfaces.StreamableFunc = &GetFunc{} // GetFunc is special function which looks up the stored `Any` field in the // value resource that it gets it from. If it is initialized with a fixed Type @@ -90,11 +90,8 @@ type GetFunc struct { init *interfaces.Init - key string - args []types.Value - - last types.Value - result types.Value // last calculated output + input chan string // stream of inputs + key *string // the active key watchChan chan struct{} } @@ -216,13 +213,13 @@ func (obj *GetFunc) Info() *interfaces.Info { // Init runs some startup code for this function. func (obj *GetFunc) Init(init *interfaces.Init) error { obj.init = init + obj.input = make(chan string) obj.watchChan = make(chan struct{}) // sender closes this when Stream ends return nil } // Stream returns the changing values that this func has over time. func (obj *GetFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes ctx, cancel := context.WithCancel(ctx) defer cancel() // important so that we cleanup the watch when exiting for { @@ -230,51 +227,36 @@ func (obj *GetFunc) Stream(ctx context.Context) error { // TODO: should this first chan be run as a priority channel to // avoid some sort of glitch? is that even possible? can our // hostname check with reality (below) fix that? - case input, ok := <-obj.init.Input: + case key, ok := <-obj.input: if !ok { - obj.init.Input = nil // don't infinite loop back - continue // no more inputs, but don't return! + obj.input = nil // don't infinite loop back + return fmt.Errorf("unexpected close") } - //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { - // return errwrap.Wrapf(err, "wrong function input") - //} - if obj.last != nil && input.Cmp(obj.last) == nil { - continue // value didn't change, skip it - } - obj.last = input // store for next - - args, err := interfaces.StructToCallableArgs(input) // []types.Value, error) - if err != nil { - return err - } - obj.args = args - - key := args[0].Str() - if key == "" { - return fmt.Errorf("can't use an empty key") - } - if obj.init.Debug { - obj.init.Logf("key: %s", key) + if obj.key != nil && *obj.key == key { + continue // nothing changed } // We don't support changing the key over time, since it // might cause the type to need to be changed. - if obj.key == "" { - obj.key = key // store it + if obj.key == nil { + obj.key = &key // store it var err error // Don't send a value right away, wait for the // first ValueWatch startup event to get one! - obj.watchChan, err = obj.init.Local.ValueWatch(ctx, obj.key) // watch for var changes + obj.watchChan, err = obj.init.Local.ValueWatch(ctx, key) // watch for var changes if err != nil { return err } - - } else if obj.key != key { - return fmt.Errorf("can't change key, previously: `%s`", obj.key) + continue // we get values on the watch chan, not here! } - continue // we get values on the watch chan, not here! + if *obj.key == key { + continue // skip duplicates + } + + // *obj.key != key + return fmt.Errorf("can't change key, previously: `%s`", *obj.key) case _, ok := <-obj.watchChan: if !ok { // closed @@ -284,24 +266,10 @@ func (obj *GetFunc) Stream(ctx context.Context) error { // return errwrap.Wrapf(err, "channel watch failed on `%s`", obj.key) //} - result, err := obj.Call(ctx, obj.args) // get the value... - if err != nil { + if err := obj.init.Event(ctx); err != nil { // send event return err } - // if the result is still the same, skip sending an update... - if obj.result != nil && result.Cmp(obj.result) == nil { - continue // result didn't change - } - obj.result = result // store new result - - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- obj.result: // send - // pass case <-ctx.Done(): return nil } @@ -316,6 +284,18 @@ func (obj *GetFunc) Call(ctx context.Context, args []types.Value) (types.Value, return nil, fmt.Errorf("not enough args") } key := args[0].Str() + if key == "" { + return nil, fmt.Errorf("can't use an empty key") + } + + // Check before we send to a chan where we'd need Stream to be running. + if obj.init == nil { + return nil, funcs.ErrCantSpeculate + } + + if obj.init.Debug { + obj.init.Logf("key: %s", key) + } typ, exists := obj.Info().Sig.Out.Map[getFieldNameValue] // type of value field if !exists || typ == nil { @@ -323,6 +303,12 @@ func (obj *GetFunc) Call(ctx context.Context, args []types.Value) (types.Value, return nil, fmt.Errorf("missing type for %s field", getFieldNameValue) } + select { + case obj.input <- key: + case <-ctx.Done(): + return nil, ctx.Err() + } + // The API will pull from the on-disk stored cache if present... This // value comes from the field in the Value resource... We only have an // on-disk cache because since functions load before resources do, we'd @@ -331,9 +317,6 @@ func (obj *GetFunc) Call(ctx context.Context, args []types.Value) (types.Value, // step that might be needed if the value started out empty... // TODO: We could even add a stored: bool field in the returned struct! isReady := true // assume true - if obj.init == nil { - return nil, funcs.ErrCantSpeculate - } val, err := obj.init.Local.ValueGet(ctx, key) if err != nil { return nil, errwrap.Wrapf(err, "channel read failed on `%s`", key) diff --git a/lang/core/world/collect/res.go b/lang/core/world/collect/res.go index 2088b39a..0a2388e5 100644 --- a/lang/core/world/collect/res.go +++ b/lang/core/world/collect/res.go @@ -75,10 +75,8 @@ func init() { type ResFunc struct { init *interfaces.Init - last types.Value // last value received to use for diff - args []types.Value - kind string - result types.Value // last calculated output + input chan string // stream of inputs + kind *string // the active kind watchChan chan error } @@ -128,13 +126,13 @@ func (obj *ResFunc) Info() *interfaces.Info { // Init runs some startup code for this function. func (obj *ResFunc) Init(init *interfaces.Init) error { obj.init = init + obj.input = make(chan string) obj.watchChan = make(chan error) // XXX: sender should close this, but did I implement that part yet??? return nil } // Stream returns the changing values that this func has over time. func (obj *ResFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes ctx, cancel := context.WithCancel(ctx) defer cancel() // important so that we cleanup the watch when exiting for { @@ -142,50 +140,35 @@ func (obj *ResFunc) Stream(ctx context.Context) error { // TODO: should this first chan be run as a priority channel to // avoid some sort of glitch? is that even possible? can our // hostname check with reality (below) fix that? - case input, ok := <-obj.init.Input: + case kind, ok := <-obj.input: if !ok { - obj.init.Input = nil // don't infinite loop back - continue // no more inputs, but don't return! + obj.input = nil // don't infinite loop back + return fmt.Errorf("unexpected close") } - //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { - // return errwrap.Wrapf(err, "wrong function input") - //} - if obj.last != nil && input.Cmp(obj.last) == nil { - continue // value didn't change, skip it - } - obj.last = input // store for next - - args, err := interfaces.StructToCallableArgs(input) // []types.Value, error) - if err != nil { - return err - } - obj.args = args - - kind := args[0].Str() - if kind == "" { - return fmt.Errorf("can't use an empty kind") - } - if obj.init.Debug { - obj.init.Logf("kind: %s", kind) + if obj.kind != nil && *obj.kind == kind { + continue // nothing changed } // TODO: support changing the key over time? - if obj.kind == "" { - obj.kind = kind // store it + if obj.kind == nil { + obj.kind = &kind // store var err error // Don't send a value right away, wait for the // first Watch startup event to get one! - obj.watchChan, err = obj.init.World.ResWatch(ctx, obj.kind) // watch for var changes + obj.watchChan, err = obj.init.World.ResWatch(ctx, kind) // watch for var changes if err != nil { return err } - - } else if obj.kind != kind { - return fmt.Errorf("can't change kind, previously: `%s`", obj.kind) + continue // we get values on the watch chan, not here! } - continue // we get values on the watch chan, not here! + if *obj.kind == kind { + continue // skip duplicates + } + + // *obj.kind != kind + return fmt.Errorf("can't change kind, previously: `%s`", *obj.kind) case err, ok := <-obj.watchChan: if !ok { // closed @@ -196,27 +179,13 @@ func (obj *ResFunc) Stream(ctx context.Context) error { return nil } if err != nil { - return errwrap.Wrapf(err, "channel watch failed on `%s`", obj.kind) + return errwrap.Wrapf(err, "channel watch failed on `%s`", *obj.kind) } - result, err := obj.Call(ctx, obj.args) // get the value... - if err != nil { + if err := obj.init.Event(ctx); err != nil { // send event return err } - // if the result is still the same, skip sending an update... - if obj.result != nil && result.Cmp(obj.result) == nil { - continue // result didn't change - } - obj.result = result // store new result - - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- obj.result: // send - // pass case <-ctx.Done(): return nil } @@ -238,6 +207,21 @@ func (obj *ResFunc) Call(ctx context.Context, args []types.Value) (types.Value, return nil, fmt.Errorf("invalid resource kind: %s", kind) } + // Check before we send to a chan where we'd need Stream to be running. + if obj.init == nil { + return nil, funcs.ErrCantSpeculate + } + + if obj.init.Debug { + obj.init.Logf("kind: %s", kind) + } + + select { + case obj.input <- kind: + case <-ctx.Done(): + return nil, ctx.Err() + } + filters := []*engine.ResFilter{} filter := &engine.ResFilter{ Kind: kind, @@ -246,9 +230,6 @@ func (obj *ResFunc) Call(ctx context.Context, args []types.Value) (types.Value, } filters = append(filters, filter) - if obj.init == nil { - return nil, funcs.ErrCantSpeculate - } resOutput, err := obj.init.World.ResCollect(ctx, filters) if err != nil { return nil, err diff --git a/lang/core/world/exchange.go b/lang/core/world/exchange.go deleted file mode 100644 index dfa6ab5b..00000000 --- a/lang/core/world/exchange.go +++ /dev/null @@ -1,214 +0,0 @@ -// Mgmt -// Copyright (C) James Shubin and the project contributors -// Written by James Shubin and the project contributors -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// Additional permission under GNU GPL version 3 section 7 -// -// If you modify this program, or any covered work, by linking or combining it -// with embedded mcl code and modules (and that the embedded mcl code and -// modules which link with this program, contain a copy of their source code in -// the authoritative form) containing parts covered by the terms of any other -// license, the licensors of this program grant you additional permission to -// convey the resulting work. Furthermore, the licensors of this program grant -// the original author, James Shubin, additional permission to update this -// additional permission if he deems it necessary to achieve the goals of this -// additional permission. - -package coreworld - -import ( - "context" - "fmt" - - "github.com/purpleidea/mgmt/lang/funcs" - "github.com/purpleidea/mgmt/lang/interfaces" - "github.com/purpleidea/mgmt/lang/types" - "github.com/purpleidea/mgmt/util/errwrap" -) - -const ( - // ExchangeFuncName is the name this function is registered as. - ExchangeFuncName = "exchange" - - // arg names... - exchangeArgNameNamespace = "namespace" - exchangeArgNameValue = "value" -) - -func init() { - funcs.ModuleRegister(ModuleName, ExchangeFuncName, func() interfaces.Func { return &ExchangeFunc{} }) -} - -// ExchangeFunc is special function which returns all the values of a given key -// in the exposed world, and sets it's own. -type ExchangeFunc struct { - init *interfaces.Init - - namespace string - - last types.Value - result types.Value // last calculated output - - watchChan chan error -} - -// String returns a simple name for this function. This is needed so this struct -// can satisfy the pgraph.Vertex interface. -func (obj *ExchangeFunc) String() string { - return ExchangeFuncName -} - -// ArgGen returns the Nth arg name for this function. -func (obj *ExchangeFunc) ArgGen(index int) (string, error) { - seq := []string{exchangeArgNameNamespace, exchangeArgNameValue} - if l := len(seq); index >= l { - return "", fmt.Errorf("index %d exceeds arg length of %d", index, l) - } - return seq[index], nil -} - -// Validate makes sure we've built our struct properly. It is usually unused for -// normal functions that users can use directly. -func (obj *ExchangeFunc) Validate() error { - return nil -} - -// Info returns some static info about itself. -func (obj *ExchangeFunc) Info() *interfaces.Info { - return &interfaces.Info{ - Pure: false, // definitely false - Memo: false, - Fast: false, - Spec: false, - // TODO: do we want to allow this to be statically polymorphic, - // and have value be any type we might want? - // output is map of: hostname => value - Sig: types.NewType(fmt.Sprintf("func(%s str, %s str) map{str: str}", exchangeArgNameNamespace, exchangeArgNameValue)), - Err: obj.Validate(), - } -} - -// Init runs some startup code for this function. -func (obj *ExchangeFunc) Init(init *interfaces.Init) error { - obj.init = init - obj.watchChan = make(chan error) // XXX: sender should close this, but did I implement that part yet??? - return nil -} - -// Stream returns the changing values that this func has over time. -func (obj *ExchangeFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes - ctx, cancel := context.WithCancel(ctx) - defer cancel() - for { - select { - // TODO: should this first chan be run as a priority channel to - // avoid some sort of glitch? is that even possible? can our - // hostname check with reality (below) fix that? - case input, ok := <-obj.init.Input: - if !ok { - obj.init.Input = nil // don't infinite loop back - continue // no more inputs, but don't return! - } - //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { - // return errwrap.Wrapf(err, "wrong function input") - //} - - if obj.last != nil && input.Cmp(obj.last) == nil { - continue // value didn't change, skip it - } - obj.last = input // store for next - - namespace := input.Struct()[exchangeArgNameNamespace].Str() - if namespace == "" { - return fmt.Errorf("can't use an empty namespace") - } - if obj.init.Debug { - obj.init.Logf("namespace: %s", namespace) - } - - // TODO: support changing the namespace over time... - // TODO: possibly removing our stored value there first! - if obj.namespace == "" { - obj.namespace = namespace // store it - var err error - obj.watchChan, err = obj.init.World.StrMapWatch(ctx, obj.namespace) // watch for var changes - if err != nil { - return err - } - - } else if obj.namespace != namespace { - return fmt.Errorf("can't change namespace, previously: `%s`", obj.namespace) - } - - value := input.Struct()[exchangeArgNameValue].Str() - if obj.init.Debug { - obj.init.Logf("value: %+v", value) - } - - if err := obj.init.World.StrMapSet(ctx, obj.namespace, value); err != nil { - return errwrap.Wrapf(err, "namespace write error of `%s` to `%s`", value, obj.namespace) - } - - continue // we get values on the watch chan, not here! - - case err, ok := <-obj.watchChan: - if !ok { // closed - // XXX: if we close, perhaps the engine is - // switching etcd hosts and we should retry? - // maybe instead we should get an "etcd - // reconnect" signal, and the lang will restart? - return nil - } - if err != nil { - return errwrap.Wrapf(err, "channel watch failed on `%s`", obj.namespace) - } - - keyMap, err := obj.init.World.StrMapGet(ctx, obj.namespace) - if err != nil { - return errwrap.Wrapf(err, "channel read failed on `%s`", obj.namespace) - } - - var result types.Value - - d := types.NewMap(obj.Info().Sig.Out) - for k, v := range keyMap { - key := &types.StrValue{V: k} - val := &types.StrValue{V: v} - if err := d.Add(key, val); err != nil { - return errwrap.Wrapf(err, "map could not add key `%s`, val: `%s`", k, v) - } - } - result = d // put map into interface type - - // if the result is still the same, skip sending an update... - if obj.result != nil && result.Cmp(obj.result) == nil { - continue // result didn't change - } - obj.result = result // store new result - - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- obj.result: // send - // pass - case <-ctx.Done(): - return nil - } - } -} diff --git a/lang/core/world/getval.go b/lang/core/world/getval.go index cf144557..a08f7389 100644 --- a/lang/core/world/getval.go +++ b/lang/core/world/getval.go @@ -55,20 +55,15 @@ func init() { funcs.ModuleRegister(ModuleName, GetValFuncName, func() interfaces.Func { return &GetValFunc{} }) } -var _ interfaces.CallableFunc = &GetValFunc{} +var _ interfaces.StreamableFunc = &GetValFunc{} // GetValFunc is special function which returns the value of a given key in the // exposed world. type GetValFunc struct { init *interfaces.Init - key string - args []types.Value - - last types.Value - result types.Value // last calculated output - - watchChan chan error + input chan string // stream of inputs + key *string // the active key } // String returns a simple name for this function. This is needed so this struct @@ -109,66 +104,53 @@ func (obj *GetValFunc) Info() *interfaces.Info { // Init runs some startup code for this function. func (obj *GetValFunc) Init(init *interfaces.Init) error { obj.init = init - obj.watchChan = make(chan error) // XXX: sender should close this, but did I implement that part yet??? + obj.input = make(chan string) return nil } // Stream returns the changing values that this func has over time. func (obj *GetValFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes ctx, cancel := context.WithCancel(ctx) defer cancel() // important so that we cleanup the watch when exiting + + watchChan := make(chan error) // XXX: sender should close this, but did I implement that part yet??? + for { select { // TODO: should this first chan be run as a priority channel to // avoid some sort of glitch? is that even possible? can our // hostname check with reality (below) fix that? - case input, ok := <-obj.init.Input: + case key, ok := <-obj.input: if !ok { - obj.init.Input = nil // don't infinite loop back - continue // no more inputs, but don't return! + obj.input = nil // don't infinite loop back + return fmt.Errorf("unexpected close") } - //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { - // return errwrap.Wrapf(err, "wrong function input") - //} - if obj.last != nil && input.Cmp(obj.last) == nil { - continue // value didn't change, skip it - } - obj.last = input // store for next - - args, err := interfaces.StructToCallableArgs(input) // []types.Value, error) - if err != nil { - return err - } - obj.args = args - - key := args[0].Str() - if key == "" { - return fmt.Errorf("can't use an empty key") - } - if obj.init.Debug { - obj.init.Logf("key: %s", key) + if obj.key != nil && *obj.key == key { + continue // nothing changed } // TODO: support changing the key over time... - if obj.key == "" { - obj.key = key // store it + if obj.key == nil { + obj.key = &key // store var err error // Don't send a value right away, wait for the // first ValueWatch startup event to get one! - obj.watchChan, err = obj.init.World.StrWatch(ctx, obj.key) // watch for var changes + watchChan, err = obj.init.World.StrWatch(ctx, key) // watch for var changes if err != nil { return err } - - } else if obj.key != key { - return fmt.Errorf("can't change key, previously: `%s`", obj.key) + continue // we get values on the watch chan, not here! } - continue // we get values on the watch chan, not here! + if *obj.key == key { + continue // skip duplicates + } - case err, ok := <-obj.watchChan: + // *obj.key != key + return fmt.Errorf("can't change key, previously: `%s`", *obj.key) + + case err, ok := <-watchChan: if !ok { // closed // XXX: if we close, perhaps the engine is // switching etcd hosts and we should retry? @@ -177,27 +159,13 @@ func (obj *GetValFunc) Stream(ctx context.Context) error { return nil } if err != nil { - return errwrap.Wrapf(err, "channel watch failed on `%s`", obj.key) + return errwrap.Wrapf(err, "channel watch failed on `%s`", *obj.key) } - result, err := obj.Call(ctx, obj.args) // get the value... - if err != nil { + if err := obj.init.Event(ctx); err != nil { // send event return err } - // if the result is still the same, skip sending an update... - if obj.result != nil && result.Cmp(obj.result) == nil { - continue // result didn't change - } - obj.result = result // store new result - - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- obj.result: // send - // pass case <-ctx.Done(): return nil } @@ -213,9 +181,25 @@ func (obj *GetValFunc) Call(ctx context.Context, args []types.Value) (types.Valu } key := args[0].Str() exists := true // assume true + if key == "" { + return nil, fmt.Errorf("can't use an empty key") + } + + // Check before we send to a chan where we'd need Stream to be running. if obj.init == nil { return nil, funcs.ErrCantSpeculate } + + if obj.init.Debug { + obj.init.Logf("key: %s", key) + } + + select { + case obj.input <- key: + case <-ctx.Done(): + return nil, ctx.Err() + } + val, err := obj.init.World.StrGet(ctx, key) if err != nil && obj.init.World.StrIsNotExist(err) { exists = false // val doesn't exist diff --git a/lang/core/world/kvlookup.go b/lang/core/world/kvlookup.go index c8c444bc..42de2c8a 100644 --- a/lang/core/world/kvlookup.go +++ b/lang/core/world/kvlookup.go @@ -51,18 +51,17 @@ func init() { funcs.ModuleRegister(ModuleName, KVLookupFuncName, func() interfaces.Func { return &KVLookupFunc{} }) } -var _ interfaces.CallableFunc = &KVLookupFunc{} +var _ interfaces.StreamableFunc = &KVLookupFunc{} // KVLookupFunc is special function which returns all the values of a given key // in the exposed world. It is similar to exchange, but it does not set a key. +// Since exchange has been deprecated, you will want to use this in conjunction +// with a resource to set the desired value. type KVLookupFunc struct { init *interfaces.Init - namespace string - args []types.Value - - last types.Value - result types.Value // last calculated output + input chan string // stream of inputs + namespace *string // the active namespace watchChan chan error } @@ -104,74 +103,43 @@ func (obj *KVLookupFunc) Info() *interfaces.Info { // Init runs some startup code for this function. func (obj *KVLookupFunc) Init(init *interfaces.Init) error { obj.init = init + obj.input = make(chan string) obj.watchChan = make(chan error) // XXX: sender should close this, but did I implement that part yet??? return nil } // Stream returns the changing values that this func has over time. func (obj *KVLookupFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes ctx, cancel := context.WithCancel(ctx) - defer cancel() + defer cancel() // important so that we cleanup the watch when exiting for { select { // TODO: should this first chan be run as a priority channel to // avoid some sort of glitch? is that even possible? can our // hostname check with reality (below) fix that? - case input, ok := <-obj.init.Input: + case namespace, ok := <-obj.input: if !ok { - obj.init.Input = nil // don't infinite loop back - continue // no more inputs, but don't return! - } - //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { - // return errwrap.Wrapf(err, "wrong function input") - //} - - if obj.last != nil && input.Cmp(obj.last) == nil { - continue // value didn't change, skip it - } - obj.last = input // store for next - - args, err := interfaces.StructToCallableArgs(input) // []types.Value, error) - if err != nil { - return err - } - obj.args = args - - namespace := args[0].Str() - if namespace == "" { - return fmt.Errorf("can't use an empty namespace") - } - if obj.init.Debug { - obj.init.Logf("namespace: %s", namespace) + obj.input = nil // don't infinite loop back + return fmt.Errorf("unexpected close") } // TODO: support changing the namespace over time... - // TODO: possibly removing our stored value there first! - if obj.namespace == "" { - obj.namespace = namespace // store it + if obj.namespace == nil { + obj.namespace = &namespace // store it var err error - obj.watchChan, err = obj.init.World.StrMapWatch(ctx, obj.namespace) // watch for var changes + obj.watchChan, err = obj.init.World.StrMapWatch(ctx, namespace) // watch for var changes if err != nil { return err } - - result, err := obj.Call(ctx, obj.args) // build the map... - if err != nil { - return err - } - select { - case obj.init.Output <- result: // send one! - // pass - case <-ctx.Done(): - return nil - } - - } else if obj.namespace != namespace { - return fmt.Errorf("can't change namespace, previously: `%s`", obj.namespace) + continue // we get values on the watch chan, not here! } - continue // we get values on the watch chan, not here! + if *obj.namespace == namespace { + continue // skip duplicates + } + + // *obj.namespace != namespace + return fmt.Errorf("can't change namespace, previously: `%s`", *obj.namespace) case err, ok := <-obj.watchChan: if !ok { // closed @@ -182,27 +150,13 @@ func (obj *KVLookupFunc) Stream(ctx context.Context) error { return nil } if err != nil { - return errwrap.Wrapf(err, "channel watch failed on `%s`", obj.namespace) + return errwrap.Wrapf(err, "channel watch failed on `%s`", *obj.namespace) } - result, err := obj.Call(ctx, obj.args) // build the map... - if err != nil { + if err := obj.init.Event(ctx); err != nil { // send event return err } - // if the result is still the same, skip sending an update... - if obj.result != nil && result.Cmp(obj.result) == nil { - continue // result didn't change - } - obj.result = result // store new result - - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- obj.result: // send - // pass case <-ctx.Done(): return nil } @@ -217,9 +171,25 @@ func (obj *KVLookupFunc) Call(ctx context.Context, args []types.Value) (types.Va return nil, fmt.Errorf("not enough args") } namespace := args[0].Str() + if namespace == "" { + return nil, fmt.Errorf("can't use an empty namespace") + } + + // Check before we send to a chan where we'd need Stream to be running. if obj.init == nil { return nil, funcs.ErrCantSpeculate } + + if obj.init.Debug { + obj.init.Logf("namespace: %s", namespace) + } + + select { + case obj.input <- namespace: + case <-ctx.Done(): + return nil, ctx.Err() + } + keyMap, err := obj.init.World.StrMapGet(ctx, namespace) if err != nil { return nil, errwrap.Wrapf(err, "channel read failed on `%s`", namespace) diff --git a/lang/core/world/schedule.go b/lang/core/world/schedule.go index 452498d0..0a440ca3 100644 --- a/lang/core/world/schedule.go +++ b/lang/core/world/schedule.go @@ -27,24 +27,12 @@ // additional permission if he deems it necessary to achieve the goals of this // additional permission. -// test with: -// time ./mgmt run --hostname h1 --tmp-prefix --no-pgp lang examples/lang/schedule0.mcl -// time ./mgmt run --hostname h2 --seeds=http://127.0.0.1:2379 --client-urls=http://127.0.0.1:2381 --server-urls=http://127.0.0.1:2382 --tmp-prefix --no-pgp lang examples/lang/schedule0.mcl -// time ./mgmt run --hostname h3 --seeds=http://127.0.0.1:2379 --client-urls=http://127.0.0.1:2383 --server-urls=http://127.0.0.1:2384 --tmp-prefix --no-pgp lang examples/lang/schedule0.mcl -// kill h2 (should see h1 and h3 pick [h1, h3] instead) -// restart h2 (should see [h1, h3] as before) -// kill h3 (should see h1 and h2 pick [h1, h2] instead) -// restart h3 (should see [h1, h2] as before) -// kill h3 -// kill h2 -// kill h1... all done! - package coreworld import ( "context" "fmt" - "sort" + "sync" "github.com/purpleidea/mgmt/engine" "github.com/purpleidea/mgmt/etcd/scheduler" // XXX: abstract this if possible @@ -58,48 +46,25 @@ const ( // ScheduleFuncName is the name this function is registered as. ScheduleFuncName = "schedule" - // DefaultStrategy is the strategy to use if none has been specified. - DefaultStrategy = "rr" - - // StrictScheduleOpts specifies whether the opts passed into the - // scheduler must be strictly what we're expecting, and nothing more. - // If this was false, then we'd allow an opts struct that had a field - // that wasn't used by the scheduler. This could be useful if we need to - // migrate to a newer version of the function. It's probably best to - // keep this strict. - StrictScheduleOpts = true - // arg names... scheduleArgNameNamespace = "namespace" - scheduleArgNameOpts = "opts" ) func init() { funcs.ModuleRegister(ModuleName, ScheduleFuncName, func() interfaces.Func { return &ScheduleFunc{} }) } -var _ interfaces.BuildableFunc = &ScheduleFunc{} // ensure it meets this expectation - // ScheduleFunc is special function which determines where code should run in // the cluster. type ScheduleFunc struct { - Type *types.Type // this is the type of opts used if specified - - built bool // was this function built yet? - - init *interfaces.Init - - args []types.Value - + init *interfaces.Init world engine.SchedulerWorld - namespace string - scheduler *scheduler.Result + input chan string // stream of inputs + namespace *string // the active namespace - last types.Value - result types.Value // last calculated output - - watchChan chan *schedulerResult + mutex *sync.Mutex // guards value + value []string // list of hosts } // String returns a simple name for this function. This is needed so this struct @@ -108,19 +73,9 @@ func (obj *ScheduleFunc) String() string { return ScheduleFuncName } -// validOpts returns the available mapping of valid opts fields to types. -func (obj *ScheduleFunc) validOpts() map[string]*types.Type { - return map[string]*types.Type{ - "strategy": types.TypeStr, - "max": types.TypeInt, - "reuse": types.TypeBool, - "ttl": types.TypeInt, - } -} - // ArgGen returns the Nth arg name for this function. func (obj *ScheduleFunc) ArgGen(index int) (string, error) { - seq := []string{scheduleArgNameNamespace, scheduleArgNameOpts} // 2nd arg is optional + seq := []string{scheduleArgNameNamespace} if l := len(seq); index >= l { return "", fmt.Errorf("index %d exceeds arg length of %d", index, l) } @@ -130,163 +85,24 @@ func (obj *ScheduleFunc) ArgGen(index int) (string, error) { // helper func (obj *ScheduleFunc) sig() *types.Type { sig := types.NewType(fmt.Sprintf("func(%s str) []str", scheduleArgNameNamespace)) // simplest form - if obj.Type != nil { - sig = types.NewType(fmt.Sprintf("func(%s str, %s %s) []str", scheduleArgNameNamespace, scheduleArgNameOpts, obj.Type.String())) - } return sig } -// FuncInfer takes partial type and value information from the call site of this -// function so that it can build an appropriate type signature for it. The type -// signature may include unification variables. -func (obj *ScheduleFunc) FuncInfer(partialType *types.Type, partialValues []types.Value) (*types.Type, []*interfaces.UnificationInvariant, error) { - // func(namespace str) []str - // OR - // func(namespace str, opts ?1) []str - - if l := len(partialValues); l < 1 || l > 2 { - return nil, nil, fmt.Errorf("must have at either one or two args") - } - - var typ *types.Type - if len(partialValues) == 1 { - typ = types.NewType(fmt.Sprintf("func(%s str) []str", scheduleArgNameNamespace)) - } - - if len(partialValues) == 2 { - typ = types.NewType(fmt.Sprintf("func(%s str, %s ?1) []str", scheduleArgNameNamespace, scheduleArgNameOpts)) - } - - return typ, []*interfaces.UnificationInvariant{}, nil -} - -// Build is run to turn the polymorphic, undetermined function, into the -// specific statically typed version. It is usually run after Unify completes, -// and must be run before Info() and any of the other Func interface methods are -// used. This function is idempotent, as long as the arg isn't changed between -// runs. -func (obj *ScheduleFunc) Build(typ *types.Type) (*types.Type, error) { - // typ is the KindFunc signature we're trying to build... - if typ.Kind != types.KindFunc { - return nil, fmt.Errorf("input type must be of kind func") - } - - if len(typ.Ord) != 1 && len(typ.Ord) != 2 { - return nil, fmt.Errorf("the schedule function needs either one or two args") - } - if typ.Out == nil { - return nil, fmt.Errorf("return type of function must be specified") - } - if typ.Map == nil { - return nil, fmt.Errorf("invalid input type") - } - - if err := typ.Out.Cmp(types.TypeListStr); err != nil { - return nil, errwrap.Wrapf(err, "return type must be a list of strings") - } - - tNamespace, exists := typ.Map[typ.Ord[0]] - if !exists || tNamespace == nil { - return nil, fmt.Errorf("first arg must be specified") - } - - if len(typ.Ord) == 1 { - obj.Type = nil - obj.built = true - return obj.sig(), nil // done early, 2nd arg is absent! - } - tOpts, exists := typ.Map[typ.Ord[1]] - if !exists || tOpts == nil { - return nil, fmt.Errorf("second argument was missing") - } - - if tOpts.Kind != types.KindStruct { - return nil, fmt.Errorf("second argument must be of kind struct") - } - - validOpts := obj.validOpts() - - if StrictScheduleOpts { - // strict opts field checking! - for _, name := range tOpts.Ord { - t := tOpts.Map[name] - value, exists := validOpts[name] - if !exists { - return nil, fmt.Errorf("unexpected opts field: `%s`", name) - } - - if err := t.Cmp(value); err != nil { - return nil, errwrap.Wrapf(err, "expected different type for opts field: `%s`", name) - } - } - - } else { - // permissive field checking... - validOptsSorted := []string{} - for name := range validOpts { - validOptsSorted = append(validOptsSorted, name) - } - sort.Strings(validOptsSorted) - for _, name := range validOptsSorted { - value := validOpts[name] // type - - t, exists := tOpts.Map[name] - if !exists { - continue // ignore it - } - - // if it exists, check the type - if err := t.Cmp(value); err != nil { - return nil, errwrap.Wrapf(err, "expected different type for opts field: `%s`", name) - } - } - } - - obj.Type = tOpts // type of opts struct, even an empty: `struct{}` - obj.built = true - return obj.sig(), nil -} - -// Copy is implemented so that the type value is not lost if we copy this -// function. -func (obj *ScheduleFunc) Copy() interfaces.Func { - return &ScheduleFunc{ - Type: obj.Type, // don't copy because we use this after unification - built: obj.built, - - init: obj.init, // likely gets overwritten anyways - } -} - // Validate tells us if the input struct takes a valid form. func (obj *ScheduleFunc) Validate() error { - if !obj.built { - return fmt.Errorf("function wasn't built yet") - } - // obj.Type can be nil if no 2nd arg is given, or a struct (even empty!) - if obj.Type != nil && obj.Type.Kind != types.KindStruct { // build must be run first - return fmt.Errorf("type must be nil or a struct") - } return nil } // Info returns some static info about itself. Build must be called before this // will return correct data. func (obj *ScheduleFunc) Info() *interfaces.Info { - // Since this function implements FuncInfer we want sig to return nil to - // avoid an accidental return of unification variables when we should be - // getting them from FuncInfer, and not from here. (During unification!) - var sig *types.Type - if obj.built { - sig = obj.sig() // helper - } return &interfaces.Info{ Pure: false, // definitely false Memo: false, Fast: false, Spec: false, // output is list of hostnames chosen - Sig: sig, // func kind + Sig: obj.sig(), // func kind Err: obj.Validate(), } } @@ -294,155 +110,58 @@ func (obj *ScheduleFunc) Info() *interfaces.Info { // Init runs some startup code for this function. func (obj *ScheduleFunc) Init(init *interfaces.Init) error { obj.init = init - world, ok := obj.init.World.(engine.SchedulerWorld) if !ok { return fmt.Errorf("world backend does not support the SchedulerWorld interface") } obj.world = world - obj.watchChan = make(chan *schedulerResult) + obj.input = make(chan string) + + obj.mutex = &sync.Mutex{} + obj.value = []string{} // empty + //obj.init.Debug = true // use this for local debugging return nil } // Stream returns the changing values that this func has over time. func (obj *ScheduleFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes + ctx, cancel := context.WithCancel(ctx) + defer cancel() // important so that we cleanup the watch when exiting + + watchChan := make(chan *scheduler.ScheduledResult) // XXX: sender should close this, but did I implement that part yet??? + for { select { // TODO: should this first chan be run as a priority channel to // avoid some sort of glitch? is that even possible? can our // hostname check with reality (below) fix that? - case input, ok := <-obj.init.Input: + case namespace, ok := <-obj.input: if !ok { - obj.init.Input = nil // don't infinite loop back - continue // no more inputs, but don't return! - } - //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { - // return errwrap.Wrapf(err, "wrong function input") - //} - - if obj.last != nil && input.Cmp(obj.last) == nil { - continue // value didn't change, skip it - } - obj.last = input // store for next - - args, err := interfaces.StructToCallableArgs(input) // []types.Value, error) - if err != nil { - return err - } - obj.args = args - - namespace := args[0].Str() - - //namespace := input.Struct()[scheduleArgNameNamespace].Str() - if namespace == "" { - return fmt.Errorf("can't use an empty namespace") - } - - opts := make(map[string]types.Value) // empty "struct" - if val, exists := input.Struct()[scheduleArgNameOpts]; exists { - opts = val.Struct() - } - - if obj.init.Debug { - obj.init.Logf("namespace: %s", namespace) - } - - schedulerOpts := []scheduler.Option{} - // don't add bad or zero-value options - - defaultStrategy := true - if val, exists := opts["strategy"]; exists { - if strategy := val.Str(); strategy != "" { - if obj.init.Debug { - obj.init.Logf("opts: strategy: %s", strategy) - } - defaultStrategy = false - schedulerOpts = append(schedulerOpts, scheduler.StrategyKind(strategy)) - } - } - if defaultStrategy { // we always need to add one! - schedulerOpts = append(schedulerOpts, scheduler.StrategyKind(DefaultStrategy)) - } - if val, exists := opts["max"]; exists { - // TODO: check for overflow - if max := int(val.Int()); max > 0 { - if obj.init.Debug { - obj.init.Logf("opts: max: %d", max) - } - schedulerOpts = append(schedulerOpts, scheduler.MaxCount(max)) - } - } - if val, exists := opts["reuse"]; exists { - reuse := val.Bool() - if obj.init.Debug { - obj.init.Logf("opts: reuse: %t", reuse) - } - schedulerOpts = append(schedulerOpts, scheduler.ReuseLease(reuse)) - } - if val, exists := opts["ttl"]; exists { - // TODO: check for overflow - if ttl := int(val.Int()); ttl > 0 { - if obj.init.Debug { - obj.init.Logf("opts: ttl: %d", ttl) - } - schedulerOpts = append(schedulerOpts, scheduler.SessionTTL(ttl)) - } + obj.input = nil // don't infinite loop back + return fmt.Errorf("unexpected close") } // TODO: support changing the namespace over time... - // TODO: possibly removing our stored value there first! - if obj.namespace == "" { - obj.namespace = namespace // store it - - if obj.init.Debug { - obj.init.Logf("starting scheduler...") - } + if obj.namespace == nil { + obj.namespace = &namespace // store it var err error - obj.scheduler, err = obj.world.Scheduler(obj.namespace, schedulerOpts...) + watchChan, err = obj.world.Scheduled(ctx, namespace) // watch for var changes if err != nil { - return errwrap.Wrapf(err, "can't create scheduler") + return err } - - // process the stream of scheduling output... - go func() { - defer close(obj.watchChan) - // XXX: maybe we could share the parent - // ctx, but I have to work out the - // ordering logic first. For now this is - // just a port of what it was before. - newCtx, cancel := context.WithCancel(context.Background()) - go func() { - defer cancel() // unblock Next() - defer obj.scheduler.Shutdown() - select { - case <-ctx.Done(): - return - } - }() - for { - hosts, err := obj.scheduler.Next(newCtx) - select { - case obj.watchChan <- &schedulerResult{ - hosts: hosts, - err: err, - }: - - case <-ctx.Done(): - return - } - } - }() - - } else if obj.namespace != namespace { - return fmt.Errorf("can't change namespace, previously: `%s`", obj.namespace) + continue // we get values on the watch chan, not here! } - continue // we send values on the watch chan, not here! + if *obj.namespace == namespace { + continue // skip duplicates + } - case schedulerResult, ok := <-obj.watchChan: + // *obj.namespace != namespace + return fmt.Errorf("can't change namespace, previously: `%s`", *obj.namespace) + + case scheduledResult, ok := <-watchChan: if !ok { // closed // XXX: maybe etcd reconnected? (fix etcd implementation) @@ -452,52 +171,76 @@ func (obj *ScheduleFunc) Stream(ctx context.Context) error { // reconnect" signal, and the lang will restart? return nil } - if err := schedulerResult.err; err != nil { - if err == scheduler.ErrEndOfResults { - //return nil // TODO: we should probably fix the reconnect issue and use this here - return fmt.Errorf("scheduler shutdown, reconnect bug?") // XXX: fix etcd reconnects - } - return errwrap.Wrapf(err, "channel watch failed on `%s`", obj.namespace) + if scheduledResult == nil { + return fmt.Errorf("unexpected nil result") + } + if err := scheduledResult.Err; err != nil { + return errwrap.Wrapf(err, "scheduler result error") } if obj.init.Debug { - obj.init.Logf("got hosts: %+v", schedulerResult.hosts) + obj.init.Logf("got hosts: %+v", scheduledResult.Hosts) + } + obj.mutex.Lock() + obj.value = scheduledResult.Hosts // store it + obj.mutex.Unlock() + + if err := obj.init.Event(ctx); err != nil { // send event + return err } - var result types.Value - l := types.NewList(obj.Info().Sig.Out) - for _, val := range schedulerResult.hosts { - if err := l.Add(&types.StrValue{V: val}); err != nil { - return errwrap.Wrapf(err, "list could not add val: `%s`", val) - } - } - result = l // set list as result - - if obj.init.Debug { - obj.init.Logf("result: %+v", result) - } - - // if the result is still the same, skip sending an update... - if obj.result != nil && result.Cmp(obj.result) == nil { - continue // result didn't change - } - obj.result = result // store new result - - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- obj.result: // send - // pass case <-ctx.Done(): return nil } } } -// schedulerResult combines our internal events into a single message packet. -type schedulerResult struct { - hosts []string - err error +// Call this function with the input args and return the value if it is possible +// to do so at this time. +func (obj *ScheduleFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) { + if len(args) < 1 { + return nil, fmt.Errorf("not enough args") + } + namespace := args[0].Str() + + if namespace == "" { + return nil, fmt.Errorf("can't use an empty namespace") + } + + // Check before we send to a chan where we'd need Stream to be running. + if obj.init == nil { + return nil, funcs.ErrCantSpeculate + } + + if obj.init.Debug { + obj.init.Logf("namespace: %s", namespace) + } + + // Tell the Stream what we're watching now... This doesn't block because + // Stream should always be ready to consume unless it's closing down... + // If it dies, then a ctx closure should come soon. + select { + case obj.input <- namespace: + case <-ctx.Done(): + return nil, ctx.Err() + } + + obj.mutex.Lock() // TODO: could be a read lock + value := obj.value // initially we might get an empty list + obj.mutex.Unlock() + + var result types.Value + l := types.NewList(obj.Info().Sig.Out) + for _, val := range value { + if err := l.Add(&types.StrValue{V: val}); err != nil { + return nil, errwrap.Wrapf(err, "list could not add val: `%s`", val) + } + } + result = l // set list as result + + if obj.init.Debug { + obj.init.Logf("result: %+v", result) + } + + return result, nil } diff --git a/lang/funcs/dage/dage.go b/lang/funcs/dage/dage.go index e6e89189..df27b128 100644 --- a/lang/funcs/dage/dage.go +++ b/lang/funcs/dage/dage.go @@ -35,7 +35,6 @@ import ( "context" "fmt" "os" - "sort" "strings" "sync" "time" @@ -43,16 +42,55 @@ import ( "github.com/purpleidea/mgmt/engine" "github.com/purpleidea/mgmt/engine/local" "github.com/purpleidea/mgmt/lang/funcs/ref" - "github.com/purpleidea/mgmt/lang/funcs/structs" "github.com/purpleidea/mgmt/lang/funcs/txn" "github.com/purpleidea/mgmt/lang/interfaces" "github.com/purpleidea/mgmt/lang/types" "github.com/purpleidea/mgmt/pgraph" + "github.com/purpleidea/mgmt/util" "github.com/purpleidea/mgmt/util/errwrap" ) // Engine implements a dag engine which lets us "run" a dag of functions, but -// also allows us to modify it while we are running. +// also allows us to modify it while we are running. The functions we support +// can take one of two forms. +// +// 1) A function that supports the normal interfaces.Func API. It has Call() in +// particular. Most functions are done with only this API. +// +// 2) A function which adds Stream() to support the StreamableFunc API. Some +// functions use this along with Event() to notify of a new value. +// +// 3) A third *possible* (but not currently implemented API) would be one that +// has Stream() but takes an Input and Output channel of values instead. This is +// similar to what we previously had. Of note, every input must generate one +// output value. (And more spontaneous output values are allowed as well.) +// +// Of note, functions that support Call() Can also cause an interrupt to happen. +// It's not clear if this (3) option would be allowed to have a Call() method or +// not, and if it would send interrupts on the Output channel. +// +// Of additional note, some functions also require the "ShapelyFunc" API to work +// correctly. Use of this is rare. +// +// XXX: If this engine continuously receives function events at a higher speed +// than it can process, then it will bog down and consume memory infinitely. We +// should consider adding some sort of warning or error if we get to a certain +// size. +// +// XXX: It's likely that this engine could be made even more efficient by more +// cleverly traversing through the graph. Instead of a topological sort, we +// could have some fancy map that determines what's remaining to go through, so +// that when we "interrupt" we don't needlessly repeatedly visit nodes again and +// trigger the "epoch skip" situations. We could also do the incremental +// toposort so that we don't properly re-run the whole algorithm over and over +// if we're always just computing changes. +// +// XXX: We could consider grouping multiple incoming events into a single +// descent into the DAG. It's not clear if this kind of de-duplication would +// break some "glitch-free" aspects or not. It would probably improve +// performance but we'd have to be careful about how we did it. +// +// XXX: Respect the info().Pure and info().Memo fields somewhere... type Engine struct { // Name is the name used for the instance of the engine and in the graph // that is held within it. @@ -65,93 +103,44 @@ type Engine struct { Debug bool Logf func(format string, v ...interface{}) - // Callback can be specified as an alternative to using the Stream - // method to get events. If the context on it is cancelled, then it must - // shutdown quickly, because this means we are closing and want to - // disconnect. Whether you want to respect that is up to you, but the - // engine will not be able to close until you do. If specified, and an - // error has occurred, it will set that error property. - Callback func(context.Context, error) - - graph *pgraph.Graph // guarded by graphMutex - table map[interfaces.Func]types.Value // guarded by tableMutex - state map[interfaces.Func]*state - - // graphMutex wraps access to the table map. - graphMutex *sync.Mutex // TODO: &sync.RWMutex{} ? - - // tableMutex wraps access to the table map. - tableMutex *sync.RWMutex + // graph is the internal graph. It is only changed during interrupt. + graph *pgraph.Graph // refCount keeps track of vertex and edge references across the entire // graph. refCount *ref.Count - // wgTxn blocks shutdown until the initial Txn has Reversed. - wgTxn *sync.WaitGroup - - // firstTxn checks to make sure wgTxn is only used for the first Txn. - firstTxn bool + // state stores some per-vertex (function) state + state map[interfaces.Func]*state + // wg counts every concurrent process here. wg *sync.WaitGroup - // pause/resume state machine signals - pauseChan chan struct{} - pausedChan chan struct{} - resumeChan chan struct{} - resumedChan chan struct{} + // ag is the aggregation channel, which receives events from any of the + // StreamableFunc's that are running. + // XXX: add a mechanism to detect if it gets too full + ag *util.InfiniteChan[*state] + //ag chan *state - // resend tracks which new nodes might need a new notification - resend map[interfaces.Func]struct{} + // cancel can be called to shutdown Run() after it's started of course. + cancel func() - // nodeWaitFns is a list of cleanup functions to run after we've begun - // resume, but before we've resumed completely. These are actions that - // we would like to do when paused from a deleteVertex operation, but - // that would deadlock if we did. - nodeWaitFns []func() + // streamChan is used to send the stream of tables to the outside world. + streamChan chan interfaces.Table - // nodeWaitMutex wraps access to the nodeWaitFns list. - nodeWaitMutex *sync.Mutex + // interrupt specifies that a txn "commit" just happened. + interrupt bool - // streamChan is used to send notifications to the outside world. - streamChan chan error + // topoSort is the last topological sort we ran. + topoSort []pgraph.Vertex - loaded bool // are all of the funcs loaded? - loadedChan chan struct{} // funcs loaded signal + // ops is a list of operations to run during interrupt. This is usually + // a delete vertex, but others are possible. + ops []ops - startedChan chan struct{} // closes when Run() starts - - // wakeChan contains a message when someone has asked for us to wake up. - wakeChan chan struct{} - - // ag is the aggregation channel which cues up outgoing events. - ag chan error - - // leafSend specifies if we should do an ag send because we have - // activity at a leaf. - leafSend bool - - // isClosed tracks nodes that have closed. This list is purged as they - // are removed from the graph. - isClosed map[*state]struct{} - - // activity tracks nodes that are ready to send to ag. The main process - // loop decides if we have the correct set to do so. A corresponding - // value of true means we have regular activity, and a value of false - // means the node closed. - activity map[*state]struct{} - - // stateMutex wraps access to the isClosed and activity maps. - stateMutex *sync.Mutex - - // stats holds some statistics and other debugging information. - stats *stats // guarded by statsMutex - - // statsMutex wraps access to the stats data. - statsMutex *sync.RWMutex - - // graphvizMutex wraps access to the Graphviz method. - graphvizMutex *sync.Mutex + // err contains the last error after a shutdown occurs. + err error + errMutex *sync.Mutex // guards err // graphvizCount keeps a running tally of how many graphs we've // generated. This is useful for displaying a sequence (timeline) of @@ -163,92 +152,530 @@ type Engine struct { graphvizDirectory string } -// Setup sets up the internal datastructures needed for this engine. +// Setup sets up the internal datastructures needed for this engine. We use this +// earlier step before Run() because it's usually not called concurrently, which +// makes it easier to catch the obvious errors before Run() runs in a goroutine. func (obj *Engine) Setup() error { var err error obj.graph, err = pgraph.NewGraph(obj.Name) if err != nil { return err } - obj.table = make(map[interfaces.Func]types.Value) obj.state = make(map[interfaces.Func]*state) - obj.graphMutex = &sync.Mutex{} // TODO: &sync.RWMutex{} ? - obj.tableMutex = &sync.RWMutex{} obj.refCount = (&ref.Count{}).Init() - obj.wgTxn = &sync.WaitGroup{} - obj.wg = &sync.WaitGroup{} + obj.errMutex = &sync.Mutex{} - obj.pauseChan = make(chan struct{}) - obj.pausedChan = make(chan struct{}) - obj.resumeChan = make(chan struct{}) - obj.resumedChan = make(chan struct{}) + //obj.ag = make(chan *state, 1) // for group events + //obj.ag = make(chan *state) // normal no buffer, we can't drop any + obj.ag = util.NewInfiniteChan[*state]() // lock-free but unbounded - obj.resend = make(map[interfaces.Func]struct{}) + obj.streamChan = make(chan interfaces.Table) - obj.nodeWaitFns = []func(){} + obj.ops = []ops{} - obj.nodeWaitMutex = &sync.Mutex{} - - obj.streamChan = make(chan error) - obj.loadedChan = make(chan struct{}) - obj.startedChan = make(chan struct{}) - - obj.wakeChan = make(chan struct{}, 1) // hold up to one message - - obj.ag = make(chan error) - - obj.isClosed = make(map[*state]struct{}) - - obj.activity = make(map[*state]struct{}) - obj.stateMutex = &sync.Mutex{} - - obj.stats = &stats{ - runningList: make(map[*state]struct{}), - loadedList: make(map[*state]bool), - inputList: make(map[*state]int64), - } - obj.statsMutex = &sync.RWMutex{} - - obj.graphvizMutex = &sync.Mutex{} return nil } -// Cleanup cleans up and frees memory and resources after everything is done. -func (obj *Engine) Cleanup() error { - obj.wg.Wait() // don't cleanup these before Run() finished - close(obj.pauseChan) // free - close(obj.pausedChan) - close(obj.resumeChan) - close(obj.resumedChan) +// Run kicks off the function engine. You must add the initial graph via Txn +// *before* you run this function. +// XXX: try and fix the engine so you can run either Txn or Run first. +func (obj *Engine) Run(ctx context.Context) error { + if obj.refCount == nil { // any arbitrary flag would be fine here + return fmt.Errorf("you must run Setup before first use") + } + + //obj.wg = &sync.WaitGroup{} // in Setup + defer obj.wg.Wait() + + // cancel to allow someone to shut everything down... + ctx, cancel := context.WithCancel(ctx) + defer cancel() + obj.cancel = cancel + + //obj.streamChan = make(chan Table) // in Setup + defer close(obj.streamChan) + + err := obj.process(ctx, 1) // start like this for now + obj.errAppend(err) + return err +} + +// process could be combined with Run, but it is left separate in case we try to +// build a recursive process operation that runs on a subgraph. It would need an +// incoming graph argument as well, I would expect. +func (obj *Engine) process(ctx context.Context, epoch int64) error { + + mapping := make(map[pgraph.Vertex]int) + start := 0 + table := make(interfaces.Table) // map[interfaces.Func]types.Value + +Start: + for { + // If it's our first time, we want to interrupt, because we may + // not ever get any events otherwise, and we'd block at select + // waiting on obj.ag forever. Remember that the effect() of Txn + // causes an interrupt when we add the first graph in. This + // means we need to do the initial Txn before we startup here! + if obj.interrupt { + if obj.Debug { + obj.Logf("interrupt!") + } + + // Handle delete (and other ops) first. We keep checking + // until this is empty, because a Cleanup operation + // running inside this loop could cause more vertices + // to be added, and so on. + for len(obj.ops) > 0 { + op := obj.ops[0] // run in same order added + obj.ops = obj.ops[1:] // queue + + // adds are new vertices which join the graph + if add, ok := op.(*addVertex); ok { + table[add.f] = nil // for symmetry + + if err := add.fn(ctx); err != nil { // Init! + return err + } + + continue + } + + // deletes are the list of Func's (vertices) + // that were deleted in a txn. + if del, ok := op.(*deleteVertex); ok { + delete(table, del.f) // cleanup the table + + if err := del.fn(ctx); err != nil { // Cleanup! + return err + } + + continue + } + } + + // Interrupt should only happen if we changed the graph + // shape, so recompute the topological sort right here. + + // XXX: Can we efficiently edit the old topoSort by + // knowing the add/del? If the graph is shrinking, just + // remove those vertices from our current toposort. If + // the graph is growing, can we topo sort the subset and + // put them at the beginning of our old toposort? Is it + // guaranteed that "spawned nodes" will have earlier + // precedence than existing stuff? Can we be clever? Can + // we "float" anything upwards that's needed for the + // "toposort" by seeing what we're connected to, and + // sort all of that? + var err error + obj.topoSort, err = obj.graph.TopologicalSort() + if err != nil { + // programming error + return err + } + + // This interrupt must be set *after* the above deletes + // happen, because those can cause transactions to run, + // and those transactions run obj.effect() which resets + // this interrupt value back to true! + obj.interrupt = false // reset + start = 0 // restart the loop + for i, v := range obj.topoSort { // TODO: Do it once here, or repeatedly below? + mapping[v] = i + } + + goto PreIterate // skip waiting for a new event + } + + if n := obj.graph.NumVertices(); n == 0 { + // If we're here, then the engine is done, because we'd + // block forever. + return nil + } + if obj.Debug { + obj.Logf("waiting for event...") + } + select { + case node, ok := <-obj.ag.Out: + if obj.Debug { + obj.Logf("got event: %v", node) + } + if !ok { + // TODO: If we don't have events, maybe shutdown? + panic("unexpected event channel shutdown") + } + // i is 0 if missing + i, _ := mapping[node.Func] // get the node to start from... + start = i + // XXX: Should we ACK() here so that Stream can "make" + // the new value available to it's Call() starting now? + + case <-ctx.Done(): + return ctx.Err() + } + + // We have at least one event now! + + PreIterate: + valid := true // assume table is valid for this iteration + + Iterate: + // Iterate down through the graph... + for i := start; i < len(obj.topoSort); i++ { // formerly: for _, v := range obj.topoSort + start = i // set for subsequent runs + v := obj.topoSort[i] + f, ok := v.(interfaces.Func) + if !ok { + panic("not a Func") + } + if obj.Debug { + obj.Logf("topo(%d): %p %+v", i, f, f) + } + mapping[v] = i // store for subsequent loops + + node, exists := obj.state[f] + if !exists { + panic(fmt.Sprintf("node state missing: %s", f)) + } + + streamableFunc, isStreamable := f.(interfaces.StreamableFunc) + if isStreamable && !node.started { // don't start twice + obj.wg.Add(1) + go func() { + defer obj.wg.Done() + // XXX: I think the design should be that + // if this ever shuts down, then the + // function engine should shut down, but + // that the individual Call() can error... + // This is inline with our os.Readfilewait + // function which models the logic we want... + // If the call errors AND we have the Except + // feature, then we want that Except to run, + // but if we get a new event, then we should + // try again. basically revive itself after + // an errored Call function. Of course if + // Stream shuts down, we're nuked, so maybe + // we might want to retry... So maybe a resource + // could tweak the retry params for such a + // function??? Or maybe a #pragma kind of + // directive thing above each function??? + err := streamableFunc.Stream(ctx) + if err == nil { + return + } + obj.errAppend(err) + obj.cancel() // error + }() + node.started = true + } + + if node.epoch >= epoch { // we already did this one + if obj.Debug { + obj.Logf("epoch skip: %p %v", f, f) + } + continue + } + + // XXX: memoize until graph shape changes? + incoming := obj.graph.IncomingGraphVertices(f) // []pgraph.Vertex + + // Not all of the incoming edges have been added yet. + // We start by doing the "easy" count, and if it fails, + // we fall back on the slightly more expensive, and + // accurate count. This is because logical edges can be + // combined into a single physical edge. This happens if + // we have the same arg (a, b) passed to the same func. + if n := len(node.Func.Info().Sig.Ord); n != len(incoming) && n != realEdgeCount(obj.graph.IncomingGraphEdges(f)) { + if obj.Debug { + obj.Logf("edge skip: %p %v", f, f) + } + + valid = false + // If we skip here, we also want to skip any of + // the vertices that depend on this one. This is + // because the toposort might offer our children + // before a non-dependent node which might be + // the node that causes the interrupt which adds + // the edge which is currently not added yet. + continue + } + + // if no incoming edges, no incoming data, so this noop's + + si := &types.Type{ + // input to functions are structs + Kind: types.KindStruct, + Map: node.Func.Info().Sig.Map, + Ord: node.Func.Info().Sig.Ord, + } + st := types.NewStruct(si) + // The above builds a struct with fields + // populated for each key (empty values) + // so we need to very carefully check if + // every field is received before we can + // safely send it downstream to an edge. + need := make(map[string]struct{}) // keys we need + for _, k := range node.Func.Info().Sig.Ord { + need[k] = struct{}{} + } + + for _, vv := range incoming { + ff, ok := vv.(interfaces.Func) + if !ok { + panic("not a Func") + } + + // XXX: do we need a lock around reading obj.state? + fromNode, exists := obj.state[ff] + if !exists { + panic(fmt.Sprintf("missing node state: %s", ff)) + } + + // Node we pull from should be newer epoch than us! + if node.epoch >= fromNode.epoch { + if obj.Debug { + obj.Logf("inner epoch skip: %p %v", f, f) + //obj.Logf("inner epoch skip: NODE(%p is %d): %v FROM(%p is %d) %v", f, node.epoch, f, ff, fromNode.epoch, ff) + } + // Don't set non-valid here because if + // we have *two* FuncValue's that both + // interrupt, the first one will happen, + // and then the reset of the graph can + // be updated to the current epoch, but + // when the full graph is ready here, we + // would skip because of this bool! + //valid = false // don't do this! + continue Iterate + } + + value := fromNode.result + if value == nil { + //if valid { // must be a programming err! + panic(fmt.Sprintf("unexpected nil node result from: %s", ff)) + //} + // We're reading from a node which got + // skipped because it didn't have all of + // its edges yet. (or a programming bug) + //continue Iterate + // The fromNode epoch check above should + // make this additional check redundant. + } + + // set each arg, since one value + // could get used for multiple + // function inputs (shared edge) + // XXX: refactor this edge look up for efficiency since we just did IncomingGraphVertices? + edge := obj.graph.Adjacency()[ff][f] + if edge == nil { + panic(fmt.Sprintf("edge is nil from `%s` to `%s`", ff, f)) + } + args := edge.(*interfaces.FuncEdge).Args + for _, arg := range args { + // Skip edge is unused at this time. + //if arg == "" { // XXX: special skip edge! + // // XXX: we could maybe detect this at the incoming loop above instead + // continue + //} + // populate struct + if err := st.Set(arg, value); err != nil { + //panic(fmt.Sprintf("struct set failure on `%s` from `%s`: %v", node, fromNode, err)) + keys := []string{} + for k := range st.Struct() { + keys = append(keys, k) + } + panic(fmt.Sprintf("struct set failure on `%s` from `%s`: %v, has: %v", node, fromNode, err, keys)) + } + if _, exists := need[arg]; !exists { + keys := []string{} + for k := range st.Struct() { + keys = append(keys, k) + } + // could be either a duplicate or an unwanted field (edge name) + panic(fmt.Sprintf("unexpected struct key `%s` on `%s` from `%s`, has(%d): %v", arg, node, fromNode, len(keys), keys)) + } + delete(need, arg) + } + } + // We just looped through all the incoming edges. + + // XXX: Can we do the above bits -> struct, and then the + // struct -> list here, all in one faster step for perf? + args, err := interfaces.StructToCallableArgs(st) // []types.Value, error) + if err != nil { + panic(fmt.Sprintf("struct to callable failure on `%s`: %v, has: %v", node, err, st)) + } + + // Call the function. + if obj.Debug { + obj.Logf("call: %v", f) + } + //node.result, err = f.Call(ctx, args) + node.result, err = obj.call(f, ctx, args) // recovers! + // XXX: On error lookup the fallback value if it exists. + // XXX: This might cause an interrupt + graph addition. + if err == interfaces.ErrInterrupt { + // re-run topological sort... at the top! + + obj.interrupt = true // should be set in obj.effect + continue Start + } + if obj.interrupt { + // We have a function which caused an interrupt, + // but which didn't return ErrInterrupt. This is + // a programming error by the function. + return fmt.Errorf("function didn't interrupt correctly: %s", node) + } + if err != nil { + return err + } + if node.result == nil && len(obj.graph.OutgoingGraphVertices(f)) > 0 { + // XXX: this check may not work if we have our + // "empty" named edges added on here... + return fmt.Errorf("unexpected nil value from node: %s", node) + } + old := node.epoch + node.epoch = epoch // store it after a successful call + if obj.Debug { + obj.Logf("set epoch(%d) to %d: %p %v", old, epoch, f, f) + } + + // XXX: Should we check here to see if we can shutdown? + // For a given node, if Stream is not running, and no + // incoming nodes are still open, and if we're Pure, and + // we can memoize, then why not shutdown this node and + // remove it from the graph? Run a graph interrupt to + // delete this vertex. This will run Cleanup. Is it safe + // to also delete the table entry? Is it needed or used? + + if node.result == nil { + // got an end of line vertex that would normally + // send a dummy value... don't store in table... + continue + } + table[f] = node.result // build up our table of values + + } // end of single graph traversal + + if !valid { // don't send table yet, it's not complete + continue + } + + // Send a table of the complete set of values, which should all + // have the same epoch, and send it as an event to the outside. + // We need a copy of the map since we'll keep modifying it now. + // The table must get cleaned up over time to be consistent. It + // currently happens in interrupt as a result of a node delete. + + cp := table.Copy() + if obj.Debug { + obj.Logf("table:") + for k, v := range cp { + obj.Logf("table[%p %v]: %p %+v", k, k, v, v) + } + } + select { + case obj.streamChan <- cp: + + case <-ctx.Done(): + return ctx.Err() + } + + // XXX: implement epoch rollover by relabelling all nodes + epoch++ // increment it after a successful traversal + if obj.Debug { + obj.Logf("epoch(%d) increment to %d", epoch-1, epoch) + } + + } // end big for loop +} + +// event is ultimately called from a function to trigger an event in the engine. +// We'd like for this to never block, because that makes it much easier to +// prevent deadlocks in some tricky functions. On the other side, we don't want +// to necessarily merge events if we want to ensure each sent event gets seen in +// order. A buffered channel would accomplish this, but then it would need a +// fixed size, and if it reached the capacity we'd be in the deadlock situation +// again. Instead, we use a buffered channel of size one, and a queue of data +// which stores the event information. +func (obj *Engine) event(ctx context.Context, state *state) error { + //f := state.Func // for reference, how to get the Vertex/Func pointer! + + select { + case obj.ag.In <- state: // buffered to avoid blocking issues + // tell function engine who had an event... deal with it before + // we get to handle subsequent ones... + case <-ctx.Done(): + return ctx.Err() + } + return nil } +// effect runs at the end of the transaction, but before it returns. +// XXX: we don't need delta ops if we can just plug into our implementations of +// the addVertex and deleteVertex ... +func (obj *Engine) effect( /*delta *DeltaOps*/ ) error { + obj.interrupt = true + + // The toposort runs in interrupt. We could save `delta` if it's needed. + //var err error + //obj.topoSort, err = obj.graph.TopologicalSort() + //return err + return nil +} + +// call is a helper to handle the recovering if needed from a function call. +func (obj *Engine) call(f interfaces.Func, ctx context.Context, args []types.Value) (result types.Value, reterr error) { + defer func() { + // catch programming errors + if r := recover(); r != nil { + obj.Logf("panic in process: %+v", r) + reterr = fmt.Errorf("panic in process: %+v", r) + } + }() + + return f.Call(ctx, args) +} + +// Stream returns a channel that you can follow to get aggregated graph events. +// Do not block reading from this channel as you can hold up the entire engine. +func (obj *Engine) Stream() <-chan interfaces.Table { + return obj.streamChan +} + +// Err will contain the last error when Stream shuts down. It waits for all the +// running processes to exit before it returns. +func (obj *Engine) Err() error { + obj.wg.Wait() + return obj.err +} + // Txn returns a transaction that is suitable for adding and removing from the // graph. You must run Setup before this method is called. func (obj *Engine) Txn() interfaces.Txn { if obj.refCount == nil { - panic("you must run setup before first use") + panic("you must run Setup before first use") } // The very first initial Txn must have a wait group to make sure if we // shutdown (in error) that we can Reverse things before the Lock/Unlock // loop shutsdown. - var free func() - if !obj.firstTxn { - obj.firstTxn = true - obj.wgTxn.Add(1) - free = func() { - obj.wgTxn.Done() - } - } + //var free func() + //if !obj.firstTxn { + // obj.firstTxn = true + // obj.wgTxn.Add(1) + // free = func() { + // obj.wgTxn.Done() + // } + //} return (&txn.GraphTxn{ - Lock: obj.Lock, - Unlock: obj.Unlock, + Post: obj.effect, + Lock: func() {}, // noop for now + Unlock: func() {}, // noop for now GraphAPI: obj, RefCount: obj.refCount, // reference counting - FreeFunc: free, + //FreeFunc: free, }).Init() } @@ -278,8 +705,6 @@ func (obj *Engine) addVertex(f interfaces.Func) error { return errwrap.Wrapf(err, "did not Validate node: %s", f) } - input := make(chan types.Value) - output := make(chan types.Value) txn := obj.Txn() // This is the one of two places where we modify this map. To avoid @@ -290,24 +715,21 @@ func (obj *Engine) addVertex(f interfaces.Func) error { Func: f, name: f.String(), // cache a name to avoid locks - input: input, - output: output, - txn: txn, + txn: txn, - running: false, - wg: &sync.WaitGroup{}, - - rwmutex: &sync.RWMutex{}, + //running: false, + //epoch: 0, } init := &interfaces.Init{ Hostname: obj.Hostname, - Input: node.input, - Output: node.output, - Txn: node.txn, - Local: obj.Local, - World: obj.World, - Debug: obj.Debug, + Event: func(ctx context.Context) error { + return obj.event(ctx, node) // pass state to avoid search + }, + Txn: node.txn, + Local: obj.Local, + World: obj.World, + Debug: obj.Debug, Logf: func(format string, v ...interface{}) { // safe Logf in case f.String contains %? chars... s := f.String() + ": " + fmt.Sprintf(format, v...) @@ -315,20 +737,25 @@ func (obj *Engine) addVertex(f interfaces.Func) error { }, } - if err := f.Init(init); err != nil { - return err + op := &addVertex{ + f: f, + fn: func(ctx context.Context) error { + return f.Init(init) // TODO: should this take a ctx? + }, } - // only now, do we modify the graph - obj.state[f] = node - obj.graph.AddVertex(f) + obj.ops = append(obj.ops, op) // mark for cleanup during interrupt + + obj.state[f] = node // do this here b/c we rely on knowing it in real-time + + obj.graph.AddVertex(f) // Txn relies on this happening now while it runs. return nil } // AddVertex is the thread-safe way to add a vertex. You will need to call the // engine Lock method before using this and the Unlock method afterwards. func (obj *Engine) AddVertex(f interfaces.Func) error { - obj.graphMutex.Lock() - defer obj.graphMutex.Unlock() + // No mutex needed here since this func runs in a non-concurrent Txn. + if obj.Debug { obj.Logf("Engine:AddVertex: %p %s", f, f) } @@ -342,24 +769,26 @@ func (obj *Engine) AddVertex(f interfaces.Func) error { // already part of the graph. You should only create DAG's as this function // engine cannot handle cycles and this method will error if you cause a cycle. func (obj *Engine) AddEdge(f1, f2 interfaces.Func, fe *interfaces.FuncEdge) error { - obj.graphMutex.Lock() - defer obj.graphMutex.Unlock() + // No mutex needed here since this func runs in a non-concurrent Txn. + if obj.Debug { obj.Logf("Engine:AddEdge %p %s: %p %s -> %p %s", fe, fe, f1, f1, f2, f2) } - // safety check to avoid cycles - g := obj.graph.Copy() - //g.AddVertex(f1) - //g.AddVertex(f2) - g.AddEdge(f1, f2, fe) - if _, err := g.TopologicalSort(); err != nil { - return err // not a dag + if obj.Debug { // not needed unless we have buggy graph building code + // safety check to avoid cycles + g := obj.graph.Copy() + //g.AddVertex(f1) + //g.AddVertex(f2) + g.AddEdge(f1, f2, fe) + if _, err := g.TopologicalSort(); err != nil { + return err // not a dag + } + // if we didn't cycle, we can modify the real graph safely... } - // if we didn't cycle, we can modify the real graph safely... // Does the graph already have these nodes in it? - hasf1 := obj.graph.HasVertex(f1) + //hasf1 := obj.graph.HasVertex(f1) //hasf2 := obj.graph.HasVertex(f2) if err := obj.addVertex(f1); err != nil { // lockless version @@ -376,16 +805,19 @@ func (obj *Engine) AddEdge(f1, f2 interfaces.Func, fe *interfaces.FuncEdge) erro // But there's no guarantee we didn't AddVertex(f2); AddEdge(f1, f2, e), // so resend if f1 already exists. Otherwise it's not a new notification. // previously: `if hasf1 && !hasf2` - if hasf1 { - obj.resend[f2] = struct{}{} // resend notification to me - } + //if hasf1 { + // //obj.resend[f2] = struct{}{} // resend notification to me + //} obj.graph.AddEdge(f1, f2, fe) // replaces any existing edge here - // This shouldn't error, since the test graph didn't find a cycle. - if _, err := obj.graph.TopologicalSort(); err != nil { - // programming error - panic(err) // not a dag + // This shouldn't error, since the test graph didn't find a cycle. But + // we don't really need to do it, since the interrupt will run it too. + if obj.Debug { // not needed unless we have buggy graph building code + if _, err := obj.graph.TopologicalSort(); err != nil { + // programming error + panic(err) // not a dag + } } return nil @@ -399,40 +831,37 @@ func (obj *Engine) deleteVertex(f interfaces.Func) error { if !exists { return fmt.Errorf("vertex %p %s doesn't exist", f, f) } - - if node.running { - // cancel the running vertex - node.cancel() // cancel inner ctx - - // We store this work to be performed later on in the main loop - // because this Wait() might be blocked by a defer Commit, which - // is itself blocked because this deleteVertex operation is part - // of a Commit. - obj.nodeWaitMutex.Lock() - obj.nodeWaitFns = append(obj.nodeWaitFns, func() { - node.wg.Wait() // While waiting, the Stream might cause a new Reverse Commit - node.txn.Free() // Clean up when done! - obj.stateMutex.Lock() - delete(obj.isClosed, node) // avoid memory leak - obj.stateMutex.Unlock() - }) - obj.nodeWaitMutex.Unlock() - } + _ = node // This is the one of two places where we modify this map. To avoid // concurrent writes, we only do this when we're locked! Anywhere that // can read where we are locked must have a mutex around it or do the // lookup when we're in an unlocked state. - delete(obj.state, f) - obj.graph.DeleteVertex(f) + + op := &deleteVertex{ + f: f, + fn: func(ctx context.Context) error { + // XXX: do we run f.Done() first ? Did it run elsewhere? + cleanableFunc, ok := f.(interfaces.CleanableFunc) + if !ok { + return nil + } + return cleanableFunc.Cleanup(ctx) + }, + } + obj.ops = append(obj.ops, op) // mark for cleanup during interrupt + + delete(obj.state, f) // do this here b/c we rely on knowing it in real-time + + obj.graph.DeleteVertex(f) // Txn relies on this happening now while it runs. return nil } // DeleteVertex is the thread-safe way to delete a vertex. You will need to call // the engine Lock method before using this and the Unlock method afterwards. func (obj *Engine) DeleteVertex(f interfaces.Func) error { - obj.graphMutex.Lock() - defer obj.graphMutex.Unlock() + // No mutex needed here since this func runs in a non-concurrent Txn. + if obj.Debug { obj.Logf("Engine:DeleteVertex: %p %s", f, f) } @@ -443,8 +872,8 @@ func (obj *Engine) DeleteVertex(f interfaces.Func) error { // DeleteEdge is the thread-safe way to delete an edge. You will need to call // the engine Lock method before using this and the Unlock method afterwards. func (obj *Engine) DeleteEdge(fe *interfaces.FuncEdge) error { - obj.graphMutex.Lock() - defer obj.graphMutex.Unlock() + // No mutex needed here since this func runs in a non-concurrent Txn. + if obj.Debug { f1, f2, found := obj.graph.LookupEdge(fe) if found { @@ -466,8 +895,7 @@ func (obj *Engine) DeleteEdge(fe *interfaces.FuncEdge) error { // You will need to call the engine Lock method before using this and the Unlock // method afterwards. func (obj *Engine) HasVertex(f interfaces.Func) bool { - obj.graphMutex.Lock() // XXX: should this be a RLock? - defer obj.graphMutex.Unlock() // XXX: should this be an RUnlock? + // No mutex needed here since this func runs in a non-concurrent Txn. return obj.graph.HasVertex(f) } @@ -476,8 +904,7 @@ func (obj *Engine) HasVertex(f interfaces.Func) bool { // between an edge in the graph. You will need to call the engine Lock method // before using this and the Unlock method afterwards. func (obj *Engine) LookupEdge(fe *interfaces.FuncEdge) (interfaces.Func, interfaces.Func, bool) { - obj.graphMutex.Lock() // XXX: should this be a RLock? - defer obj.graphMutex.Unlock() // XXX: should this be an RUnlock? + // No mutex needed here since this func runs in a non-concurrent Txn. v1, v2, found := obj.graph.LookupEdge(fe) if !found { @@ -503,8 +930,7 @@ func (obj *Engine) LookupEdge(fe *interfaces.FuncEdge) (interfaces.Func, interfa // DeleteEdge to remove it. You will need to call the engine Lock method before // using this and the Unlock method afterwards. func (obj *Engine) FindEdge(f1, f2 interfaces.Func) *interfaces.FuncEdge { - obj.graphMutex.Lock() // XXX: should this be a RLock? - defer obj.graphMutex.Unlock() // XXX: should this be an RUnlock? + // No mutex needed here since this func runs in a non-concurrent Txn. edge := obj.graph.FindEdge(f1, f2) if edge == nil { @@ -520,972 +946,21 @@ func (obj *Engine) FindEdge(f1, f2 interfaces.Func) *interfaces.FuncEdge { // Graph returns a copy of the contained graph. func (obj *Engine) Graph() *pgraph.Graph { - //obj.graphMutex.Lock() // XXX: should this be a RLock? - //defer obj.graphMutex.Unlock() // XXX: should this be an RUnlock? + // No mutex needed here since this func runs in a non-concurrent Txn. return obj.graph.Copy() } -// Lock must be used before modifying the running graph. Make sure to Unlock -// when done. -// XXX: should Lock take a context if we want to bail mid-way? -// TODO: could we replace pauseChan with SubscribedSignal ? -func (obj *Engine) Lock() { // pause - select { - case obj.pauseChan <- struct{}{}: - } - //obj.rwmutex.Lock() // TODO: or should it go right before pauseChan? - - // waiting for the pause to move to paused... - select { - case <-obj.pausedChan: - } - // this mutex locks at start of Run() and unlocks at finish of Run() - obj.graphMutex.Unlock() // safe to make changes now -} - -// Unlock must be used after modifying the running graph. Make sure to Lock -// beforehand. -// XXX: should Unlock take a context if we want to bail mid-way? -func (obj *Engine) Unlock() { // resume - // this mutex locks at start of Run() and unlocks at finish of Run() - obj.graphMutex.Lock() // no more changes are allowed - select { - case obj.resumeChan <- struct{}{}: - } - //obj.rwmutex.Unlock() // TODO: or should it go right after resumedChan? - - // waiting for the resume to move to resumed... - select { - case <-obj.resumedChan: - } -} - -// wake sends a message to the wake queue to wake up the main process function -// which would otherwise spin unnecessarily. This can be called anytime, and -// doesn't hurt, it only wastes cpu if there's nothing to do. This does NOT ever -// block, and that's important so it can be called from anywhere. -func (obj *Engine) wake(name string) { - // The mutex guards the len check to avoid this function sending two - // messages down the channel, because the second would block if the - // consumer isn't fast enough. This mutex makes this method effectively - // asynchronous. - //obj.wakeMutex.Lock() - //defer obj.wakeMutex.Unlock() - //if len(obj.wakeChan) > 0 { // collapse duplicate, pending wake signals - // return - //} - select { - case obj.wakeChan <- struct{}{}: // send to chan of length 1 - if obj.Debug { - obj.Logf("wake sent from: %s", name) - } - default: // this is a cheap alternative to avoid the mutex altogether! - if obj.Debug { - obj.Logf("wake skip from: %s", name) - } - // skip sending, we already have a message pending! - } -} - -// runNodeWaitFns is a helper to run the cleanup nodeWaitFns list. It clears the -// list after it runs. -func (obj *Engine) runNodeWaitFns() { - // The lock is probably not needed here, but it won't hurt either. - obj.nodeWaitMutex.Lock() - defer obj.nodeWaitMutex.Unlock() - for _, fn := range obj.nodeWaitFns { - fn() - } - obj.nodeWaitFns = []func(){} // clear -} - -// process is the inner loop that runs through the entire graph. It can be -// called successively safely, as it is roughly idempotent, and is used to push -// values through the graph. If it is interrupted, it can pick up where it left -// off on the next run. This does however require it to re-check some things, -// but that is the price we pay for being always available to unblock. -// Importantly, re-running this resumes work in progress even if there was -// caching, and that if interrupted, it'll be queued again so as to not drop a -// wakeChan notification! We know we've read all the pending incoming values, -// because the Stream reader call wake(). -func (obj *Engine) process(ctx context.Context) (reterr error) { - defer func() { - // catch programming errors - if r := recover(); r != nil { - obj.Logf("Panic in process: %+v", r) - reterr = fmt.Errorf("panic in process: %+v", r) - } - }() - - // Toposort in dependency order. - topoSort, err := obj.graph.TopologicalSort() - if err != nil { - return err - } - - loaded := true // assume we emitted at least one value for now... - - outDegree := obj.graph.OutDegree() // map[Vertex]int - - for _, v := range topoSort { - f, ok := v.(interfaces.Func) - if !ok { - panic("not a Func") - } - node, exists := obj.state[f] - if !exists { - panic(fmt.Sprintf("missing node in iterate: %s", f)) - } - - out, exists := outDegree[f] - if !exists { - panic(fmt.Sprintf("missing out degree in iterate: %s", f)) - } - //outgoing := obj.graph.OutgoingGraphVertices(f) // []pgraph.Vertex - //node.isLeaf = len(outgoing) == 0 - node.isLeaf = out == 0 // store - - // TODO: the obj.loaded stuff isn't really consumed currently - node.rwmutex.RLock() - if !node.loaded { - loaded = false // we were wrong - } - node.rwmutex.RUnlock() - - // TODO: memoize since graph shape doesn't change in this loop! - incoming := obj.graph.IncomingGraphVertices(f) // []pgraph.Vertex - - // no incoming edges, so no incoming data - if len(incoming) == 0 || node.inputClosed { // we do this below - if !node.inputClosed { - node.inputClosed = true - close(node.input) - } - continue - } // else, process input data below... - - ready := true // assume all input values are ready for now... - inputClosed := true // assume all inputs have closed for now... - si := &types.Type{ - // input to functions are structs - Kind: types.KindStruct, - Map: node.Func.Info().Sig.Map, - Ord: node.Func.Info().Sig.Ord, - } - st := types.NewStruct(si) - // The above builds a struct with fields - // populated for each key (empty values) - // so we need to very carefully check if - // every field is received before we can - // safely send it downstream to an edge. - need := make(map[string]struct{}) // keys we need - for _, k := range node.Func.Info().Sig.Ord { - need[k] = struct{}{} - } - - for _, vv := range incoming { - ff, ok := vv.(interfaces.Func) - if !ok { - panic("not a Func") - } - obj.tableMutex.RLock() - value, exists := obj.table[ff] - obj.tableMutex.RUnlock() - if !exists { - ready = false // nope! - inputClosed = false // can't be, it's not even ready yet - break - } - // XXX: do we need a lock around reading obj.state? - fromNode, exists := obj.state[ff] - if !exists { - panic(fmt.Sprintf("missing node in notify: %s", ff)) - } - if !fromNode.outputClosed { - inputClosed = false // if any still open, then we are - } - - // set each arg, since one value - // could get used for multiple - // function inputs (shared edge) - args := obj.graph.Adjacency()[ff][f].(*interfaces.FuncEdge).Args - for _, arg := range args { - // populate struct - if err := st.Set(arg, value); err != nil { - //panic(fmt.Sprintf("struct set failure on `%s` from `%s`: %v", node, fromNode, err)) - keys := []string{} - for k := range st.Struct() { - keys = append(keys, k) - } - panic(fmt.Sprintf("struct set failure on `%s` from `%s`: %v, has: %v", node, fromNode, err, keys)) - } - if _, exists := need[arg]; !exists { - keys := []string{} - for k := range st.Struct() { - keys = append(keys, k) - } - // could be either a duplicate or an unwanted field (edge name) - panic(fmt.Sprintf("unexpected struct key on `%s` from `%s`: %v, has: %v", node, fromNode, err, keys)) - } - delete(need, arg) - } - } - - if !ready || len(need) != 0 { - continue // definitely continue, don't break here - } - - // previously it was closed, skip sending - if node.inputClosed { - continue - } - - // XXX: respect the info.Pure and info.Memo fields somewhere... - - // XXX: keep track of some state about who i sent to last before - // being interrupted so that I can avoid resending to some nodes - // if it's not necessary... - - // It's critical to avoid deadlock with this sending select that - // any events that could happen during this send can be - // preempted and that future executions of this function can be - // resumed. We must return with an error to let folks know that - // we were interrupted. - if obj.Debug { - obj.Logf("send to func `%s`", node) - } - select { - case node.input <- st: // send to function - obj.statsMutex.Lock() - val, _ := obj.stats.inputList[node] // val is # or zero - obj.stats.inputList[node] = val + 1 // increment - obj.statsMutex.Unlock() - // pass - - case <-node.ctx.Done(): // node died - obj.wake("node.ctx.Done()") // interrupted, so queue again - // This scenario *can* happen, although it is rare. It - // triggered the old `chFn && errFn == context.Canceled` - // case which we've now removed. - //return node.ctx.Err() // old behaviour which was wrong - continue // probably best to return and come finish later - - case <-ctx.Done(): - obj.wake("node ctx.Done()") // interrupted, so queue again - return ctx.Err() - } - - // It's okay if this section gets preempted and we re-run this - // function. The worst that happens is we end up sending the - // same input data a second time. This means that we could in - // theory be causing unnecessary graph changes (and locks which - // cause preemption here) if nodes that cause locks aren't - // skipping duplicate/identical input values! - if inputClosed && !node.inputClosed { - node.inputClosed = true - close(node.input) - } - - // XXX: Do we need to somehow wait to make sure that node has - // the time to send at least one output? - // XXX: We could add a counter to each input that gets passed - // through the function... Eg: if we pass in 4, we should wait - // until a 4 comes out the output side. But we'd need to change - // the signature of func for this... - - } // end topoSort loop - - // It's okay if this section gets preempted and we re-run this bit here. - obj.loaded = loaded // this gets reset when graph adds new nodes - - if !loaded { - return nil - } - - // Check each leaf and make sure they're all ready to send, for us to - // send anything to ag channel. In addition, we need at least one send - // message from any of the valid isLeaf nodes. Since this only runs if - // everyone is loaded, we just need to check for activity leaf nodes. - obj.stateMutex.Lock() - for node := range obj.activity { - if obj.leafSend { - break // early - } - - // down here we need `true` activity! - if node.isLeaf { // calculated above in the previous loop - obj.leafSend = true - break - } - } - obj.activity = make(map[*state]struct{}) // clear - //clear(obj.activity) // new clear - - // This check happens here after the send loop to make sure one value - // got in and we didn't close it off too early. - for node := range obj.isClosed { // these are closed - node.outputClosed = true - } - obj.stateMutex.Unlock() - - if !obj.leafSend { - return nil - } - - select { - case obj.ag <- nil: // send to aggregate channel if we have events - obj.Logf("aggregated send") - obj.leafSend = false // reset - - case <-ctx.Done(): - obj.leafSend = true // since we skipped the ag send! - obj.wake("process ctx.Done()") // interrupted, so queue again - return ctx.Err() - - // XXX: should we even allow this default case? - //default: - // // exit if we're not ready to send to ag - // obj.leafSend = true // since we skipped the ag send! - // obj.wake("process default") // interrupted, so queue again - } - - return nil -} - -// Run kicks off the main engine. This takes a mutex. When we're "paused" the -// mutex is temporarily released until we "resume". Those operations transition -// with the engine Lock and Unlock methods. It is recommended to only add -// vertices to the engine after it's running. If you add them before Run, then -// Run will cause a Lock/Unlock to occur to cycle them in. Lock and Unlock race -// with the cancellation of this Run main loop. Make sure to only call one at a -// time. -func (obj *Engine) Run(ctx context.Context) (reterr error) { - obj.graphMutex.Lock() - defer obj.graphMutex.Unlock() - - // XXX: can the above defer get called while we are already unlocked? - // XXX: is it a possibility if we use <-Started() ? - - wg := &sync.WaitGroup{} - defer wg.Wait() - - defer func() { - // catch programming errors - if r := recover(); r != nil { - obj.Logf("Panic in Run: %+v", r) - reterr = fmt.Errorf("panic in Run: %+v", r) - } - }() - - ctx, cancel := context.WithCancel(ctx) // wrap parent - defer cancel() - - // Add a wait before the "started" signal runs so that Cleanup waits. - obj.wg.Add(1) - defer obj.wg.Done() - - // Send the start signal. - close(obj.startedChan) - - if n := obj.graph.NumVertices(); n > 0 { // hack to make the api easier - obj.Logf("graph contained %d vertices before Run", n) - wg.Add(1) - go func() { - defer wg.Done() - // kick the engine once to pull in any vertices from - // before we started running! - defer obj.Unlock() - obj.Lock() - }() - } - - once := &sync.Once{} - loadedSignal := func() { close(obj.loadedChan) } // only run once! - - // aggregate events channel - wg.Add(1) - go func() { - defer wg.Done() - defer close(obj.streamChan) - drain := false - for { - var err error - var ok bool - select { - case err, ok = <-obj.ag: // aggregated channel - if !ok { - return // channel shutdown - } - } - - if drain { - continue // no need to send more errors - } - - // TODO: check obj.loaded first? - once.Do(loadedSignal) - - // now send event... - if obj.Callback != nil { - // send stream signal (callback variant) - obj.Callback(ctx, err) - } else { - // send stream signal - select { - // send events or errors on streamChan - case obj.streamChan <- err: // send - case <-ctx.Done(): // when asked to exit - return - } - } - if err != nil { - cancel() // cancel the context! - //return // let the obj.ag channel drain - drain = true - } - } - }() - - // wgAg is a wait group that waits for all senders to the ag chan. - // Exceptionally, we don't close the ag channel until wgFor has also - // closed, because it can send to wg in process(). - wgAg := &sync.WaitGroup{} - wgFor := &sync.WaitGroup{} - - // We need to keep the main loop running until everyone else has shut - // down. When the top context closes, we wait for everyone to finish, - // and then we shut down this main context. - //mainCtx, mainCancel := context.WithCancel(ctx) // wrap parent - mainCtx, mainCancel := context.WithCancel(context.Background()) // DON'T wrap parent, close on your own terms - defer mainCancel() - - // close the aggregate channel when everyone is done with it... - wg.Add(1) - go func() { - defer wg.Done() - select { - case <-ctx.Done(): - } - - // don't wait and close ag before we're really done with Run() - wgAg.Wait() // wait for last ag user to close - obj.wgTxn.Wait() // wait for first txn as well - mainCancel() // only cancel after wgAg goroutines are done - wgFor.Wait() // wait for process loop to close before closing - close(obj.ag) // last one closes the ag channel - }() - - wgFn := &sync.WaitGroup{} // wg for process function runner - defer wgFn.Wait() // extra safety - - defer obj.runNodeWaitFns() // just in case - - wgFor.Add(1) // make sure we wait for the below process loop to exit... - defer wgFor.Done() - - // errProcess and processBreakFn are used to help exit following an err. - // This approach is needed because if we simply exited, we'd block the - // main loop below because various Stream functions are waiting on the - // Lock/Unlock cycle to be able to finish cleanly, shutdown, and unblock - // all the waitgroups so we can exit. - var errProcess error - var pausedProcess bool - processBreakFn := func(err error /*, paused bool*/) { - if err == nil { // a nil error won't cause ag to shutdown below - panic("expected error, not nil") - } - if obj.Debug { - obj.Logf("process break") - } - select { - case obj.ag <- err: // send error to aggregate channel - case <-ctx.Done(): - } - cancel() // to unblock - //mainCancel() // NO! - errProcess = err // set above error - //pausedProcess = paused // set this inline directly - } - if obj.Debug { - defer obj.Logf("exited main loop") - } - // we start off "running", but we'll have an empty graph initially... - for { - - // After we've resumed, we can try to exit. (shortcut) - // NOTE: If someone calls Lock(), which would send to - // obj.pauseChan, it *won't* deadlock here because mainCtx is - // only closed when all the worker waitgroups close first! - select { - case <-mainCtx.Done(): // when asked to exit - return errProcess // we exit happily - default: - } - - // run through our graph, check for pause request occasionally - for { - pausedProcess = false // reset - // if we're in errProcess, we skip the process loop! - if errProcess != nil { - break // skip this process loop - } - - // Start the process run for this iteration of the loop. - ctxFn, cancelFn := context.WithCancel(context.Background()) - // we run cancelFn() below to cleanup! - var errFn error - chanFn := make(chan struct{}) // normal exit signal - wgFn.Add(1) - go func() { - defer wgFn.Done() - defer close(chanFn) // signal that I exited - for { - if obj.Debug { - obj.Logf("process...") - } - if errFn = obj.process(ctxFn); errFn != nil { // store - if errFn != context.Canceled { - obj.Logf("process end err: %+v...", errFn) - } - return - } - if obj.Debug { - obj.Logf("process end...") - } - // If process finishes without error, we - // should sit here and wait until we get - // run again from a wake-up, or we exit. - select { - case <-obj.wakeChan: // wait until something has actually woken up... - if obj.Debug { - obj.Logf("process wakeup...") - } - // loop! - case <-ctxFn.Done(): - errFn = context.Canceled - return - } - } - }() - - chFn := false - chPause := false - ctxExit := false - select { - //case <-obj.wakeChan: - // this happens entirely in the process inner, inner loop now. - - case <-chanFn: // process exited on it's own in error! - chFn = true - - case <-obj.pauseChan: - if obj.Debug { - obj.Logf("pausing...") - } - chPause = true - - case <-mainCtx.Done(): // when asked to exit - //return nil // we exit happily - ctxExit = true - } - - //fmt.Printf("chPause: %+v\n", chPause) // debug - //fmt.Printf("ctxExit: %+v\n", ctxExit) // debug - - cancelFn() // cancel the process function - wgFn.Wait() // wait for the process function to return - - pausedProcess = chPause // tell the below select - if errFn == nil { - // break on errors (needs to know if paused) - processBreakFn(fmt.Errorf("unexpected nil error in process")) - break - } - if errFn != nil && errFn != context.Canceled { - // break on errors (needs to know if paused) - processBreakFn(errwrap.Wrapf(errFn, "process error")) - break - } - //if errFn == context.Canceled { - // // ignore, we asked for it - //} - - if ctxExit { - return nil // we exit happily - } - if chPause { - break - } - - // This used to happen if a node (in the list we are - // sending to) dies, and we returned with: - // `case <-node.ctx.Done():` // node died - // return node.ctx.Err() - // which caused this scenario. - if chFn && errFn == context.Canceled { // very rare case - // programming error - processBreakFn(fmt.Errorf("legacy unhandled process state")) - break - } - - // programming error - //return fmt.Errorf("unhandled process state") - processBreakFn(fmt.Errorf("unhandled process state")) - break - } - // if we're in errProcess, we need to add back in the pauseChan! - if errProcess != nil && !pausedProcess { - select { - case <-obj.pauseChan: - if obj.Debug { - obj.Logf("lower pausing...") - } - - // do we want this exit case? YES - case <-mainCtx.Done(): // when asked to exit - return errProcess - } - } - - // Toposort for paused workers. We run this before the actual - // pause completes, because the second we are paused, the graph - // could then immediately change. We don't need a lock in here - // because the mutex only unlocks when pause is complete below. - //topoSort1, err := obj.graph.TopologicalSort() - //if err != nil { - // return err - //} - //for _, v := range topoSort1 {} - - // pause is complete - // no exit case from here, must be fully running or paused... - select { - case obj.pausedChan <- struct{}{}: - if obj.Debug { - obj.Logf("paused!") - } - } - - // - // the graph changes shape right here... we are locked right now - // - - // wait until resumed/unlocked - select { - case <-obj.resumeChan: - if obj.Debug { - obj.Logf("resuming...") - } - } - - // Do any cleanup needed from delete vertex. Or do we? - // We've ascertained that while we want this stuff to shutdown, - // and while we also know that a Stream() function running is a - // part of what we're waiting for to exit, it doesn't matter - // that it exits now! This is actually causing a deadlock - // because the pending Stream exit, might be calling a new - // Reverse commit, which means we're deadlocked. It's safe for - // the Stream to keep running, all it might do is needlessly add - // a new value to obj.table which won't bother us since we won't - // even use it in process. We _do_ want to wait for all of these - // before the final exit, but we already have that in a defer. - //obj.runNodeWaitFns() - - // Toposort to run/resume workers. (Bottom of toposort first!) - topoSort2, err := obj.graph.TopologicalSort() - if err != nil { - return err - } - reversed := pgraph.Reverse(topoSort2) - for _, v := range reversed { - f, ok := v.(interfaces.Func) - if !ok { - panic("not a Func") - } - node, exists := obj.state[f] - if !exists { - panic(fmt.Sprintf("missing node in iterate: %s", f)) - } - - if node.running { // it's not a new vertex - continue - } - obj.loaded = false // reset this - node.running = true - - obj.statsMutex.Lock() - val, _ := obj.stats.inputList[node] // val is # or zero - obj.stats.inputList[node] = val // initialize to zero - obj.statsMutex.Unlock() - - innerCtx, innerCancel := context.WithCancel(ctx) // wrap parent (not mainCtx) - // we defer innerCancel() in the goroutine to cleanup! - node.ctx = innerCtx - node.cancel = innerCancel - - // run mainloop - wgAg.Add(1) - node.wg.Add(1) - go func(f interfaces.Func, node *state) { - defer node.wg.Done() - defer wgAg.Done() - defer node.cancel() // if we close, clean up and send the signal to anyone watching - if obj.Debug { - obj.Logf("Running func `%s`", node) - obj.statsMutex.Lock() - obj.stats.runningList[node] = struct{}{} - obj.stats.loadedList[node] = false - obj.statsMutex.Unlock() - } - - fn := func(nodeCtx context.Context) (reterr error) { - // NOTE: Comment out this defer to make - // debugging a lot easier. - defer func() { - // catch programming errors - if r := recover(); r != nil { - obj.Logf("Panic in Stream of func `%s`: %+v", node, r) - reterr = fmt.Errorf("panic in Stream of func `%s`: %+v", node, r) - } - }() - return f.Stream(nodeCtx) - } - runErr := fn(node.ctx) // wrap with recover() - if obj.Debug { - obj.Logf("Exiting func `%s`", node) - obj.statsMutex.Lock() - delete(obj.stats.runningList, node) - obj.statsMutex.Unlock() - } - if runErr != nil { - err := fmt.Errorf("func `%s` errored: %+v", node, runErr) - displayer, ok := node.Func.(interfaces.TextDisplayer) - if ok { - if highlight := displayer.HighlightText(); highlight != "" { - obj.Logf("%s: %s", err.Error(), highlight) - } - } - - obj.Logf("%s", err.Error()) - // send to a aggregate channel - // the first to error will cause ag to - // shutdown, so make sure we can exit... - select { - case obj.ag <- runErr: // send to aggregate channel - case <-node.ctx.Done(): - } - } - // if node never loaded, then we error in the node.output loop! - }(f, node) - - // consume output - wgAg.Add(1) - node.wg.Add(1) - go func(f interfaces.Func, node *state) { - defer node.wg.Done() - defer wgAg.Done() - defer func() { - // We record the fact that output - // closed, so we can eventually close - // the downstream node's input. - obj.stateMutex.Lock() - obj.isClosed[node] = struct{}{} // closed! - obj.stateMutex.Unlock() - // TODO: is this wake necessary? - obj.wake("closed") // closed, so wake up - }() - - for value := range node.output { // read from channel - if value == nil { - // bug in implementation of that func! - s := fmt.Sprintf("func `%s` sent nil value", node) - obj.Logf(s) - panic(s) - } - - obj.tableMutex.RLock() - cached, exists := obj.table[f] - obj.tableMutex.RUnlock() - if !exists { // first value received - // RACE: do this AFTER value is present! - //node.loaded = true // not yet please - if obj.Debug { - obj.Logf("func `%s` started", node) - } - } else if value.Cmp(cached) == nil { - // skip if new value is same as previous - // if this happens often, it *might* be - // a bug in the function implementation - // FIXME: do we need to disable engine - // caching when using hysteresis? - if obj.Debug { - obj.Logf("func `%s` skipped", node) - } - continue - } - obj.tableMutex.Lock() - obj.table[f] = value // save the latest - obj.tableMutex.Unlock() - node.rwmutex.Lock() - node.loaded = true // set *after* value is in :) - //obj.Logf("func `%s` changed", node) - node.rwmutex.Unlock() - - obj.statsMutex.Lock() - obj.stats.loadedList[node] = true - obj.statsMutex.Unlock() - - // Send a message to tell our ag channel - // that we might have sent an aggregated - // message here. They should check if we - // are a leaf and if we glitch or not... - // Make sure we do this before the wake. - obj.stateMutex.Lock() - obj.activity[node] = struct{}{} // activity! - obj.stateMutex.Unlock() - - obj.wake("new value") // new value, so send wake up - - } // end for - - // no more output values are coming... - //obj.Logf("func `%s` stopped", node) - - // nodes that never loaded will cause the engine to hang - if !node.loaded { - select { - case obj.ag <- fmt.Errorf("func `%s` stopped before it was loaded", node): - case <-node.ctx.Done(): - return - } - } - - }(f, node) - - } // end for - - // Send new notifications in case any new edges are sending away - // to these... They might have already missed the notifications! - for k := range obj.resend { // resend TO these! - node, exists := obj.state[k] - if !exists { - continue - } - // Run as a goroutine to avoid erroring in parent thread. - wg.Add(1) - go func(node *state) { - defer wg.Done() - if obj.Debug { - obj.Logf("resend to func `%s`", node) - } - obj.wake("resend") // new value, so send wake up - }(node) - } - obj.resend = make(map[interfaces.Func]struct{}) // reset - - // now check their states... - //for _, v := range reversed { - // v, ok := v.(interfaces.Func) - // if !ok { - // panic("not a Func") - // } - // // wait for startup? - // close(obj.state[v].startup) XXX: once? - //} - - // resume is complete - // no exit case from here, must be fully running or paused... - select { - case obj.resumedChan <- struct{}{}: - if obj.Debug { - obj.Logf("resumed!") - } - } - - } // end for -} - -// Stream returns a channel that you can follow to get aggregated graph events. -// Do not block reading from this channel as you can hold up the entire engine. -func (obj *Engine) Stream() <-chan error { - return obj.streamChan -} - -// Loaded returns a channel that closes when the function engine loads. -func (obj *Engine) Loaded() <-chan struct{} { - return obj.loadedChan -} - -// Table returns a copy of the populated data table of values. We return a copy -// because since these values are constantly changing, we need an atomic -// snapshot to present to the consumer of this API. -// TODO: is this globally glitch consistent? -// TODO: do we need an API to return a single value? (wrapped in read locks) -func (obj *Engine) Table() map[interfaces.Func]types.Value { - obj.tableMutex.RLock() - defer obj.tableMutex.RUnlock() - table := make(map[interfaces.Func]types.Value) - for k, v := range obj.table { - //table[k] = v.Copy() // TODO: do we need to copy these values? - table[k] = v - } - return table -} - -// Apply is similar to Table in that it gives you access to the internal output -// table of data, the difference being that it instead passes this information -// to a function of your choosing and holds a read/write lock during the entire -// time that your function is synchronously executing. If you use this function -// to spawn any goroutines that read or write data, then you're asking for a -// panic. -// XXX: does this need to be a Lock? Can it be an RLock? Check callers! -func (obj *Engine) Apply(fn func(map[interfaces.Func]types.Value) error) error { - // XXX: does this need to be a Lock? Can it be an RLock? Check callers! - obj.tableMutex.Lock() // differs from above RLock around obj.table - defer obj.tableMutex.Unlock() - table := make(map[interfaces.Func]types.Value) - for k, v := range obj.table { - //table[k] = v.Copy() // TODO: do we need to copy these values? - table[k] = v - } - - return fn(table) -} - -// Started returns a channel that closes when the Run function finishes starting -// up. This is useful so that we can wait before calling any of the mutex things -// that would normally panic if Run wasn't started up first. -func (obj *Engine) Started() <-chan struct{} { - return obj.startedChan -} - -// NumVertices returns the number of vertices in the current graph. -func (obj *Engine) NumVertices() int { - // XXX: would this deadlock if we added this? - //obj.graphMutex.Lock() // XXX: should this be a RLock? - //defer obj.graphMutex.Unlock() // XXX: should this be an RUnlock? - return obj.graph.NumVertices() -} - -// Stats returns some statistics in a human-readable form. -func (obj *Engine) Stats() string { - defer obj.statsMutex.RUnlock() - obj.statsMutex.RLock() - - return obj.stats.String() -} - // ExecGraphviz writes out the diagram of a graph to be used for visualization // and debugging. You must not modify the graph (eg: during Lock) when calling // this method. func (obj *Engine) ExecGraphviz(dir string) error { - // XXX: would this deadlock if we added this? - //obj.graphMutex.Lock() // XXX: should this be a RLock? - //defer obj.graphMutex.Unlock() // XXX: should this be an RUnlock? + // No mutex needed here since this func runs in a non-concurrent Txn. - obj.graphvizMutex.Lock() - defer obj.graphvizMutex.Unlock() + // No mutex is needed at this time because we only run this in txn's and + // it should only be run with debugging enabled. Bring your own mutex. + //obj.graphvizMutex.Lock() + //defer obj.graphvizMutex.Unlock() obj.graphvizCount++ // increment @@ -1505,35 +980,35 @@ func (obj *Engine) ExecGraphviz(dir string) error { return err } - dashedEdges, err := pgraph.NewGraph("dashedEdges") - if err != nil { - return err - } - for _, v1 := range obj.graph.Vertices() { - // if it's a ChannelBasedSinkFunc... - if cb, ok := v1.(*structs.ChannelBasedSinkFunc); ok { - // ...then add a dashed edge to its output - dashedEdges.AddEdge(v1, cb.Target, &pgraph.SimpleEdge{ - Name: "channel", // secret channel - }) - } - // if it's a ChannelBasedSourceFunc... - if cb, ok := v1.(*structs.ChannelBasedSourceFunc); ok { - // ...then add a dashed edge from its input - dashedEdges.AddEdge(cb.Source, v1, &pgraph.SimpleEdge{ - Name: "channel", // secret channel - }) - } - } + //dashedEdges, err := pgraph.NewGraph("dashedEdges") + //if err != nil { + // return err + //} + //for _, v1 := range obj.graph.Vertices() { + // // if it's a ChannelBasedSinkFunc... + // if cb, ok := v1.(*structs.ChannelBasedSinkFunc); ok { + // // ...then add a dashed edge to its output + // dashedEdges.AddEdge(v1, cb.Target, &pgraph.SimpleEdge{ + // Name: "channel", // secret channel + // }) + // } + // // if it's a ChannelBasedSourceFunc... + // if cb, ok := v1.(*structs.ChannelBasedSourceFunc); ok { + // // ...then add a dashed edge from its input + // dashedEdges.AddEdge(cb.Source, v1, &pgraph.SimpleEdge{ + // Name: "channel", // secret channel + // }) + // } + //} gv := &pgraph.Graphviz{ Name: obj.graph.GetName(), Filename: fmt.Sprintf("%s/%d.dot", dir, obj.graphvizCount), Graphs: map[*pgraph.Graph]*pgraph.GraphvizOpts{ obj.graph: nil, - dashedEdges: { - Style: "dashed", - }, + //dashedEdges: { + // Style: "dashed", + //}, }, } @@ -1543,29 +1018,30 @@ func (obj *Engine) ExecGraphviz(dir string) error { return nil } +// errAppend is a simple helper function. +func (obj *Engine) errAppend(err error) { + obj.errMutex.Lock() + obj.err = errwrap.Append(obj.err, err) + obj.errMutex.Unlock() +} + // state tracks some internal vertex-specific state information. type state struct { Func interfaces.Func name string // cache a name here for safer concurrency - input chan types.Value // the top level type must be a struct - output chan types.Value - txn interfaces.Txn // API of GraphTxn struct to pass to each function + txn interfaces.Txn // API of GraphTxn struct to pass to each function - //init bool // have we run Init on our func? - //ready bool // has it received all the args it needs at least once? - loaded bool // has the func run at least once ? - inputClosed bool // is our input closed? - outputClosed bool // is our output closed? + // started is true if this is a StreamableFunc, and Stream was started. + started bool - isLeaf bool // is my out degree zero? + // epoch represents the "iteration count" through the graph. All values + // in a returned table should be part of the same epoch. This guarantees + // that they're all consistent with respect to each other. + epoch int64 // if this rolls over, we've been running for too many years - running bool - wg *sync.WaitGroup - ctx context.Context // per state ctx (inner ctx) - cancel func() // cancel above inner ctx - - rwmutex *sync.RWMutex // concurrency guard for reading/modifying this state + // result is the latest output from calling this function. + result types.Value } // String implements the fmt.Stringer interface for pretty printing! @@ -1577,69 +1053,34 @@ func (obj *state) String() string { return obj.Func.String() } -// stats holds some statistics and other debugging information. -type stats struct { - - // runningList keeps track of which nodes are still running. - runningList map[*state]struct{} - - // loadedList keeps track of which nodes have loaded. - loadedList map[*state]bool - - // inputList keeps track of the number of inputs each node received. - inputList map[*state]int64 +// ops is either an addVertex or deleteVertex operation. +type ops interface { } -// String implements the fmt.Stringer interface for printing out our collected -// statistics! -func (obj *stats) String() string { - // XXX: just build the lock into *stats instead of into our dage obj - s := "stats:\n" - { - s += "\trunning:\n" - names := []string{} - for k := range obj.runningList { - names = append(names, k.String()) - } - sort.Strings(names) - for _, name := range names { - s += fmt.Sprintf("\t * %s\n", name) - } - } - { - nodes := []*state{} - for k := range obj.loadedList { - nodes = append(nodes, k) - } - sort.Slice(nodes, func(i, j int) bool { return nodes[i].String() < nodes[j].String() }) - - s += "\tloaded:\n" - for _, node := range nodes { - if !obj.loadedList[node] { - continue - } - s += fmt.Sprintf("\t * %s\n", node) - } - - s += "\tnot loaded:\n" - for _, node := range nodes { - if obj.loadedList[node] { - continue - } - s += fmt.Sprintf("\t * %s\n", node) - } - } - { - s += "\tinput count:\n" - nodes := []*state{} - for k := range obj.inputList { - nodes = append(nodes, k) - } - //sort.Slice(nodes, func(i, j int) bool { return nodes[i].String() < nodes[j].String() }) - sort.Slice(nodes, func(i, j int) bool { return obj.inputList[nodes[i]] < obj.inputList[nodes[j]] }) - for _, node := range nodes { - s += fmt.Sprintf("\t * (%d) %s\n", obj.inputList[node], node) - } - } - return s +// addVertex is one of the "ops" that are possible. +type addVertex struct { + f interfaces.Func + fn func(context.Context) error +} + +// deleteVertex is one of the "ops" that are possible. +type deleteVertex struct { + f interfaces.Func + fn func(context.Context) error +} + +// realEdgeCount tells us how many "logical" edges there are. We have shared +// edges which represent more than one value, when the same value is passed more +// than once. This takes those into account correctly. +func realEdgeCount(edges []pgraph.Edge) int { + total := 0 + for _, edge := range edges { + fe, ok := edge.(*interfaces.FuncEdge) + if !ok { + total++ + continue + } + total += len(fe.Args) // these can represent more than one edge! + } + return total } diff --git a/lang/funcs/dage/dage_test.go b/lang/funcs/dage/dage_test.go deleted file mode 100644 index 84dd93b4..00000000 --- a/lang/funcs/dage/dage_test.go +++ /dev/null @@ -1,807 +0,0 @@ -// Mgmt -// Copyright (C) James Shubin and the project contributors -// Written by James Shubin and the project contributors -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// Additional permission under GNU GPL version 3 section 7 -// -// If you modify this program, or any covered work, by linking or combining it -// with embedded mcl code and modules (and that the embedded mcl code and -// modules which link with this program, contain a copy of their source code in -// the authoritative form) containing parts covered by the terms of any other -// license, the licensors of this program grant you additional permission to -// convey the resulting work. Furthermore, the licensors of this program grant -// the original author, James Shubin, additional permission to update this -// additional permission if he deems it necessary to achieve the goals of this -// additional permission. - -package dage - -import ( - "context" - "fmt" - "sync" - "testing" - "time" - - "github.com/purpleidea/mgmt/lang/interfaces" - "github.com/purpleidea/mgmt/lang/types" - "github.com/purpleidea/mgmt/util" -) - -var _ interfaces.GraphAPI = &Engine{} // ensure it meets this expectation - -type testFunc struct { - Name string - Type *types.Type - Func func(types.Value) (types.Value, error) - Meta *meta - - value types.Value - init *interfaces.Init -} - -func (obj *testFunc) String() string { return obj.Name } - -func (obj *testFunc) Info() *interfaces.Info { - return &interfaces.Info{ - Pure: true, - Memo: false, // TODO: should this be something we specify here? - Sig: obj.Type, - Err: obj.Validate(), - } -} - -func (obj *testFunc) Validate() error { - if obj.Meta == nil { - return fmt.Errorf("test case error: did you add the vertex to the vertices list?") - } - return nil -} - -func (obj *testFunc) Init(init *interfaces.Init) error { - obj.init = init - return nil -} - -func (obj *testFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes - defer obj.init.Logf("stream closed") - obj.init.Logf("stream startup") - - // make some placeholder value because obj.value is nil - constValue, err := types.ValueOfGolang("hello") - if err != nil { - return err // unlikely - } - - for { - select { - case input, ok := <-obj.init.Input: - if !ok { - obj.init.Logf("stream input closed") - obj.init.Input = nil // don't get two closes - // already sent one value, so we can shutdown - if obj.value != nil { - return nil // can't output any more - } - - obj.value = constValue - } else { - obj.init.Logf("stream got input type(%T) value: (%+v)", input, input) - if obj.Func == nil { - obj.value = constValue - } - - if obj.Func != nil { - //obj.init.Logf("running internal function...") - v, err := obj.Func(input) // run me! - if err != nil { - return err - } - obj.value = v - } - } - - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- obj.value: // send anything - // add some monitoring... - obj.Meta.wg.Add(1) - go func() { - // no need for lock here - defer obj.Meta.wg.Done() - if obj.Meta.debug { - obj.Meta.logf("sending an internal event!") - } - - select { - case obj.Meta.Events[obj.Name] <- struct{}{}: - case <-obj.Meta.ctx.Done(): - } - }() - - case <-ctx.Done(): - return nil - } - } -} - -type meta struct { - EventCount int - Event chan struct{} - Events map[string]chan struct{} - - ctx context.Context - wg *sync.WaitGroup - mutex *sync.Mutex - - debug bool - logf func(format string, v ...interface{}) -} - -func (obj *meta) Lock() { obj.mutex.Lock() } -func (obj *meta) Unlock() { obj.mutex.Unlock() } - -type dageTestOp func(*Engine, interfaces.Txn, *meta) error - -func TestDageTable(t *testing.T) { - - type test struct { // an individual test - name string - vertices []interfaces.Func - actions []dageTestOp - } - testCases := []test{} - { - testCases = append(testCases, test{ - name: "empty graph", - vertices: []interfaces.Func{}, - actions: []dageTestOp{ - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - engine.Lock() - time.Sleep(1 * time.Second) // XXX: unfortunate - defer engine.Unlock() - return nil - }, - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - time.Sleep(1 * time.Second) // XXX: unfortunate - meta.Lock() - defer meta.Unlock() - // We don't expect an empty graph to send events. - if meta.EventCount != 0 { - return fmt.Errorf("got too many stream events") - } - return nil - }, - }, - }) - } - { - f1 := &testFunc{Name: "f1", Type: types.NewType("func() str")} - - testCases = append(testCases, test{ - name: "simple add vertex", - vertices: []interfaces.Func{f1}, // so the test engine can pass in debug/observability handles - actions: []dageTestOp{ - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - engine.Lock() - defer engine.Unlock() - return engine.AddVertex(f1) - }, - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - time.Sleep(1 * time.Second) // XXX: unfortunate - meta.Lock() - defer meta.Unlock() - if meta.EventCount < 1 { - return fmt.Errorf("didn't get any stream events") - } - return nil - }, - }, - }) - } - { - f1 := &testFunc{Name: "f1", Type: types.NewType("func() str")} - // e1 arg name must match incoming edge to it - f2 := &testFunc{Name: "f2", Type: types.NewType("func(e1 str) str")} - e1 := testEdge("e1") - - testCases = append(testCases, test{ - name: "simple add edge", - vertices: []interfaces.Func{f1, f2}, - actions: []dageTestOp{ - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - engine.Lock() - defer engine.Unlock() - return engine.AddVertex(f1) - }, - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - time.Sleep(1 * time.Second) // XXX: unfortunate - engine.Lock() - defer engine.Unlock() - // This newly added node should get a notification after it starts. - return engine.AddEdge(f1, f2, e1) - }, - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - time.Sleep(1 * time.Second) // XXX: unfortunate - meta.Lock() - defer meta.Unlock() - if meta.EventCount < 2 { - return fmt.Errorf("didn't get enough stream events") - } - return nil - }, - }, - }) - } - { - // diamond - f1 := &testFunc{Name: "f1", Type: types.NewType("func() str")} - f2 := &testFunc{Name: "f2", Type: types.NewType("func(e1 str) str")} - f3 := &testFunc{Name: "f3", Type: types.NewType("func(e2 str) str")} - f4 := &testFunc{Name: "f4", Type: types.NewType("func(e3 str, e4 str) str")} - e1 := testEdge("e1") - e2 := testEdge("e2") - e3 := testEdge("e3") - e4 := testEdge("e4") - - testCases = append(testCases, test{ - name: "simple add multiple edges", - vertices: []interfaces.Func{f1, f2, f3, f4}, - actions: []dageTestOp{ - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - engine.Lock() - defer engine.Unlock() - return engine.AddVertex(f1) - }, - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - engine.Lock() - defer engine.Unlock() - if err := engine.AddEdge(f1, f2, e1); err != nil { - return err - } - if err := engine.AddEdge(f1, f3, e2); err != nil { - return err - } - return nil - }, - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - engine.Lock() - defer engine.Unlock() - if err := engine.AddEdge(f2, f4, e3); err != nil { - return err - } - if err := engine.AddEdge(f3, f4, e4); err != nil { - return err - } - return nil - }, - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - //meta.Lock() - //defer meta.Unlock() - num := 1 - for { - if num == 0 { - break - } - select { - case _, ok := <-meta.Event: - if !ok { - return fmt.Errorf("unexpectedly channel close") - } - num-- - if meta.debug { - meta.logf("got an event!") - } - case <-meta.ctx.Done(): - return meta.ctx.Err() - } - } - return nil - }, - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - meta.Lock() - defer meta.Unlock() - if meta.EventCount < 1 { - return fmt.Errorf("didn't get enough stream events") - } - return nil - }, - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - //meta.Lock() - //defer meta.Unlock() - num := 1 - for { - if num == 0 { - break - } - bt := util.BlockedTimer{Seconds: 2} - defer bt.Cancel() - bt.Printf("waiting for f4...\n") - select { - case _, ok := <-meta.Events["f4"]: - bt.Cancel() - if !ok { - return fmt.Errorf("unexpectedly channel close") - } - num-- - if meta.debug { - meta.logf("got an event from f4!") - } - case <-meta.ctx.Done(): - return meta.ctx.Err() - } - } - return nil - }, - }, - }) - } - { - f1 := &testFunc{Name: "f1", Type: types.NewType("func() str")} - - testCases = append(testCases, test{ - name: "simple add/delete vertex", - vertices: []interfaces.Func{f1}, - actions: []dageTestOp{ - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - engine.Lock() - defer engine.Unlock() - return engine.AddVertex(f1) - }, - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - time.Sleep(1 * time.Second) // XXX: unfortunate - meta.Lock() - defer meta.Unlock() - if meta.EventCount < 1 { - return fmt.Errorf("didn't get enough stream events") - } - return nil - }, - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - engine.Lock() - defer engine.Unlock() - - //meta.Lock() - //defer meta.Unlock() - if meta.debug { - meta.logf("about to delete vertex f1!") - defer meta.logf("done deleting vertex f1!") - } - - return engine.DeleteVertex(f1) - }, - }, - }) - } - { - f1 := &testFunc{Name: "f1", Type: types.NewType("func() str")} - // e1 arg name must match incoming edge to it - f2 := &testFunc{Name: "f2", Type: types.NewType("func(e1 str) str")} - e1 := testEdge("e1") - - testCases = append(testCases, test{ - name: "simple add/delete edge", - vertices: []interfaces.Func{f1, f2}, - actions: []dageTestOp{ - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - engine.Lock() - defer engine.Unlock() - return engine.AddVertex(f1) - }, - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - time.Sleep(1 * time.Second) // XXX: unfortunate - engine.Lock() - defer engine.Unlock() - // This newly added node should get a notification after it starts. - return engine.AddEdge(f1, f2, e1) - }, - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - time.Sleep(1 * time.Second) // XXX: unfortunate - meta.Lock() - defer meta.Unlock() - if meta.EventCount < 2 { - return fmt.Errorf("didn't get enough stream events") - } - return nil - }, - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - engine.Lock() - defer engine.Unlock() - return engine.DeleteEdge(e1) - }, - }, - }) - } - - // the following tests use the txn instead of direct locks - { - f1 := &testFunc{Name: "f1", Type: types.NewType("func() str")} - - testCases = append(testCases, test{ - name: "txn simple add vertex", - vertices: []interfaces.Func{f1}, // so the test engine can pass in debug/observability handles - actions: []dageTestOp{ - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - return txn.AddVertex(f1).Commit() - }, - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - time.Sleep(1 * time.Second) // XXX: unfortunate - meta.Lock() - defer meta.Unlock() - if meta.EventCount < 1 { - return fmt.Errorf("didn't get any stream events") - } - return nil - }, - }, - }) - } - { - f1 := &testFunc{Name: "f1", Type: types.NewType("func() str")} - // e1 arg name must match incoming edge to it - f2 := &testFunc{Name: "f2", Type: types.NewType("func(e1 str) str")} - e1 := testEdge("e1") - - testCases = append(testCases, test{ - name: "txn simple add edge", - vertices: []interfaces.Func{f1, f2}, - actions: []dageTestOp{ - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - return txn.AddVertex(f1).Commit() - }, - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - time.Sleep(1 * time.Second) // XXX: unfortunate - // This newly added node should get a notification after it starts. - return txn.AddEdge(f1, f2, e1).Commit() - }, - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - time.Sleep(1 * time.Second) // XXX: unfortunate - meta.Lock() - defer meta.Unlock() - if meta.EventCount < 2 { - return fmt.Errorf("didn't get enough stream events") - } - return nil - }, - }, - }) - } - { - // diamond - f1 := &testFunc{Name: "f1", Type: types.NewType("func() str")} - f2 := &testFunc{Name: "f2", Type: types.NewType("func(e1 str) str")} - f3 := &testFunc{Name: "f3", Type: types.NewType("func(e2 str) str")} - f4 := &testFunc{Name: "f4", Type: types.NewType("func(e3 str, e4 str) str")} - e1 := testEdge("e1") - e2 := testEdge("e2") - e3 := testEdge("e3") - e4 := testEdge("e4") - - testCases = append(testCases, test{ - name: "txn simple add multiple edges", - vertices: []interfaces.Func{f1, f2, f3, f4}, - actions: []dageTestOp{ - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - return txn.AddVertex(f1).Commit() - }, - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - return txn.AddEdge(f1, f2, e1).AddEdge(f1, f3, e2).Commit() - }, - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - return txn.AddEdge(f2, f4, e3).AddEdge(f3, f4, e4).Commit() - }, - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - //meta.Lock() - //defer meta.Unlock() - num := 1 - for { - if num == 0 { - break - } - select { - case _, ok := <-meta.Event: - if !ok { - return fmt.Errorf("unexpectedly channel close") - } - num-- - if meta.debug { - meta.logf("got an event!") - } - case <-meta.ctx.Done(): - return meta.ctx.Err() - } - } - return nil - }, - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - meta.Lock() - defer meta.Unlock() - if meta.EventCount < 1 { - return fmt.Errorf("didn't get enough stream events") - } - return nil - }, - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - //meta.Lock() - //defer meta.Unlock() - num := 1 - for { - if num == 0 { - break - } - bt := util.BlockedTimer{Seconds: 2} - defer bt.Cancel() - bt.Printf("waiting for f4...\n") - select { - case _, ok := <-meta.Events["f4"]: - bt.Cancel() - if !ok { - return fmt.Errorf("unexpectedly channel close") - } - num-- - if meta.debug { - meta.logf("got an event from f4!") - } - case <-meta.ctx.Done(): - return meta.ctx.Err() - } - } - return nil - }, - }, - }) - } - { - f1 := &testFunc{Name: "f1", Type: types.NewType("func() str")} - - testCases = append(testCases, test{ - name: "txn simple add/delete vertex", - vertices: []interfaces.Func{f1}, - actions: []dageTestOp{ - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - return txn.AddVertex(f1).Commit() - }, - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - time.Sleep(1 * time.Second) // XXX: unfortunate - meta.Lock() - defer meta.Unlock() - if meta.EventCount < 1 { - return fmt.Errorf("didn't get enough stream events") - } - return nil - }, - func(engine *Engine, txn interfaces.Txn, meta *meta) error { - //meta.Lock() - //defer meta.Unlock() - if meta.debug { - meta.logf("about to delete vertex f1!") - defer meta.logf("done deleting vertex f1!") - } - - return txn.DeleteVertex(f1).Commit() - }, - }, - }) - } - //{ - // f1 := &testFunc{Name: "f1", Type: types.NewType("func() str")} - // // e1 arg name must match incoming edge to it - // f2 := &testFunc{Name: "f2", Type: types.NewType("func(e1 str) str")} - // e1 := testEdge("e1") - // - // testCases = append(testCases, test{ - // name: "txn simple add/delete edge", - // vertices: []interfaces.Func{f1, f2}, - // actions: []dageTestOp{ - // func(engine *Engine, txn interfaces.Txn, meta *meta) error { - // return txn.AddVertex(f1).Commit() - // }, - // func(engine *Engine, txn interfaces.Txn, meta *meta) error { - // time.Sleep(1 * time.Second) // XXX: unfortunate - // // This newly added node should get a notification after it starts. - // return txn.AddEdge(f1, f2, e1).Commit() - // }, - // func(engine *Engine, txn interfaces.Txn, meta *meta) error { - // time.Sleep(1 * time.Second) // XXX: unfortunate - // meta.Lock() - // defer meta.Unlock() - // if meta.EventCount < 2 { - // return fmt.Errorf("didn't get enough stream events") - // } - // return nil - // }, - // func(engine *Engine, txn interfaces.Txn, meta *meta) error { - // return txn.DeleteEdge(e1).Commit() // XXX: not implemented - // }, - // }, - // }) - //} - - if testing.Short() { - t.Logf("available tests:") - } - names := []string{} - for index, tc := range testCases { // run all the tests - if tc.name == "" { - t.Errorf("test #%d: not named", index) - continue - } - if util.StrInList(tc.name, names) { - t.Errorf("test #%d: duplicate sub test name of: %s", index, tc.name) - continue - } - names = append(names, tc.name) - - //if index != 3 { // hack to run a subset (useful for debugging) - //if tc.name != "simple txn" { - // continue - //} - - testName := fmt.Sprintf("test #%d (%s)", index, tc.name) - if testing.Short() { // make listing tests easier - t.Logf("%s", testName) - continue - } - t.Run(testName, func(t *testing.T) { - name, vertices, actions := tc.name, tc.vertices, tc.actions - - t.Logf("\n\ntest #%d (%s) ----------------\n\n", index, name) - - //logf := func(format string, v ...interface{}) { - // t.Logf(fmt.Sprintf("test #%d", index)+": "+format, v...) - //} - - //now := time.Now() - - wg := &sync.WaitGroup{} - defer wg.Wait() // defer is correct b/c we're in a func! - - min := 5 * time.Second // approx min time needed for the test - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - if deadline, ok := t.Deadline(); ok { - d := deadline.Add(-min) - //t.Logf(" now: %+v", now) - //t.Logf(" d: %+v", d) - newCtx, cancel := context.WithDeadline(ctx, d) - ctx = newCtx - defer cancel() - } - - debug := testing.Verbose() // set via the -test.v flag to `go test` - - meta := &meta{ - Event: make(chan struct{}), - Events: make(map[string]chan struct{}), - - ctx: ctx, - wg: &sync.WaitGroup{}, - mutex: &sync.Mutex{}, - - debug: debug, - logf: func(format string, v ...interface{}) { - // safe Logf in case f.String contains %? chars... - s := fmt.Sprintf(format, v...) - t.Logf("%s", s) - }, - } - defer meta.wg.Wait() - - for _, f := range vertices { - testFunc, ok := f.(*testFunc) - if !ok { - t.Errorf("bad test function: %+v", f) - return - } - meta.Events[testFunc.Name] = make(chan struct{}) - testFunc.Meta = meta // add the handle - } - - wg.Add(1) - go func() { - defer wg.Done() - select { - case <-ctx.Done(): - t.Logf("cancelling test...") - } - }() - - engine := &Engine{ - Name: "dage", - - Debug: debug, - Logf: t.Logf, - } - - if err := engine.Setup(); err != nil { - t.Errorf("could not setup engine: %+v", err) - return - } - defer engine.Cleanup() - - wg.Add(1) - go func() { - defer wg.Done() - if err := engine.Run(ctx); err != nil { - t.Errorf("error while running engine: %+v", err) - return - } - t.Logf("engine shutdown cleanly...") - }() - - <-engine.Started() // wait for startup (will not block forever) - - txn := engine.Txn() - defer txn.Free() // remember to call Free() - - wg.Add(1) - go func() { - defer wg.Done() - ch := engine.Stream() - for { - select { - case err, ok := <-ch: // channel must close to shutdown - if !ok { - return - } - meta.Lock() - meta.EventCount++ - meta.Unlock() - meta.wg.Add(1) - go func() { - // no need for lock here - defer meta.wg.Done() - if meta.debug { - meta.logf("sending an event!") - } - select { - case meta.Event <- struct{}{}: - case <-meta.ctx.Done(): - } - }() - if err != nil { - t.Errorf("graph error event: %v", err) - continue - } - t.Logf("graph stream event!") - } - } - }() - - // Run a list of actions. Any error kills it all. - t.Logf("starting actions...") - for i, action := range actions { - t.Logf("running action %d...", i+1) - if err := action(engine, txn, meta); err != nil { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: action #%d failed with: %+v", index, i, err) - break // so that cancel runs - } - } - - t.Logf("test done...") - cancel() - }) - } - - if testing.Short() { - t.Skip("skipping all tests...") - } -} diff --git a/lang/funcs/operators/operators.go b/lang/funcs/operators/operators.go index 954d34b4..34f04a2f 100644 --- a/lang/funcs/operators/operators.go +++ b/lang/funcs/operators/operators.go @@ -681,70 +681,6 @@ func (obj *OperatorFunc) Init(init *interfaces.Init) error { return nil } -// Stream returns the changing values that this func has over time. -func (obj *OperatorFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes - for { - select { - case input, ok := <-obj.init.Input: - if !ok { - return nil // can't output any more - } - //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { - // return errwrap.Wrapf(err, "wrong function input") - //} - - if obj.last != nil && input.Cmp(obj.last) == nil { - continue // value didn't change, skip it - } - obj.last = input // store for next - - // programming error safety check... - programmingError := false - keys := []string{} - for k := range input.Struct() { - keys = append(keys, k) - if !util.StrInList(k, obj.Type.Ord) { - programmingError = true - } - } - if programmingError { - return fmt.Errorf("bad args, got: %v, want: %v", keys, obj.Type.Ord) - } - - args, err := interfaces.StructToCallableArgs(input) // []types.Value, error) - if err != nil { - return err - } - - result, err := obj.Call(ctx, args) // (Value, error) - if err != nil { - return errwrap.Wrapf(err, "problem running function") - } - if result == nil { - return fmt.Errorf("computed function output was nil") - } - - // if previous input was `2 + 4`, but now it - // changed to `1 + 5`, the result is still the - // same, so we can skip sending an update... - if obj.result != nil && result.Cmp(obj.result) == nil { - continue // result didn't change - } - obj.result = result // store new result - - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- obj.result: // send - case <-ctx.Done(): - return nil - } - } -} - // Copy is implemented so that the obj.Type value is not lost if we copy this // function. func (obj *OperatorFunc) Copy() interfaces.Func { diff --git a/lang/funcs/structs/call.go b/lang/funcs/structs/call.go index 1ec9800f..5ab458b5 100644 --- a/lang/funcs/structs/call.go +++ b/lang/funcs/structs/call.go @@ -32,7 +32,6 @@ package structs import ( "context" "fmt" - "sync" "github.com/purpleidea/mgmt/lang/interfaces" "github.com/purpleidea/mgmt/lang/types" @@ -60,8 +59,9 @@ type CallFunc struct { FuncType *types.Type // the type of the function EdgeName string // name of the edge used - ArgVertices []interfaces.Func - + // These two fields are identical to what is used by a ShapelyFunc. + // TODO: Consider just using that interface here instead? + ArgVertices []interfaces.Func OutputVertex interfaces.Func init *interfaces.Init @@ -111,8 +111,10 @@ func (obj *CallFunc) Info() *interfaces.Info { } return &interfaces.Info{ - Pure: true, - Memo: false, // TODO: ??? + Pure: false, // TODO: ??? + Memo: false, + Fast: false, + Spec: false, Sig: typ, Err: obj.Validate(), } @@ -125,90 +127,6 @@ func (obj *CallFunc) Init(init *interfaces.Init) error { return nil } -// Stream takes an input struct in the format as described in the Func and Graph -// methods of the Expr, and returns the actual expected value as a stream based -// on the changing inputs to that value. -func (obj *CallFunc) Stream(ctx context.Context) error { - // XXX: is there a sync.Once sort of solution that would be more elegant here? - mutex := &sync.Mutex{} - done := false - send := func(ctx context.Context, b bool) error { - mutex.Lock() - defer mutex.Unlock() - if done { - return nil - } - done = true - defer close(obj.init.Output) // the sender closes - - if !b { - return nil - } - - // send dummy value to the output - select { - case obj.init.Output <- types.NewFloat(): // XXX: dummy value - case <-ctx.Done(): - return ctx.Err() - } - - return nil - } - defer send(ctx, false) // just close - - defer func() { - obj.init.Txn.Reverse() - }() - - for { - select { - case input, ok := <-obj.init.Input: - if !ok { - obj.init.Input = nil // block looping back here - if !done { - return fmt.Errorf("input closed without ever sending anything") - } - return nil - } - - value, exists := input.Struct()[obj.EdgeName] - if !exists { - return fmt.Errorf("programming error, can't find edge") - } - - newFuncValue, ok := value.(*full.FuncValue) - if !ok { - return fmt.Errorf("programming error, can't convert to *FuncValue") - } - - // It's important to have this compare step to avoid - // redundant graph replacements which slow things down, - // but also cause the engine to lock, which can preempt - // the process scheduler, which can cause duplicate or - // unnecessary re-sending of values here, which causes - // the whole process to repeat ad-nauseum. - if newFuncValue == obj.lastFuncValue { - continue - } - // If we have a new function, then we need to replace - // the subgraph with a new one that uses the new - // function. - obj.lastFuncValue = newFuncValue - - if err := obj.replaceSubGraph(newFuncValue); err != nil { - return errwrap.Wrapf(err, "could not replace subgraph") - } - - send(ctx, true) // send dummy and then close - - continue - - case <-ctx.Done(): - return nil - } - } -} - func (obj *CallFunc) replaceSubGraph(newFuncValue *full.FuncValue) error { // Create a subgraph which looks as follows. // @@ -233,16 +151,61 @@ func (obj *CallFunc) replaceSubGraph(newFuncValue *full.FuncValue) error { // methods called on it. Nothing else. It will _not_ call Commit or // Reverse. It adds to the graph, and our Commit and Reverse operations // are the ones that actually make the change. - outputFunc, err := newFuncValue.CallWithFuncs(obj.init.Txn, obj.ArgVertices) + outputFunc, err := newFuncValue.CallWithFuncs(obj.init.Txn, obj.ArgVertices, obj.OutputVertex) if err != nil { return errwrap.Wrapf(err, "could not call newFuncValue.Call()") } // create the new subgraph - edgeName := OutputFuncArgName - edge := &interfaces.FuncEdge{Args: []string{edgeName}} + edge := &interfaces.FuncEdge{Args: []string{OutputFuncArgName}} // "out" obj.init.Txn.AddVertex(outputFunc) - obj.init.Txn.AddEdge(outputFunc, obj.OutputVertex, edge) + + // XXX: We don't want to do this for ShapelyFunc's. This is a hack b/c I + // wasn't sure how to make this more consistent elsewhere. Look at the + // "hack" edge in iter.map and iter.filter as those need this hack. + // XXX: maybe this interface could return the funcSubgraphOutput node? + if _, ok := outputFunc.(interfaces.ShapelyFunc); !ok { + obj.init.Txn.AddEdge(outputFunc, obj.OutputVertex, edge) + } return obj.init.Txn.Commit() } + +// Call this function with the input args and return the value if it is possible +// to do so at this time. +func (obj *CallFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) { + if len(args) < 1 { + return nil, fmt.Errorf("not enough args") + } + value := args[0] + + newFuncValue, ok := value.(*full.FuncValue) + if !ok { + return nil, fmt.Errorf("programming error, can't convert to *FuncValue") + } + + if newFuncValue != obj.lastFuncValue { + // If we have a new function, then we need to replace the + // subgraph with a new one that uses the new function. + obj.lastFuncValue = newFuncValue + + // This does *not* deadlock, because running a Txn, can't cause + // a second Txn to run automatically. What can happen following + // this replacement and subsequent Txn execution, is that we'll + // run the interrupt which then lets the new Call functions run + // and they then can call more Txn exections and so on... + if err := obj.replaceSubGraph(newFuncValue); err != nil { + return nil, errwrap.Wrapf(err, "could not replace subgraph") + } + + return nil, interfaces.ErrInterrupt + } + + // send dummy value to the output + return types.NewNil(), nil // dummy value +} + +// Cleanup runs after that function was removed from the graph. +func (obj *CallFunc) Cleanup(ctx context.Context) error { + return obj.init.Txn.Reverse() +} diff --git a/lang/funcs/structs/channel_based_sink.go b/lang/funcs/structs/channel_based_sink.go deleted file mode 100644 index 586f310c..00000000 --- a/lang/funcs/structs/channel_based_sink.go +++ /dev/null @@ -1,156 +0,0 @@ -// Mgmt -// Copyright (C) James Shubin and the project contributors -// Written by James Shubin and the project contributors -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// Additional permission under GNU GPL version 3 section 7 -// -// If you modify this program, or any covered work, by linking or combining it -// with embedded mcl code and modules (and that the embedded mcl code and -// modules which link with this program, contain a copy of their source code in -// the authoritative form) containing parts covered by the terms of any other -// license, the licensors of this program grant you additional permission to -// convey the resulting work. Furthermore, the licensors of this program grant -// the original author, James Shubin, additional permission to update this -// additional permission if he deems it necessary to achieve the goals of this -// additional permission. - -package structs - -import ( - "context" - "fmt" - - "github.com/purpleidea/mgmt/lang/interfaces" - "github.com/purpleidea/mgmt/lang/types" -) - -const ( - // ChannelBasedSinkFuncArgName is the name for the edge which connects - // the input value to ChannelBasedSinkFunc. - ChannelBasedSinkFuncArgName = "channelBasedSinkFuncArg" -) - -// ChannelBasedSinkFunc is a Func which receives values from upstream nodes and -// emits them to a golang channel. -type ChannelBasedSinkFunc struct { - Name string - EdgeName string - Target interfaces.Func // for drawing dashed edges in the Graphviz visualization - - Chan chan types.Value - Type *types.Type - - init *interfaces.Init - last types.Value // last value received to use for diff -} - -// String returns a simple name for this function. This is needed so this struct -// can satisfy the pgraph.Vertex interface. -func (obj *ChannelBasedSinkFunc) String() string { - return obj.Name -} - -// ArgGen returns the Nth arg name for this function. -func (obj *ChannelBasedSinkFunc) ArgGen(index int) (string, error) { - if index != 0 { - return "", fmt.Errorf("the ChannelBasedSinkFunc only has one argument") - } - return obj.EdgeName, nil -} - -// Validate makes sure we've built our struct properly. It is usually unused for -// normal functions that users can use directly. -func (obj *ChannelBasedSinkFunc) Validate() error { - if obj.Chan == nil { - return fmt.Errorf("the Chan was not set") - } - return nil -} - -// Info returns some static info about itself. -func (obj *ChannelBasedSinkFunc) Info() *interfaces.Info { - return &interfaces.Info{ - Pure: false, - Memo: false, - Sig: types.NewType(fmt.Sprintf("func(%s %s) %s", obj.EdgeName, obj.Type, obj.Type)), - Err: obj.Validate(), - } -} - -// Init runs some startup code for this function. -func (obj *ChannelBasedSinkFunc) Init(init *interfaces.Init) error { - obj.init = init - return nil -} - -// Stream returns the changing values that this func has over time. -func (obj *ChannelBasedSinkFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes - defer close(obj.Chan) // the sender closes - - for { - select { - case input, ok := <-obj.init.Input: - if !ok { - return nil // can't output any more - } - - args, err := interfaces.StructToCallableArgs(input) // []types.Value, error) - if err != nil { - return err - } - - result, err := obj.Call(ctx, args) // get the value... - if err != nil { - return err - } - - if obj.last != nil && result.Cmp(obj.last) == nil { - continue // value didn't change, skip it - } - obj.last = result // store so we can send after this select - - case <-ctx.Done(): - return nil - } - - select { - case obj.Chan <- obj.last: // send - case <-ctx.Done(): - return nil - } - - // Also send the value downstream. If we don't, then when we - // close the Output channel, the function engine is going to - // complain that we closed that channel without sending it any - // value. - select { - case obj.init.Output <- obj.last: // send - case <-ctx.Done(): - return nil - } - } -} - -// Call this function with the input args and return the value if it is possible -// to do so at this time. -// XXX: Is it correct to implement this here for this particular function? -func (obj *ChannelBasedSinkFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) { - if len(args) != 1 { - return nil, fmt.Errorf("programming error, can't find edge") - } - return args[0], nil -} diff --git a/lang/funcs/structs/channel_based_source.go b/lang/funcs/structs/channel_based_source.go deleted file mode 100644 index 44cea079..00000000 --- a/lang/funcs/structs/channel_based_source.go +++ /dev/null @@ -1,125 +0,0 @@ -// Mgmt -// Copyright (C) James Shubin and the project contributors -// Written by James Shubin and the project contributors -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -// -// Additional permission under GNU GPL version 3 section 7 -// -// If you modify this program, or any covered work, by linking or combining it -// with embedded mcl code and modules (and that the embedded mcl code and -// modules which link with this program, contain a copy of their source code in -// the authoritative form) containing parts covered by the terms of any other -// license, the licensors of this program grant you additional permission to -// convey the resulting work. Furthermore, the licensors of this program grant -// the original author, James Shubin, additional permission to update this -// additional permission if he deems it necessary to achieve the goals of this -// additional permission. - -package structs - -import ( - "context" - "fmt" - - "github.com/purpleidea/mgmt/lang/interfaces" - "github.com/purpleidea/mgmt/lang/types" -) - -// ChannelBasedSourceFunc is a Func which receives values from a golang channel -// and emits them to the downstream nodes. -type ChannelBasedSourceFunc struct { - Name string - Source interfaces.Func // for drawing dashed edges in the Graphviz visualization - - Chan chan types.Value - Type *types.Type - - init *interfaces.Init - last types.Value // last value received to use for diff -} - -// String returns a simple name for this function. This is needed so this struct -// can satisfy the pgraph.Vertex interface. -func (obj *ChannelBasedSourceFunc) String() string { - return "ChannelBasedSourceFunc" -} - -// ArgGen returns the Nth arg name for this function. -func (obj *ChannelBasedSourceFunc) ArgGen(index int) (string, error) { - return "", fmt.Errorf("the ChannelBasedSourceFunc doesn't have any arguments") -} - -// Validate makes sure we've built our struct properly. It is usually unused for -// normal functions that users can use directly. -func (obj *ChannelBasedSourceFunc) Validate() error { - if obj.Chan == nil { - return fmt.Errorf("the Chan was not set") - } - return nil -} - -// Info returns some static info about itself. -func (obj *ChannelBasedSourceFunc) Info() *interfaces.Info { - return &interfaces.Info{ - Pure: false, - Memo: false, - Sig: types.NewType(fmt.Sprintf("func() %s", obj.Type)), - Err: obj.Validate(), - } -} - -// Init runs some startup code for this function. -func (obj *ChannelBasedSourceFunc) Init(init *interfaces.Init) error { - obj.init = init - return nil -} - -// Stream returns the changing values that this func has over time. -func (obj *ChannelBasedSourceFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes - - for { - select { - case input, ok := <-obj.Chan: - if !ok { - return nil // can't output any more - } - - //if obj.last != nil && input.Cmp(obj.last) == nil { - // continue // value didn't change, skip it - //} - obj.last = input // store so we can send after this select - - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- obj.last: // send - case <-ctx.Done(): - return nil - } - } -} - -// XXX: Is it correct to implement this here for this particular function? -// XXX: tricky since this really receives input from a secret channel... -// XXX: ADD A MUTEX AROUND READING obj.last ??? -//func (obj *ChannelBasedSourceFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) { -// if obj.last == nil { -// return nil, fmt.Errorf("programming error") -// } -// return obj.last, nil -//} diff --git a/lang/funcs/structs/composite.go b/lang/funcs/structs/composite.go index 47a77483..bf1f644c 100644 --- a/lang/funcs/structs/composite.go +++ b/lang/funcs/structs/composite.go @@ -144,75 +144,6 @@ func (obj *CompositeFunc) Init(init *interfaces.Init) error { return nil } -// Stream takes an input struct in the format as described in the Func and Graph -// methods of the Expr, and returns the actual expected value as a stream based -// on the changing inputs to that value. -func (obj *CompositeFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes - for { - select { - case input, ok := <-obj.init.Input: - if !ok { - obj.init.Input = nil // don't infinite loop back - - if obj.last == nil { - result, err := obj.StructCall(ctx, obj.last) - if err != nil { - return err - } - - obj.result = result - select { - case obj.init.Output <- result: // send - // pass - case <-ctx.Done(): - return nil - } - } - - return nil // can't output any more - } - //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { - // return errwrap.Wrapf(err, "wrong function input") - //} - if obj.last != nil && input.Cmp(obj.last) == nil { - continue // value didn't change, skip it - } - obj.last = input // store for next - - // TODO: use the normal Call interface instead? - //args, err := interfaces.StructToCallableArgs(input) // []types.Value, error) - //if err != nil { - // return err - //} - //result, err := obj.Call(ctx, args) - //if err != nil { - // return err - //} - result, err := obj.StructCall(ctx, input) - if err != nil { - return err - } - - // skip sending an update... - if obj.result != nil && result.Cmp(obj.result) == nil { - continue // result didn't change - } - obj.result = result // store new result - - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- obj.result: // send - // pass - case <-ctx.Done(): - return nil - } - } -} - // StructCall is a different Call API which is sometimes easier to implement. func (obj *CompositeFunc) StructCall(ctx context.Context, st types.Value) (types.Value, error) { if st == nil { diff --git a/lang/funcs/structs/const.go b/lang/funcs/structs/const.go index 358dfd8f..b2d1aa6c 100644 --- a/lang/funcs/structs/const.go +++ b/lang/funcs/structs/const.go @@ -94,22 +94,6 @@ func (obj *ConstFunc) Init(init *interfaces.Init) error { return nil } -// Stream returns the single value that this const has, and then closes. -func (obj *ConstFunc) Stream(ctx context.Context) error { - value, err := obj.Call(ctx, nil) - if err != nil { - return err - } - select { - case obj.init.Output <- value: - // pass - case <-ctx.Done(): - return nil - } - close(obj.init.Output) // signal that we're done sending - return nil -} - // Call this function with the input args and return the value if it is possible // to do so at this time. func (obj *ConstFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) { diff --git a/lang/funcs/structs/exprif.go b/lang/funcs/structs/exprif.go index daeb60dd..dfeaefa2 100644 --- a/lang/funcs/structs/exprif.go +++ b/lang/funcs/structs/exprif.go @@ -32,7 +32,6 @@ package structs import ( "context" "fmt" - "sync" "github.com/purpleidea/mgmt/lang/interfaces" "github.com/purpleidea/mgmt/lang/types" @@ -134,78 +133,6 @@ func (obj *ExprIfFunc) Init(init *interfaces.Init) error { return nil } -// Stream takes an input struct in the format as described in the Func and Graph -// methods of the Expr, and returns the actual expected value as a stream based -// on the changing inputs to that value. -func (obj *ExprIfFunc) Stream(ctx context.Context) error { - // XXX: is there a sync.Once sort of solution that would be more elegant here? - mutex := &sync.Mutex{} - done := false - send := func(ctx context.Context, b bool) error { - mutex.Lock() - defer mutex.Unlock() - if done { - return nil - } - done = true - defer close(obj.init.Output) // the sender closes - - if !b { - return nil - } - - // send dummy value to the output - select { - case obj.init.Output <- types.NewFloat(): // XXX: dummy value - case <-ctx.Done(): - return ctx.Err() - } - - return nil - } - defer send(ctx, false) // just close - - defer func() { - obj.init.Txn.Reverse() - }() - - for { - select { - case input, ok := <-obj.init.Input: - if !ok { - obj.init.Input = nil // block looping back here - if !done { - return fmt.Errorf("input closed without ever sending anything") - } - return nil - } - - value, exists := input.Struct()[obj.EdgeName] - if !exists { - return fmt.Errorf("programming error, can't find edge") - } - - b := value.Bool() - - if obj.last != nil && *obj.last == b { - continue // result didn't change - } - obj.last = &b // store new result - - if err := obj.replaceSubGraph(b); err != nil { - return errwrap.Wrapf(err, "could not replace subgraph") - } - - send(ctx, true) // send dummy and then close - - continue - - case <-ctx.Done(): - return nil - } - } -} - func (obj *ExprIfFunc) replaceSubGraph(b bool) error { // delete the old subgraph if err := obj.init.Txn.Reverse(); err != nil { @@ -229,3 +156,30 @@ func (obj *ExprIfFunc) replaceSubGraph(b bool) error { return obj.init.Txn.Commit() } + +// Call this func and return the value if it is possible to do so at this time. +func (obj *ExprIfFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) { + if len(args) < 1 { + return nil, fmt.Errorf("not enough args") + } + value := args[0] + b := value.Bool() + + if obj.last == nil || *obj.last != b { + obj.last = &b // store new result + + if err := obj.replaceSubGraph(b); err != nil { + return nil, errwrap.Wrapf(err, "could not replace subgraph") + } + + return nil, interfaces.ErrInterrupt + } + + // send dummy value to the output + return types.NewNil(), nil // dummy value +} + +// Cleanup runs after that function was removed from the graph. +func (obj *ExprIfFunc) Cleanup(ctx context.Context) error { + return obj.init.Txn.Reverse() +} diff --git a/lang/funcs/structs/for.go b/lang/funcs/structs/for.go index 6956505f..8504ce81 100644 --- a/lang/funcs/structs/for.go +++ b/lang/funcs/structs/for.go @@ -61,6 +61,8 @@ type ForFunc struct { AppendToIterBody func(innerTxn interfaces.Txn, index int, value interfaces.Func) error ClearIterBody func(length int) + ArgVertices []interfaces.Func // only one expected + init *interfaces.Init lastInputListLength int // remember the last input list length @@ -86,6 +88,10 @@ func (obj *ForFunc) Validate() error { return fmt.Errorf("must specify an edge name") } + if len(obj.ArgVertices) != 1 { + return fmt.Errorf("function did not receive shape information") + } + return nil } @@ -96,8 +102,8 @@ func (obj *ForFunc) Info() *interfaces.Info { if obj.IndexType != nil && obj.ValueType != nil { // don't panic if called speculatively // XXX: Improve function engine so it can return no value? //typ = types.NewType(fmt.Sprintf("func(%s []%s)", obj.EdgeName, obj.ValueType)) // returns nothing - // XXX: Temporary float type to prove we're dropping the output since we don't use it. - typ = types.NewType(fmt.Sprintf("func(%s []%s) float", obj.EdgeName, obj.ValueType)) + // dummy type to prove we're dropping the output since we don't use it. + typ = types.NewType(fmt.Sprintf("func(%s []%s) nil", obj.EdgeName, obj.ValueType)) } return &interfaces.Info{ @@ -112,100 +118,10 @@ func (obj *ForFunc) Info() *interfaces.Info { func (obj *ForFunc) Init(init *interfaces.Init) error { obj.init = init obj.lastInputListLength = -1 + return nil } -// Stream takes an input struct in the format as described in the Func and Graph -// methods of the Expr, and returns the actual expected value as a stream based -// on the changing inputs to that value. -func (obj *ForFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes - - // A Func to send input lists to the subgraph. The Txn.Erase() call - // ensures that this Func is not removed when the subgraph is recreated, - // so that the function graph can propagate the last list we received to - // the subgraph. - inputChan := make(chan types.Value) - subgraphInput := &ChannelBasedSourceFunc{ - Name: "subgraphInput", - Source: obj, - Chan: inputChan, - Type: obj.listType(), - } - obj.init.Txn.AddVertex(subgraphInput) - if err := obj.init.Txn.Commit(); err != nil { - return errwrap.Wrapf(err, "commit error in Stream") - } - obj.init.Txn.Erase() // prevent the next Reverse() from removing subgraphInput - defer func() { - close(inputChan) - obj.init.Txn.Reverse() - obj.init.Txn.DeleteVertex(subgraphInput) - obj.init.Txn.Commit() - }() - - for { - select { - case input, ok := <-obj.init.Input: - if !ok { - obj.init.Input = nil // block looping back here - //canReceiveMoreListValues = false - // We don't ever shutdown here, since even if we - // don't get more lists, that last list value is - // still propagating inside of the subgraph and - // so we don't want to shutdown since that would - // reverse the txn which we only do at the very - // end on graph shutdown. - continue - } - - forList, exists := input.Struct()[obj.EdgeName] - if !exists { - return fmt.Errorf("programming error, can't find edge") - } - - // If the length of the input list has changed, then we - // need to replace the subgraph with a new one that has - // that many "tentacles". Basically the shape of the - // graph depends on the length of the list. If we get a - // brand new list where each value is different, but - // the length is the same, then we can just flow new - // values into the list and we don't need to change the - // graph shape! Changing the graph shape is more - // expensive, so we don't do it when not necessary. - n := len(forList.List()) - - //if forList.Cmp(obj.lastForList) != nil // don't! - if n != obj.lastInputListLength { - //obj.lastForList = forList - obj.lastInputListLength = n - // replaceSubGraph uses the above two values - if err := obj.replaceSubGraph(subgraphInput); err != nil { - return errwrap.Wrapf(err, "could not replace subgraph") - } - } - - // send the new input list to the subgraph - select { - case inputChan <- forList: - case <-ctx.Done(): - return nil - } - - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- &types.FloatValue{ - V: 42.0, // XXX: temporary - }: - case <-ctx.Done(): - return nil - } - } -} - func (obj *ForFunc) replaceSubGraph(subgraphInput interfaces.Func) error { // delete the old subgraph if err := obj.init.Txn.Reverse(); err != nil { @@ -254,3 +170,41 @@ func (obj *ForFunc) replaceSubGraph(subgraphInput interfaces.Func) error { func (obj *ForFunc) listType() *types.Type { return types.NewType(fmt.Sprintf("[]%s", obj.ValueType)) } + +// Call this function with the input args and return the value if it is possible +// to do so at this time. +func (obj *ForFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) { + if len(args) < 1 { + return nil, fmt.Errorf("not enough args") + } + forList := args[0] + n := len(forList.List()) + + // If the length of the input list has changed, then we need to replace + // the subgraph with a new one that has that many "tentacles". Basically + // the shape of the graph depends on the length of the list. If we get a + // brand new list where each value is different, but the length is the + // same, then we can just flow new values into the list and we don't + // need to change the graph shape! Changing the graph shape is more + // expensive, so we don't do it when not necessary. + if n != obj.lastInputListLength { + subgraphInput := obj.ArgVertices[0] + + //obj.lastForList = forList + obj.lastInputListLength = n + // replaceSubGraph uses the above two values + if err := obj.replaceSubGraph(subgraphInput); err != nil { + return nil, errwrap.Wrapf(err, "could not replace subgraph") + } + + return nil, interfaces.ErrInterrupt + } + + // send dummy value to the output + return types.NewNil(), nil // dummy value +} + +// Cleanup runs after that function was removed from the graph. +func (obj *ForFunc) Cleanup(ctx context.Context) error { + return obj.init.Txn.Reverse() +} diff --git a/lang/funcs/structs/forkv.go b/lang/funcs/structs/forkv.go index 88dfc698..d132d585 100644 --- a/lang/funcs/structs/forkv.go +++ b/lang/funcs/structs/forkv.go @@ -61,6 +61,8 @@ type ForKVFunc struct { SetOnIterBody func(innerTxn interfaces.Txn, ptr types.Value, key, val interfaces.Func) error ClearIterBody func(length int) + ArgVertices []interfaces.Func // only one expected + init *interfaces.Init lastForKVMap types.Value // remember the last value @@ -87,6 +89,10 @@ func (obj *ForKVFunc) Validate() error { return fmt.Errorf("must specify an edge name") } + if len(obj.ArgVertices) != 1 { + return fmt.Errorf("function did not receive shape information") + } + return nil } @@ -97,8 +103,8 @@ func (obj *ForKVFunc) Info() *interfaces.Info { if obj.KeyType != nil && obj.ValType != nil { // don't panic if called speculatively // XXX: Improve function engine so it can return no value? //typ = types.NewType(fmt.Sprintf("func(%s map{%s: %s})", obj.EdgeName, obj.KeyType, obj.ValType)) // returns nothing - // XXX: Temporary float type to prove we're dropping the output since we don't use it. - typ = types.NewType(fmt.Sprintf("func(%s map{%s: %s}) float", obj.EdgeName, obj.KeyType, obj.ValType)) + // dummy type to prove we're dropping the output since we don't use it. + typ = types.NewType(fmt.Sprintf("func(%s map{%s: %s}) nil", obj.EdgeName, obj.KeyType, obj.ValType)) } return &interfaces.Info{ @@ -117,102 +123,6 @@ func (obj *ForKVFunc) Init(init *interfaces.Init) error { return nil } -// Stream takes an input struct in the format as described in the Func and Graph -// methods of the Expr, and returns the actual expected value as a stream based -// on the changing inputs to that value. -func (obj *ForKVFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes - - // A Func to send input maps to the subgraph. The Txn.Erase() call - // ensures that this Func is not removed when the subgraph is recreated, - // so that the function graph can propagate the last map we received to - // the subgraph. - inputChan := make(chan types.Value) - subgraphInput := &ChannelBasedSourceFunc{ - Name: "subgraphInput", - Source: obj, - Chan: inputChan, - Type: obj.mapType(), - } - obj.init.Txn.AddVertex(subgraphInput) - if err := obj.init.Txn.Commit(); err != nil { - return errwrap.Wrapf(err, "commit error in Stream") - } - obj.init.Txn.Erase() // prevent the next Reverse() from removing subgraphInput - defer func() { - close(inputChan) - obj.init.Txn.Reverse() - obj.init.Txn.DeleteVertex(subgraphInput) - obj.init.Txn.Commit() - }() - - for { - select { - case input, ok := <-obj.init.Input: - if !ok { - obj.init.Input = nil // block looping back here - //canReceiveMoreMapValues = false - // We don't ever shutdown here, since even if we - // don't get more maps, that last map value is - // still propagating inside of the subgraph and - // so we don't want to shutdown since that would - // reverse the txn which we only do at the very - // end on graph shutdown. - continue - } - - forKVMap, exists := input.Struct()[obj.EdgeName] - if !exists { - return fmt.Errorf("programming error, can't find edge") - } - - // It's important to have this compare step to avoid - // redundant graph replacements which slow things down, - // but also cause the engine to lock, which can preempt - // the process scheduler, which can cause duplicate or - // unnecessary re-sending of values here, which causes - // the whole process to repeat ad-nauseum. - n := len(forKVMap.Map()) - - // If the keys are the same, that's enough! We don't - // need to rebuild the graph unless any of the keys - // change, since those are our unique identifiers into - // the whole loop. As a result, we don't compare between - // the entire two map, since while we could rebuild the - // graph on any change, it's easier to leave it as is - // and simply push new values down the already built - // graph if any value changes. - if obj.lastInputMapLength != n || obj.cmpMapKeys(forKVMap) != nil { - // TODO: Technically we only need to save keys! - obj.lastForKVMap = forKVMap - obj.lastInputMapLength = n - // replaceSubGraph uses the above two values - if err := obj.replaceSubGraph(subgraphInput); err != nil { - return errwrap.Wrapf(err, "could not replace subgraph") - } - } - - // send the new input map to the subgraph - select { - case inputChan <- forKVMap: - case <-ctx.Done(): - return nil - } - - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- &types.FloatValue{ - V: 42.0, // XXX: temporary - }: - case <-ctx.Done(): - return nil - } - } -} - func (obj *ForKVFunc) replaceSubGraph(subgraphInput interfaces.Func) error { // delete the old subgraph if err := obj.init.Txn.Reverse(); err != nil { @@ -314,3 +224,41 @@ func (obj *ForKVFunc) cmpMapKeys(m types.Value) error { return nil } + +// Call this function with the input args and return the value if it is possible +// to do so at this time. +func (obj *ForKVFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) { + if len(args) < 1 { + return nil, fmt.Errorf("not enough args") + } + forKVMap := args[0] + n := len(forKVMap.Map()) + + // If the keys are the same, that's enough! We don't need to rebuild the + // graph unless any of the keys/ change, since those are our unique + // identifiers into the whole loop. As a result, we don't compare + // between the entire two maps, since while we could rebuild the graph + // on any change, it's easier to leave it as is and simply push new + // values down the already built graph if any value changes. + if obj.lastInputMapLength != n || obj.cmpMapKeys(forKVMap) != nil { + subgraphInput := obj.ArgVertices[0] + + // TODO: Technically we only need to save keys! + obj.lastForKVMap = forKVMap + obj.lastInputMapLength = n + // replaceSubGraph uses the above two values + if err := obj.replaceSubGraph(subgraphInput); err != nil { + return nil, errwrap.Wrapf(err, "could not replace subgraph") + } + + return nil, interfaces.ErrInterrupt + } + + // send dummy value to the output + return types.NewNil(), nil // dummy value +} + +// Cleanup runs after that function was removed from the graph. +func (obj *ForKVFunc) Cleanup(ctx context.Context) error { + return obj.init.Txn.Reverse() +} diff --git a/lang/funcs/structs/output.go b/lang/funcs/structs/output.go index 4809b4ca..99fa5062 100644 --- a/lang/funcs/structs/output.go +++ b/lang/funcs/structs/output.go @@ -81,8 +81,8 @@ func (obj *OutputFunc) Validate() error { // Info returns some static info about itself. func (obj *OutputFunc) Info() *interfaces.Info { - // XXX: contains "dummy" return type - s := fmt.Sprintf("func(%s %s, %s float) %s", obj.EdgeName, obj.Type, OutputFuncDummyArgName, obj.Type) + // contains "dummy" return type + s := fmt.Sprintf("func(%s %s, %s nil) %s", obj.EdgeName, obj.Type, OutputFuncDummyArgName, obj.Type) return &interfaces.Info{ Pure: false, Memo: false, @@ -97,44 +97,6 @@ func (obj *OutputFunc) Init(init *interfaces.Init) error { return nil } -// Stream returns the changing values that this func has over time. -func (obj *OutputFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes - - for { - select { - case input, ok := <-obj.init.Input: - if !ok { - return nil // can't output any more - } - - args, err := interfaces.StructToCallableArgs(input) // []types.Value, error) - if err != nil { - return err - } - - result, err := obj.Call(ctx, args) // get the value... - if err != nil { - return err - } - - if obj.last != nil && result.Cmp(obj.last) == nil { - continue // value didn't change, skip it - } - obj.last = result // store so we can send after this select - - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- obj.last: // send - case <-ctx.Done(): - return nil - } - } -} - // Call this function with the input args and return the value if it is possible // to do so at this time. func (obj *OutputFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) { diff --git a/lang/funcs/structs/stmtif.go b/lang/funcs/structs/stmtif.go new file mode 100644 index 00000000..03180d14 --- /dev/null +++ b/lang/funcs/structs/stmtif.go @@ -0,0 +1,163 @@ +// Mgmt +// Copyright (C) James Shubin and the project contributors +// Written by James Shubin and the project contributors +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . +// +// Additional permission under GNU GPL version 3 section 7 +// +// If you modify this program, or any covered work, by linking or combining it +// with embedded mcl code and modules (and that the embedded mcl code and +// modules which link with this program, contain a copy of their source code in +// the authoritative form) containing parts covered by the terms of any other +// license, the licensors of this program grant you additional permission to +// convey the resulting work. Furthermore, the licensors of this program grant +// the original author, James Shubin, additional permission to update this +// additional permission if he deems it necessary to achieve the goals of this +// additional permission. + +package structs + +import ( + "context" + "fmt" + + "github.com/purpleidea/mgmt/lang/interfaces" + "github.com/purpleidea/mgmt/lang/types" + "github.com/purpleidea/mgmt/util/errwrap" +) + +const ( + // StmtIfFuncName is the unique name identifier for this function. + StmtIfFuncName = "stmtif" + + // StmtIfFuncArgNameCondition is the name for the edge which connects + // the input condition to StmtIfFunc. + StmtIfFuncArgNameCondition = "condition" +) + +// StmtIfFunc is a function that builds the correct body based on the +// conditional value it gets. +type StmtIfFunc struct { + interfaces.Textarea + + EdgeName string // name of the edge used + + // Env is the captured environment from when Graph for StmtIf was built. + Env *interfaces.Env + + // Then is the Stmt for the "then" branch. We do *not* want this to be a + // *pgraph.Graph, as we actually need to call Graph(env) ourself during + // runtime to get the correct subgraph out of that appropriate branch. + Then interfaces.Stmt + + // Else is the Stmt for the "else" branch. See "Then" for more details. + Else interfaces.Stmt + + init *interfaces.Init + last *bool // last value received to use for diff + needsReverse bool +} + +// String returns a simple name for this function. This is needed so this struct +// can satisfy the pgraph.Vertex interface. +func (obj *StmtIfFunc) String() string { + return StmtIfFuncName +} + +// Validate tells us if the input struct takes a valid form. +func (obj *StmtIfFunc) Validate() error { + if obj.EdgeName == "" { + return fmt.Errorf("must specify an edge name") + } + + return nil +} + +// Info returns some static info about itself. +func (obj *StmtIfFunc) Info() *interfaces.Info { + // dummy type to prove we're dropping the output since we don't use it. + typ := types.NewType(fmt.Sprintf("func(%s bool) nil", obj.EdgeName)) + + return &interfaces.Info{ + Pure: true, + Memo: false, // TODO: ??? + Sig: typ, + Err: obj.Validate(), + } +} + +// Init runs some startup code for this if statement function. +func (obj *StmtIfFunc) Init(init *interfaces.Init) error { + obj.init = init + return nil +} + +func (obj *StmtIfFunc) replaceSubGraph(b bool) error { + if obj.needsReverse { // not on the first run + // delete the old subgraph + if err := obj.init.Txn.Reverse(); err != nil { + return errwrap.Wrapf(err, "could not Reverse") + } + } + obj.needsReverse = true + + if b && obj.Then != nil { + g, err := obj.Then.Graph(obj.Env) + if err != nil { + return err + } + obj.init.Txn.AddGraph(g) + } + if !b && obj.Else != nil { + g, err := obj.Else.Graph(obj.Env) + if err != nil { + return err + } + obj.init.Txn.AddGraph(g) + } + + return obj.init.Txn.Commit() +} + +// Call this func and return the value if it is possible to do so at this time. +func (obj *StmtIfFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) { + if len(args) < 1 { + return nil, fmt.Errorf("not enough args") + } + value := args[0] + b := value.Bool() + + if obj.last == nil || *obj.last != b { + obj.last = &b // store new result + + if err := obj.replaceSubGraph(b); err != nil { + return nil, errwrap.Wrapf(err, "could not replace subgraph") + } + + return nil, interfaces.ErrInterrupt + } + + // send dummy value to the output + return types.NewNil(), nil // dummy value +} + +// Cleanup runs after that function was removed from the graph. +func (obj *StmtIfFunc) Cleanup(ctx context.Context) error { + if !obj.needsReverse { // not needed if we never replaced graph + return nil + } + + return obj.init.Txn.Reverse() +} diff --git a/lang/funcs/structs/util.go b/lang/funcs/structs/util.go index 91b4f170..9ecf3e77 100644 --- a/lang/funcs/structs/util.go +++ b/lang/funcs/structs/util.go @@ -70,7 +70,7 @@ func SimpleFnToDirectFunc(name string, fv *types.FuncValue) interfaces.Func { // *full.FuncValue. func SimpleFnToFuncValue(name string, fv *types.FuncValue) *full.FuncValue { return &full.FuncValue{ - V: func(txn interfaces.Txn, args []interfaces.Func) (interfaces.Func, error) { + V: func(txn interfaces.Txn, args []interfaces.Func, out interfaces.Func) (interfaces.Func, error) { wrappedFunc := SimpleFnToDirectFunc(name, fv) txn.AddVertex(wrappedFunc) for i, arg := range args { @@ -79,6 +79,13 @@ func SimpleFnToFuncValue(name string, fv *types.FuncValue) *full.FuncValue { Args: []string{argName}, }) } + + // XXX: do we need to use the `out` arg here? + // XXX: eg: via .SetShape(args, out) + //if shapelyFunc, ok := wrappedFunc.(interfaces.ShapelyFunc); ok { + // shapelyFunc.SetShape(args, out) + //} + return wrappedFunc, nil }, F: nil, // unused @@ -98,16 +105,22 @@ func SimpleFnToConstFunc(name string, fv *types.FuncValue) interfaces.Func { // only be called once. func FuncToFullFuncValue(makeFunc func() interfaces.Func, typ *types.Type) *full.FuncValue { - v := func(txn interfaces.Txn, args []interfaces.Func) (interfaces.Func, error) { + v := func(txn interfaces.Txn, args []interfaces.Func, out interfaces.Func) (interfaces.Func, error) { valueTransformingFunc := makeFunc() // do this once here - buildableFunc, ok := valueTransformingFunc.(interfaces.BuildableFunc) - if ok { + if buildableFunc, ok := valueTransformingFunc.(interfaces.BuildableFunc); ok { // Set the type in case it's not already done. if _, err := buildableFunc.Build(typ); err != nil { // programming error? return nil, err } } + + // XXX: is this the best way to pass this stuff in? + // XXX: do we even want to do this here? is it redundant or bad? + if shapelyFunc, ok := valueTransformingFunc.(interfaces.ShapelyFunc); ok { + shapelyFunc.SetShape(args, out) + } + for i, arg := range args { argName := typ.Ord[i] txn.AddEdge(arg, valueTransformingFunc, &interfaces.FuncEdge{ @@ -118,10 +131,10 @@ func FuncToFullFuncValue(makeFunc func() interfaces.Func, typ *types.Type) *full } var f interfaces.FuncSig - callableFunc, ok := makeFunc().(interfaces.CallableFunc) - if ok { - f = callableFunc.Call - } + fn := makeFunc() + //if _, ok := fn.(interfaces.StreamableFunc); !ok { // XXX: is this what we want now? + f = fn.Call + //} // This has the "V" implementation and the simpler "F" implementation // which can occasionally be used if the interfaces.Func supports that! diff --git a/lang/funcs/txn/txn.go b/lang/funcs/txn/txn.go index 3cd0f01e..750a4ced 100644 --- a/lang/funcs/txn/txn.go +++ b/lang/funcs/txn/txn.go @@ -277,6 +277,11 @@ func (obj *opDeleteVertex) String() string { // functions in their Stream method to modify the function graph while it is // "running". type GraphTxn struct { + + // Post runs some effects after the Commit has run succesfully, but + // before it exits. + Post func( /* deltaOps */ ) error + // Lock is a handle to the lock function to call before the operation. Lock func() @@ -313,6 +318,16 @@ func (obj *GraphTxn) Init() interfaces.Txn { obj.rev = []opfn{} obj.mutex = &sync.Mutex{} + if obj.Post == nil { + panic("the Post function is nil") + } + if obj.Lock == nil { + panic("the Lock function is nil") + } + if obj.Unlock == nil { + panic("the Unlock function is nil") + } + return obj // return self so it can be called in a chain } @@ -322,6 +337,7 @@ func (obj *GraphTxn) Init() interfaces.Txn { // TODO: FreeFunc isn't well supported here. Replace or remove this entirely? func (obj *GraphTxn) Copy() interfaces.Txn { txn := &GraphTxn{ + Post: obj.Post, Lock: obj.Lock, Unlock: obj.Unlock, GraphAPI: obj.GraphAPI, @@ -403,7 +419,7 @@ func (obj *GraphTxn) AddGraph(g *pgraph.Graph) interfaces.Txn { for _, v := range g.Vertices() { f, ok := v.(interfaces.Func) if !ok { - panic("not a Func") + panic("not a Func") // XXX: why does this panic not appear until we ^C ? } //obj.AddVertex(f) // easy opfn := &opAddVertex{ // replicate AddVertex @@ -505,6 +521,7 @@ func (obj *GraphTxn) commit() error { //obj.rev = append([]opfn{op}, obj.rev...) // add to front } } + //deltaOps := obj.ops // copy for delta graph effects obj.ops = []opfn{} // clear it // garbage collect anything that hit zero! @@ -514,6 +531,13 @@ func (obj *GraphTxn) commit() error { return err } + // This runs after all the operations have been completely successfully. + // XXX: We'd like to pass in the graph delta information, so that we can + // more efficiently recompute the topological sort and so on... + if err := obj.Post( /* deltaOps */ ); err != nil { + return err + } + // XXX: running this on each commit has a huge performance hit. // XXX: we could write out the .dot files and run graphviz afterwards if g, ok := obj.GraphAPI.(pgraph.Graphvizable); ok && GraphvizDebug { diff --git a/lang/funcs/txn/txn_test.go b/lang/funcs/txn/txn_test.go index ff6a05b5..e38a318d 100644 --- a/lang/funcs/txn/txn_test.go +++ b/lang/funcs/txn/txn_test.go @@ -39,6 +39,7 @@ import ( "github.com/purpleidea/mgmt/lang/funcs/ref" "github.com/purpleidea/mgmt/lang/interfaces" + "github.com/purpleidea/mgmt/lang/types" "github.com/purpleidea/mgmt/pgraph" "github.com/purpleidea/mgmt/util" ) @@ -136,11 +137,11 @@ type testNullFunc struct { name string } -func (obj *testNullFunc) String() string { return obj.name } -func (obj *testNullFunc) Info() *interfaces.Info { return nil } -func (obj *testNullFunc) Validate() error { return nil } -func (obj *testNullFunc) Init(*interfaces.Init) error { return nil } -func (obj *testNullFunc) Stream(context.Context) error { return nil } +func (obj *testNullFunc) String() string { return obj.name } +func (obj *testNullFunc) Info() *interfaces.Info { return nil } +func (obj *testNullFunc) Validate() error { return nil } +func (obj *testNullFunc) Init(*interfaces.Init) error { return nil } +func (obj *testNullFunc) Call(context.Context, []types.Value) (types.Value, error) { return nil, nil } func TestTxn1(t *testing.T) { graph, err := pgraph.NewGraph("test") @@ -153,6 +154,7 @@ func TestTxn1(t *testing.T) { graphTxn := &GraphTxn{ GraphAPI: testGraphAPI, + Post: func() error { return nil }, Lock: mutex.Lock, Unlock: mutex.Unlock, RefCount: (&ref.Count{}).Init(), @@ -496,6 +498,7 @@ func TestTxnTable(t *testing.T) { graphTxn := &GraphTxn{ GraphAPI: testGraphAPI, + Post: func() error { return nil }, Lock: mutex.Lock, Unlock: mutex.Unlock, RefCount: (&ref.Count{}).Init(), diff --git a/lang/funcs/wrapped/wrapped.go b/lang/funcs/wrapped/wrapped.go index 79a2796c..21f145bc 100644 --- a/lang/funcs/wrapped/wrapped.go +++ b/lang/funcs/wrapped/wrapped.go @@ -147,69 +147,6 @@ func (obj *Func) Init(init *interfaces.Init) error { return nil } -// Stream returns the changing values that this func has over time. -func (obj *Func) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes - for { - select { - case input, ok := <-obj.init.Input: - if !ok { - if len(obj.Fn.Type().Ord) > 0 { - return nil // can't output any more - } - // no inputs were expected, pass through once - } - if ok { - //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { - // return errwrap.Wrapf(err, "wrong function input") - //} - - if obj.last != nil && input.Cmp(obj.last) == nil { - continue // value didn't change, skip it - } - obj.last = input // store for next - } - - args, err := interfaces.StructToCallableArgs(input) // []types.Value, error) - if err != nil { - return err - } - - if obj.init.Debug { - obj.init.Logf("Calling function with: %+v", args) - } - result, err := obj.Call(ctx, args) // (Value, error) - if err != nil { - if obj.init.Debug { - obj.init.Logf("Function returned error: %+v", err) - } - return err - } - if obj.init.Debug { - obj.init.Logf("Function returned with: %+v", result) - } - - // TODO: do we want obj.result to be a pointer instead? - if obj.result == result { - continue // result didn't change - } - obj.result = result // store new result - - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- obj.result: // send - if len(obj.Fn.Type().Ord) == 0 { - return nil // no more values, we're a pure func - } - case <-ctx.Done(): - return nil - } - } -} - // Call this function with the input args and return the value if it is possible // to do so at this time. func (obj *Func) Call(ctx context.Context, args []types.Value) (types.Value, error) { diff --git a/lang/gapi/gapi.go b/lang/gapi/gapi.go index 9b4992c0..71b4ac02 100644 --- a/lang/gapi/gapi.go +++ b/lang/gapi/gapi.go @@ -87,6 +87,7 @@ type GAPI struct { initialized bool wg *sync.WaitGroup // sync group for tunnel go routines err error + errMutex *sync.Mutex // guards err } // Cli takes an *Info struct, and returns our deploy if activated, and if there @@ -511,6 +512,7 @@ func (obj *GAPI) Init(data *gapi.Data) error { } obj.data = data // store for later obj.wg = &sync.WaitGroup{} + obj.errMutex = &sync.Mutex{} obj.initialized = true if obj.InputURI == "-" { @@ -564,9 +566,11 @@ func (obj *GAPI) Next(ctx context.Context) chan gapi.Next { defer obj.wg.Wait() // wait before cleanup defer obj.wg.Done() err := obj.lang.Run(ctx) - // XXX: Temporary extra logging for catching bugs! - obj.data.Logf(Name+": %+v", err) - obj.err = err + obj.errAppend(err) + // When run terminates, the "official" error takes precedence. + if err := obj.lang.Err(); err != nil { + obj.errAppend(err) + } }() obj.wg.Add(1) @@ -581,7 +585,7 @@ func (obj *GAPI) Next(ctx context.Context) chan gapi.Next { select { case ch <- next: case <-ctx.Done(): - obj.err = ctx.Err() + obj.errAppend(ctx.Err()) } return } @@ -598,7 +602,7 @@ func (obj *GAPI) Next(ctx context.Context) chan gapi.Next { } case <-ctx.Done(): - obj.err = ctx.Err() + obj.errAppend(ctx.Err()) return } obj.data.Logf("generating new graph...") @@ -611,7 +615,7 @@ func (obj *GAPI) Next(ctx context.Context) chan gapi.Next { // unblock if we exit while waiting to send! case <-ctx.Done(): - obj.err = ctx.Err() + obj.errAppend(ctx.Err()) return } } @@ -625,3 +629,10 @@ func (obj *GAPI) Err() error { obj.wg.Wait() return obj.err } + +// errAppend is a simple helper function. +func (obj *GAPI) errAppend(err error) { + obj.errMutex.Lock() + obj.err = errwrap.Append(obj.err, err) + obj.errMutex.Unlock() +} diff --git a/lang/interfaces/error.go b/lang/interfaces/error.go index 34e7e391..5fdb6c45 100644 --- a/lang/interfaces/error.go +++ b/lang/interfaces/error.go @@ -50,4 +50,9 @@ const ( // import is missing. This might signal the downloader, or it might // signal a permanent error. ErrExpectedFileMissing = util.Error("file is currently missing") + + // ErrInterrupt is returned when a function can't immediately return a + // value to the function engine because a graph change transaction needs + // to run. + ErrInterrupt = util.Error("function call interrupted") ) diff --git a/lang/interfaces/func.go b/lang/interfaces/func.go index 1f166810..e8b1bd5d 100644 --- a/lang/interfaces/func.go +++ b/lang/interfaces/func.go @@ -60,7 +60,7 @@ func (obj Table) Copy() Table { // GraphSig is the simple signature that is used throughout our implementations. // TODO: Rename this? -type GraphSig = func(Txn, []Func) (Func, error) +type GraphSig = func(txn Txn, args []Func, out Func) (Func, error) // Compile-time guarantee that *types.FuncValue accepts a func of type FuncSig. var _ = &types.FuncValue{V: FuncSig(nil)} @@ -73,6 +73,7 @@ type Info struct { Memo bool // should the function be memoized? (false if too much output) Fast bool // is the function slow? (avoid speculative execution) Spec bool // can we speculatively execute it? (true for most) + Meld bool // should this function be a singleton? eg: datetime.now() Sig *types.Type // the signature of the function, must be KindFunc Err error // is this a valid function, or was it created improperly? } @@ -83,13 +84,10 @@ type Init struct { Hostname string // uuid for the host //Noop bool - // Input is where a chan (stream) of values will get sent to this node. - // The engine will close this `input` chan. - Input chan types.Value - - // Output is the chan (stream) of values to get sent out from this node. - // The Stream function must close this `output` chan. - Output chan types.Value + // Event is available and must be called anytime a StreamableFunc wishes + // to alert the engine of a new event. + // XXX: Why isn't this just an argument to Stream() ? + Event func(ctx context.Context) error // Txn provides a transaction API that can be used to modify the // function graph while it is "running". This should not be used by most @@ -105,11 +103,10 @@ type Init struct { Logf func(format string, v ...interface{}) } -// Func is the interface that any valid func must fulfill. It is very simple, -// but still event driven. Funcs should attempt to only send values when they -// have changed. -// TODO: should we support a static version of this interface for funcs that -// never change to avoid the overhead of the goroutine and channel listener? +// Func is the interface that any valid func must fulfill. Most functions will +// implement this. Functions that require more functionality can implement +// additional methods that will enable those functions to fulfill some of the +// additional interfaces below. type Func interface { fmt.Stringer // so that this can be stored as a Vertex @@ -128,18 +125,69 @@ type Func interface { // Init passes some important values and references to the function. Init(*Init) error - // Stream is the mainloop of the function. It reads and writes from - // channels to return the changing values that this func has over time. - // It should shutdown and cleanup when the input context is cancelled. - // It must not exit before any goroutines it spawned have terminated. - // It must close the Output chan if it's done sending new values out. It - // must send at least one value, or return an error. It may also return - // an error at anytime if it can't continue. - // XXX: Remove this from here, it should appear as StreamableFunc and - // funcs should implement StreamableFunc or CallableFunc or maybe both. + // Call this function with the input args and return the value if it is + // possible to do so at this time. To transform from the single value, + // graph representation of the callable values into a linear, standard + // args list for use here, you can use the StructToCallableArgs + // function. + // + // This may be called speculatively. If the function cannot accept a + // speculative call at that time, it must return funcs.ErrCantSpeculate. + // + // XXX: Should we have another custom error for conditions such as + // "not enough args" and other programming mistakes? + Call(ctx context.Context, args []types.Value) (types.Value, error) +} + +// StreamableFunc is a function which runs a mainloop. The mainloop can run an +// Event() method to tell the function engine when it thinks a new Call() should +// be run. +type StreamableFunc interface { + Func // implement everything in Func but add the additional requirements + + // Stream is the mainloop of the function. It calls Event() when it + // wants to notify the function engine of a new value. That new value + // can be obtained by running Call(). If this function errors, then the + // whole function engine must shutdown. It's legal for the Call() to + // error, without the Stream exiting. This Stream function should + // shutdown and cleanup when the input context is cancelled. Stream(context.Context) error } +// CleanableFunc is an interface for functions that might have some cleanup to +// run after they have been removed from the graph. It's usually useful for +// executing cleanup transactions. +type CleanableFunc interface { + Func // implement everything in Func but add the additional requirements + + // Cleanup runs after that function was removed from the graph. + // + // Cleanup is essential to remove anything from the function graph that + // we're done with when this function (node) exits. Historically this + // equivalent was done when Stream exited, however it's important that + // any Txn operations like Txn.Reverse happen here (or in Call) and not + // in Stream because (1) there may not be a Stream function for all + // Func's, and (2) it needs to happen synchronously when we expect it to + // and not where it race against something else, or where it could + // deadlock by having nested Txn's. + // + // If the Done method exists, that will run before this Cleanup does. + Cleanup(context.Context) error +} + +// DoneableFunc is an interface for functions that tells them that Call will +// never get called again. It's useful so that they can choose to shutdown the +// Stream or free any other memory. This happens if the engine knows the +// incoming values won't change any more. +type DoneableFunc interface { + Func // implement everything in Func but add the additional requirements + + // Done tells the function that it will never be called again. This will + // get called a maximum of once, and before Cleanup is called if it is + // present. + Done() error // TODO: Should we return an error? +} + // BuildableFunc is an interface for functions which need a Build or Check step. // These functions need that method called after type unification to either tell // them the precise type, and/or Check if it's a valid solution. These functions @@ -222,19 +270,6 @@ type InferableFunc interface { // TODO: Is there a better name for this? FuncInfer(partialType *types.Type, partialValues []types.Value) (*types.Type, []*UnificationInvariant, error) } -// CallableFunc is a function that can be called statically if we want to do it -// speculatively or from a resource. -type CallableFunc interface { - Func // implement everything in Func but add the additional requirements - - // Call this function with the input args and return the value if it is - // possible to do so at this time. To transform from the single value, - // graph representation of the callable values into a linear, standard - // args list for use here, you can use the StructToCallableArgs - // function. - Call(ctx context.Context, args []types.Value) (types.Value, error) -} - // CopyableFunc is an interface which extends the base Func interface with the // ability to let our compiler know how to copy a Func if that func deems it's // needed to be able to do so. @@ -302,6 +337,19 @@ type DataFunc interface { SetData(*FuncData) } +// ShapelyFunc is a function that might require some additional graph shape +// information to know where to get and output it's data from. This is typically +// only implemented by tricky functions like "iter.map" and internal functions. +type ShapelyFunc interface { + Func // implement everything in Func but add the additional requirements + + // SetShape tells the function about some important pointers. It is + // called during function graph building typically, and before the + // function engine runs this function. IOW it runs before the Init() of + // this function. + SetShape(argFuncs []Func, outputFunc Func) +} + // MetadataFunc is a function that can return some extraneous information about // itself, which is usually used for documentation generation and so on. type MetadataFunc interface { diff --git a/lang/interpret/interpret.go b/lang/interpret/interpret.go index 2cd61fc9..427c2cfe 100644 --- a/lang/interpret/interpret.go +++ b/lang/interpret/interpret.go @@ -73,7 +73,9 @@ type Interpreter struct { // requires an AST, and the table of values required to populate that AST. Type // unification, and earlier steps should obviously be run first so that you can // actually get a useful resource graph out of this instead of an error! +// XXX: add a ctx? func (obj *Interpreter) Interpret(ast interfaces.Stmt, table interfaces.Table) (*pgraph.Graph, error) { + obj.Logf("interpreting...") // build the kind,name -> res mapping obj.lookup = make(map[engine.ResPtrUID]engine.Res) diff --git a/lang/interpret_test.go b/lang/interpret_test.go index 55dd837b..b15c68eb 100644 --- a/lang/interpret_test.go +++ b/lang/interpret_test.go @@ -1259,7 +1259,7 @@ func TestAstFunc2(t *testing.T) { t.Errorf("test #%d: init error with func engine: %+v", index, err) return } - defer funcs.Cleanup() + //defer funcs.Cleanup() // XXX: can we type check things somehow? //logf("function engine validating...") @@ -1269,52 +1269,6 @@ func TestAstFunc2(t *testing.T) { // return //} - logf("function engine starting...") - wg := &sync.WaitGroup{} - defer wg.Wait() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - wg.Add(1) - go func() { - defer wg.Done() - if err := funcs.Run(ctx); err != nil { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: run error with func engine: %+v", index, err) - return - } - }() - - //wg.Add(1) - //go func() { // XXX: debugging - // defer wg.Done() - // for { - // select { - // case <-time.After(100 * time.Millisecond): // blocked functions - // t.Logf("test #%d: graphviz...", index) - // funcs.Graphviz("") // log to /tmp/... - // - // case <-ctx.Done(): - // return - // } - // } - //}() - - <-funcs.Started() // wait for startup (will not block forever) - - // Sanity checks for graph size. - if count := funcs.NumVertices(); count != 0 { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: expected empty graph on start, got %d vertices", index, count) - } - defer func() { - if count := funcs.NumVertices(); count != 0 { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: expected empty graph on exit, got %d vertices", index, count) - } - }() - defer wg.Wait() - defer cancel() - txn := funcs.Txn() defer txn.Free() // remember to call Free() txn.AddGraph(fgraph) @@ -1325,6 +1279,70 @@ func TestAstFunc2(t *testing.T) { } defer txn.Reverse() // should remove everything we added + logf("function engine starting...") + wg := &sync.WaitGroup{} + defer wg.Wait() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + wg.Add(1) + go func() { + defer wg.Done() + if err := funcs.Run(ctx); err != nil { + if err == context.Canceled { // normal test shutdown + return + } + // check in funcs.Err() instead. + //t.Errorf("test #%d: FAIL", index) + //t.Errorf("test #%d: run error with func engine: %+v", index, err) + } + }() + defer func() { + err := errwrap.WithoutContext(funcs.Err()) + if err == context.Canceled { // normal test shutdown + return + } + if err == nil { + return + } + + if (!fail || !failStream) && err != nil { + t.Errorf("test #%d: FAIL", index) + t.Errorf("test #%d: stream errored: %+v", index, err) + return + } + if failStream && err != nil { + t.Logf("test #%d: stream errored: %+v", index, err) + // Stream errors often have pointers in them, so don't compare for now. + //s := err.Error() // convert to string + //if !foundErr(s) { + // t.Errorf("test #%d: FAIL", index) + // t.Errorf("test #%d: expected different error", index) + // t.Logf("test #%d: err: %s", index, s) + // t.Logf("test #%d: exp: %s", index, expstr) + //} + return + } + if failStream && err == nil { + t.Errorf("test #%d: FAIL", index) + t.Errorf("test #%d: stream passed, expected fail", index) + return + } + }() + + // Sanity checks for graph size. + //if count := funcs.NumVertices(); count != 0 { + // t.Errorf("test #%d: FAIL", index) + // t.Errorf("test #%d: expected empty graph on start, got %d vertices", index, count) + //} + //defer func() { + // if count := funcs.NumVertices(); count != 0 { + // t.Errorf("test #%d: FAIL", index) + // t.Errorf("test #%d: expected empty graph on exit, got %d vertices", index, count) + // } + //}() + defer wg.Wait() + defer cancel() + isEmpty := make(chan struct{}) if fgraph.NumVertices() == 0 { // no funcs to load! close(isEmpty) @@ -1332,64 +1350,24 @@ func TestAstFunc2(t *testing.T) { // wait for some activity logf("stream...") - stream := funcs.Stream() - //select { - //case err, ok := <-stream: - // if !ok { - // t.Errorf("test #%d: FAIL", index) - // t.Errorf("test #%d: stream closed", index) - // return - // } - // if err != nil { - // t.Errorf("test #%d: FAIL", index) - // t.Errorf("test #%d: stream errored: %+v", index, err) - // return - // } - // - //case <-time.After(60 * time.Second): // blocked functions - // t.Errorf("test #%d: FAIL", index) - // t.Errorf("test #%d: stream timeout", index) - // return - //} + tableChan := funcs.Stream() // sometimes the <-stream seems to constantly (or for a // long time?) win the races against the <-time.After(), // so add some limit to how many times we need to stream max := 1 + var table interfaces.Table Loop: for { + var ok bool select { - case err, ok := <-stream: + case table, ok = <-tableChan: if !ok { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: stream closed", index) - return - } - if err != nil { - if (!fail || !failStream) && err != nil { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: stream errored: %+v", index, err) - return - } - if failStream && err != nil { - t.Logf("test #%d: stream errored: %+v", index, err) - // Stream errors often have pointers in them, so don't compare for now. - //s := err.Error() // convert to string - //if !foundErr(s) { - // t.Errorf("test #%d: FAIL", index) - // t.Errorf("test #%d: expected different error", index) - // t.Logf("test #%d: err: %s", index, s) - // t.Logf("test #%d: exp: %s", index, expstr) - //} - return - } - if failStream && err == nil { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: stream passed, expected fail", index) - return - } + //t.Errorf("test #%d: FAIL", index) // check in funcs.Err() instead. + t.Logf("test #%d: stream closed", index) return } + t.Logf("test #%d: got stream event!", index) max-- if max == 0 { @@ -1410,11 +1388,9 @@ func TestAstFunc2(t *testing.T) { } } - t.Logf("test #%d: %s", index, funcs.Stats()) + //t.Logf("test #%d: %s", index, funcs.Stats()) // run interpret! - table := funcs.Table() // map[interfaces.Func]types.Value - interpreter := &interpret.Interpreter{ Debug: testing.Verbose(), // set via the -test.v flag to `go test` Logf: func(format string, v ...interface{}) { @@ -2138,7 +2114,7 @@ func TestAstFunc3(t *testing.T) { t.Errorf("test #%d: init error with func engine: %+v", index, err) return } - defer funcs.Cleanup() + //defer funcs.Cleanup() // XXX: can we type check things somehow? //logf("function engine validating...") @@ -2148,52 +2124,6 @@ func TestAstFunc3(t *testing.T) { // return //} - logf("function engine starting...") - wg := &sync.WaitGroup{} - defer wg.Wait() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - wg.Add(1) - go func() { - defer wg.Done() - if err := funcs.Run(ctx); err != nil { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: run error with func engine: %+v", index, err) - return - } - }() - - //wg.Add(1) - //go func() { // XXX: debugging - // defer wg.Done() - // for { - // select { - // case <-time.After(100 * time.Millisecond): // blocked functions - // t.Logf("test #%d: graphviz...", index) - // funcs.Graphviz("") // log to /tmp/... - // - // case <-ctx.Done(): - // return - // } - // } - //}() - - <-funcs.Started() // wait for startup (will not block forever) - - // Sanity checks for graph size. - if count := funcs.NumVertices(); count != 0 { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: expected empty graph on start, got %d vertices", index, count) - } - defer func() { - if count := funcs.NumVertices(); count != 0 { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: expected empty graph on exit, got %d vertices", index, count) - } - }() - defer wg.Wait() - defer cancel() - txn := funcs.Txn() defer txn.Free() // remember to call Free() txn.AddGraph(fgraph) @@ -2204,6 +2134,74 @@ func TestAstFunc3(t *testing.T) { } defer txn.Reverse() // should remove everything we added + logf("function engine starting...") + wg := &sync.WaitGroup{} + defer wg.Wait() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + wg.Add(1) + go func() { + defer wg.Done() + if err := funcs.Run(ctx); err != nil { + if err == context.Canceled { // normal test shutdown + return + } + // check in funcs.Err() instead. + //t.Errorf("test #%d: FAIL", index) + //t.Errorf("test #%d: run error with func engine: %+v", index, err) + } + }() + defer func() { + err := errwrap.WithoutContext(funcs.Err()) + if err == context.Canceled { // normal test shutdown + return + } + if err == nil { + return + } + + t.Errorf("test #%d: FAIL", index) + t.Errorf("test #%d: stream errored: %+v", index, err) + return + + //if (!fail || !failStream) && err != nil { + // t.Errorf("test #%d: FAIL", index) + // t.Errorf("test #%d: stream errored: %+v", index, err) + // return + //} + //if failStream && err != nil { + // t.Logf("test #%d: stream errored: %+v", index, err) + // // Stream errors often have pointers in them, so don't compare for now. + // //s := err.Error() // convert to string + // //if !foundErr(s) { + // // t.Errorf("test #%d: FAIL", index) + // // t.Errorf("test #%d: expected different error", index) + // // t.Logf("test #%d: err: %s", index, s) + // // t.Logf("test #%d: exp: %s", index, expstr) + // //} + // return + //} + //if failStream && err == nil { + // t.Errorf("test #%d: FAIL", index) + // t.Errorf("test #%d: stream passed, expected fail", index) + // return + //} + }() + + // Sanity checks for graph size. + //if count := funcs.NumVertices(); count != 0 { + // t.Errorf("test #%d: FAIL", index) + // t.Errorf("test #%d: expected empty graph on start, got %d vertices", index, count) + //} + //defer func() { + // if count := funcs.NumVertices(); count != 0 { + // t.Errorf("test #%d: FAIL", index) + // t.Errorf("test #%d: expected empty graph on exit, got %d vertices", index, count) + // } + //}() + defer wg.Wait() + defer cancel() + isEmpty := make(chan struct{}) if fgraph.NumVertices() == 0 { // no funcs to load! close(isEmpty) @@ -2211,44 +2209,29 @@ func TestAstFunc3(t *testing.T) { // wait for some activity logf("stream...") - stream := funcs.Stream() - //select { - //case err, ok := <-stream: - // if !ok { - // t.Errorf("test #%d: FAIL", index) - // t.Errorf("test #%d: stream closed", index) - // return - // } - // if err != nil { - // t.Errorf("test #%d: FAIL", index) - // t.Errorf("test #%d: stream errored: %+v", index, err) - // return - // } - // - //case <-time.After(60 * time.Second): // blocked functions - // t.Errorf("test #%d: FAIL", index) - // t.Errorf("test #%d: stream timeout", index) - // return - //} + tableChan := funcs.Stream() // sometimes the <-stream seems to constantly (or for a // long time?) win the races against the <-time.After(), // so add some limit to how many times we need to stream max := 1 + var table interfaces.Table Loop: for { + var ok bool select { - case err, ok := <-stream: + case table, ok = <-tableChan: if !ok { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: stream closed", index) - return - } - if err != nil { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: stream errored: %+v", index, err) + //t.Errorf("test #%d: FAIL", index) // check in funcs.Err() instead. + t.Errorf("test #%d: FAIL", index) // check in funcs.Err() instead. + t.Logf("test #%d: stream closed", index) return } + //if err != nil { + // t.Errorf("test #%d: FAIL", index) + // t.Errorf("test #%d: stream errored: %+v", index, err) + // return + //} t.Logf("test #%d: got stream event!", index) max-- if max == 0 { @@ -2269,10 +2252,9 @@ func TestAstFunc3(t *testing.T) { } } - t.Logf("test #%d: %s", index, funcs.Stats()) + //t.Logf("test #%d: %s", index, funcs.Stats()) // run interpret! - table := funcs.Table() // map[interfaces.Func]types.Value interpreter := &interpret.Interpreter{ Debug: testing.Verbose(), // set via the -test.v flag to `go test` diff --git a/lang/interpret_test/TestAstFunc1/changing-func.txtar b/lang/interpret_test/TestAstFunc1/changing-func.txtar index 55b23b5e..a63905e4 100644 --- a/lang/interpret_test/TestAstFunc1/changing-func.txtar +++ b/lang/interpret_test/TestAstFunc1/changing-func.txtar @@ -21,21 +21,23 @@ $out2 = $fn2() test "${out1}" {} test "${out2}" {} -- OUTPUT -- +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn Edge: call -> callSubgraphOutput # dummy Edge: call -> callSubgraphOutput # dummy -Edge: const: bool(false) -> exprif # condition -Edge: const: bool(true) -> exprif # condition -Edge: exprIfSubgraphOutput -> call # fn -Edge: exprIfSubgraphOutput -> call # fn -Edge: exprif -> exprIfSubgraphOutput # dummy -Edge: exprif -> exprIfSubgraphOutput # dummy +Edge: call -> callSubgraphOutput # dummy +Edge: call -> callSubgraphOutput # dummy +Edge: callSubgraphOutput -> call # fn +Edge: callSubgraphOutput -> call # fn +Vertex: FuncValue +Vertex: FuncValue Vertex: call Vertex: call +Vertex: call +Vertex: call +Vertex: callSubgraphOutput +Vertex: callSubgraphOutput Vertex: callSubgraphOutput Vertex: callSubgraphOutput Vertex: const: bool(false) Vertex: const: bool(true) -Vertex: exprIfSubgraphOutput -Vertex: exprIfSubgraphOutput -Vertex: exprif -Vertex: exprif diff --git a/lang/interpret_test/TestAstFunc1/graph10.txtar b/lang/interpret_test/TestAstFunc1/graph10.txtar index a43fb3dc..ab3c6b94 100644 --- a/lang/interpret_test/TestAstFunc1/graph10.txtar +++ b/lang/interpret_test/TestAstFunc1/graph10.txtar @@ -8,6 +8,8 @@ test "t" { stringptr => $x, } -- OUTPUT -- +Edge: const: bool(true) -> stmtif # condition Vertex: const: bool(true) Vertex: const: str("hello") Vertex: const: str("t") +Vertex: stmtif diff --git a/lang/interpret_test/TestAstFunc1/graph11.txtar b/lang/interpret_test/TestAstFunc1/graph11.txtar index f47f5a1f..31d4fd31 100644 --- a/lang/interpret_test/TestAstFunc1/graph11.txtar +++ b/lang/interpret_test/TestAstFunc1/graph11.txtar @@ -9,6 +9,6 @@ if true { } } -- OUTPUT -- +Edge: const: bool(true) -> stmtif # condition Vertex: const: bool(true) -Vertex: const: str("t") -Vertex: const: str("world") +Vertex: stmtif diff --git a/lang/interpret_test/TestAstFunc1/graph12.txtar b/lang/interpret_test/TestAstFunc1/graph12.txtar index 852b4303..13ed1cb7 100644 --- a/lang/interpret_test/TestAstFunc1/graph12.txtar +++ b/lang/interpret_test/TestAstFunc1/graph12.txtar @@ -12,8 +12,8 @@ test "t1" { stringptr => $x, } -- OUTPUT -- +Edge: const: bool(true) -> stmtif # condition Vertex: const: bool(true) Vertex: const: str("hello") Vertex: const: str("t1") -Vertex: const: str("t2") -Vertex: const: str("world") +Vertex: stmtif diff --git a/lang/interpret_test/TestAstFunc1/graph4.txtar b/lang/interpret_test/TestAstFunc1/graph4.txtar index f9af4958..23735dc8 100644 --- a/lang/interpret_test/TestAstFunc1/graph4.txtar +++ b/lang/interpret_test/TestAstFunc1/graph4.txtar @@ -4,4 +4,6 @@ if $b { } $b = true -- OUTPUT -- +Edge: const: bool(true) -> stmtif # condition Vertex: const: bool(true) +Vertex: stmtif diff --git a/lang/interpret_test/TestAstFunc1/graph7.txtar b/lang/interpret_test/TestAstFunc1/graph7.txtar index ef239ae7..06b4213d 100644 --- a/lang/interpret_test/TestAstFunc1/graph7.txtar +++ b/lang/interpret_test/TestAstFunc1/graph7.txtar @@ -8,12 +8,6 @@ if true { } $i = 13 -- OUTPUT -- -Edge: const: int(13) -> _operator # b -Edge: const: int(42) -> _operator # a -Edge: const: str("+") -> _operator # op -Vertex: _operator +Edge: const: bool(true) -> stmtif # condition Vertex: const: bool(true) -Vertex: const: int(13) -Vertex: const: int(42) -Vertex: const: str("+") -Vertex: const: str("t") +Vertex: stmtif diff --git a/lang/interpret_test/TestAstFunc1/importscope0.txtar b/lang/interpret_test/TestAstFunc1/importscope0.txtar index 1ea70439..6224a716 100644 --- a/lang/interpret_test/TestAstFunc1/importscope0.txtar +++ b/lang/interpret_test/TestAstFunc1/importscope0.txtar @@ -16,9 +16,13 @@ class xclass { } } -- OUTPUT -- +Edge: FuncValue -> call # fn +Edge: call -> callSubgraphOutput # dummy +Edge: callSubgraphOutput -> exprif # condition Edge: exprif -> exprIfSubgraphOutput # dummy -Edge: os.is_family_debian -> exprif # condition +Vertex: FuncValue +Vertex: call +Vertex: callSubgraphOutput Vertex: const: str("hello") Vertex: exprIfSubgraphOutput Vertex: exprif -Vertex: os.is_family_debian diff --git a/lang/interpret_test/TestAstFunc1/importscope2.txtar b/lang/interpret_test/TestAstFunc1/importscope2.txtar index 907b3ffe..884c84d0 100644 --- a/lang/interpret_test/TestAstFunc1/importscope2.txtar +++ b/lang/interpret_test/TestAstFunc1/importscope2.txtar @@ -15,9 +15,13 @@ class xclass { } } -- OUTPUT -- +Edge: FuncValue -> call # fn +Edge: call -> callSubgraphOutput # dummy +Edge: callSubgraphOutput -> exprif # condition Edge: exprif -> exprIfSubgraphOutput # dummy -Edge: os.is_family_debian -> exprif # condition +Vertex: FuncValue +Vertex: call +Vertex: callSubgraphOutput Vertex: const: str("hello") Vertex: exprIfSubgraphOutput Vertex: exprif -Vertex: os.is_family_debian diff --git a/lang/interpret_test/TestAstFunc1/returned-func.txtar b/lang/interpret_test/TestAstFunc1/returned-func.txtar index 012da900..211a4e50 100644 --- a/lang/interpret_test/TestAstFunc1/returned-func.txtar +++ b/lang/interpret_test/TestAstFunc1/returned-func.txtar @@ -13,6 +13,10 @@ test "${out}" {} -- OUTPUT -- Edge: FuncValue -> call # fn Edge: call -> callSubgraphOutput # dummy +Edge: call -> callSubgraphOutput # dummy +Edge: callSubgraphOutput -> call # fn Vertex: FuncValue Vertex: call +Vertex: call +Vertex: callSubgraphOutput Vertex: callSubgraphOutput diff --git a/lang/interpret_test/TestAstFunc1/returned-lambda.txtar b/lang/interpret_test/TestAstFunc1/returned-lambda.txtar index 406f5247..0438388d 100644 --- a/lang/interpret_test/TestAstFunc1/returned-lambda.txtar +++ b/lang/interpret_test/TestAstFunc1/returned-lambda.txtar @@ -12,6 +12,10 @@ test "${out}" {} -- OUTPUT -- Edge: FuncValue -> call # fn Edge: call -> callSubgraphOutput # dummy +Edge: call -> callSubgraphOutput # dummy +Edge: callSubgraphOutput -> call # fn Vertex: FuncValue Vertex: call +Vertex: call +Vertex: callSubgraphOutput Vertex: callSubgraphOutput diff --git a/lang/interpret_test/TestAstFunc1/shadowing1.txtar b/lang/interpret_test/TestAstFunc1/shadowing1.txtar index fa711647..9956dbdf 100644 --- a/lang/interpret_test/TestAstFunc1/shadowing1.txtar +++ b/lang/interpret_test/TestAstFunc1/shadowing1.txtar @@ -6,5 +6,7 @@ if true { } test "${x}" {} -- OUTPUT -- +Edge: const: bool(true) -> stmtif # condition Vertex: const: bool(true) Vertex: const: str("hello") +Vertex: stmtif diff --git a/lang/interpret_test/TestAstFunc1/shadowing2.txtar b/lang/interpret_test/TestAstFunc1/shadowing2.txtar index 182deb79..fb9c6f29 100644 --- a/lang/interpret_test/TestAstFunc1/shadowing2.txtar +++ b/lang/interpret_test/TestAstFunc1/shadowing2.txtar @@ -6,5 +6,6 @@ if true { test "${x}" {} } -- OUTPUT -- +Edge: const: bool(true) -> stmtif # condition Vertex: const: bool(true) -Vertex: const: str("world") +Vertex: stmtif diff --git a/lang/interpret_test/TestAstFunc1/shape0.txtar b/lang/interpret_test/TestAstFunc1/shape0.txtar index b94e1ac2..a5de27d0 100644 --- a/lang/interpret_test/TestAstFunc1/shape0.txtar +++ b/lang/interpret_test/TestAstFunc1/shape0.txtar @@ -18,6 +18,7 @@ Edge: callSubgraphOutput -> printf: func(format str, a int) str # a Edge: const: str("%d") -> printf: func(format str, a int) str # format Edge: printf: func(format str, a int) str -> composite: []str # 0 Vertex: FuncValue +Vertex: FuncValue Vertex: call Vertex: callSubgraphOutput Vertex: composite: []str diff --git a/lang/interpret_test/TestAstFunc1/shape5.txtar b/lang/interpret_test/TestAstFunc1/shape5.txtar index f43fba03..b67e4195 100644 --- a/lang/interpret_test/TestAstFunc1/shape5.txtar +++ b/lang/interpret_test/TestAstFunc1/shape5.txtar @@ -19,16 +19,20 @@ $s = fmt.printf("%d", datetime.now()) test [$lambda($s),] {} -- OUTPUT -- +Edge: FuncValue -> call # fn Edge: _operator -> composite: []str # 0 +Edge: call -> callSubgraphOutput # dummy +Edge: callSubgraphOutput -> printf: func(format str, a int) str # a Edge: const: str("!") -> _operator # b Edge: const: str("%d") -> printf: func(format str, a int) str # format Edge: const: str("+") -> _operator # op -Edge: now -> printf: func(format str, a int) str # a Edge: printf: func(format str, a int) str -> _operator # a +Vertex: FuncValue Vertex: _operator +Vertex: call +Vertex: callSubgraphOutput Vertex: composite: []str Vertex: const: str("!") Vertex: const: str("%d") Vertex: const: str("+") -Vertex: now Vertex: printf: func(format str, a int) str diff --git a/lang/interpret_test/TestAstFunc1/shape6.txtar b/lang/interpret_test/TestAstFunc1/shape6.txtar index beaffabc..5f737181 100644 --- a/lang/interpret_test/TestAstFunc1/shape6.txtar +++ b/lang/interpret_test/TestAstFunc1/shape6.txtar @@ -14,20 +14,24 @@ $s = golang.template("out1: {{ . }}", $out1) test [$s,] {} -- OUTPUT -- -Edge: FuncValue -> map # function -Edge: composite: []str -> map # inputs +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: call -> callSubgraphOutput # dummy +Edge: call -> callSubgraphOutput # dummy +Edge: callSubgraphOutput -> composite: []str # 0 Edge: const: str("a") -> composite: []str # 0 Edge: const: str("bb") -> composite: []str # 1 Edge: const: str("ccc") -> composite: []str # 2 -Edge: const: str("out1: {{ . }}") -> template # template -Edge: map -> template # vars -Edge: template -> composite: []str # 0 Vertex: FuncValue +Vertex: FuncValue +Vertex: FuncValue +Vertex: call +Vertex: call +Vertex: callSubgraphOutput +Vertex: callSubgraphOutput Vertex: composite: []str Vertex: composite: []str Vertex: const: str("a") Vertex: const: str("bb") Vertex: const: str("ccc") Vertex: const: str("out1: {{ . }}") -Vertex: map -Vertex: template diff --git a/lang/interpret_test/TestAstFunc1/shape7.txtar b/lang/interpret_test/TestAstFunc1/shape7.txtar index 9c450e05..57a07f95 100644 --- a/lang/interpret_test/TestAstFunc1/shape7.txtar +++ b/lang/interpret_test/TestAstFunc1/shape7.txtar @@ -28,9 +28,12 @@ $s = golang.template("out1: {{ . }}", $out1) test [$lambda($s),] {} -- OUTPUT -- -Edge: FuncValue -> map # function +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn Edge: _operator -> composite: []str # 0 -Edge: composite: []str -> map # inputs +Edge: call -> callSubgraphOutput # dummy +Edge: call -> callSubgraphOutput # dummy +Edge: callSubgraphOutput -> _operator # a Edge: const: str("!") -> _operator # b Edge: const: str("+") -> _operator # op Edge: const: str("a") -> composite: []str # 0 @@ -38,11 +41,14 @@ Edge: const: str("bb") -> composite: []str # 1 Edge: const: str("ccc") -> composite: []str # 2 Edge: const: str("dddd") -> composite: []str # 3 Edge: const: str("eeeee") -> composite: []str # 4 -Edge: const: str("out1: {{ . }}") -> template # template -Edge: map -> template # vars -Edge: template -> _operator # a +Vertex: FuncValue +Vertex: FuncValue Vertex: FuncValue Vertex: _operator +Vertex: call +Vertex: call +Vertex: callSubgraphOutput +Vertex: callSubgraphOutput Vertex: composite: []str Vertex: composite: []str Vertex: const: str("!") @@ -53,5 +59,3 @@ Vertex: const: str("ccc") Vertex: const: str("dddd") Vertex: const: str("eeeee") Vertex: const: str("out1: {{ . }}") -Vertex: map -Vertex: template diff --git a/lang/interpret_test/TestAstFunc1/simple-func1.txtar b/lang/interpret_test/TestAstFunc1/simple-func1.txtar index 07da063a..e33e9bb2 100644 --- a/lang/interpret_test/TestAstFunc1/simple-func1.txtar +++ b/lang/interpret_test/TestAstFunc1/simple-func1.txtar @@ -7,4 +7,8 @@ $out1 = answer() test "${out1}" {} -- OUTPUT -- -Vertex: const: str("the answer is 42") +Edge: FuncValue -> call # fn +Edge: call -> callSubgraphOutput # dummy +Vertex: FuncValue +Vertex: call +Vertex: callSubgraphOutput diff --git a/lang/interpret_test/TestAstFunc1/simple-func2.txtar b/lang/interpret_test/TestAstFunc1/simple-func2.txtar index a34772b9..7ffe6d79 100644 --- a/lang/interpret_test/TestAstFunc1/simple-func2.txtar +++ b/lang/interpret_test/TestAstFunc1/simple-func2.txtar @@ -8,12 +8,20 @@ $out2 = answer() test [$out1 + $out2,] {} -- OUTPUT -- +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn Edge: _operator -> composite: []str # 0 +Edge: call -> callSubgraphOutput # dummy +Edge: call -> callSubgraphOutput # dummy +Edge: callSubgraphOutput -> _operator # a +Edge: callSubgraphOutput -> _operator # b Edge: const: str("+") -> _operator # op -Edge: const: str("the answer is 42") -> _operator # a -Edge: const: str("the answer is 42") -> _operator # b +Vertex: FuncValue +Vertex: FuncValue Vertex: _operator +Vertex: call +Vertex: call +Vertex: callSubgraphOutput +Vertex: callSubgraphOutput Vertex: composite: []str Vertex: const: str("+") -Vertex: const: str("the answer is 42") -Vertex: const: str("the answer is 42") diff --git a/lang/interpret_test/TestAstFunc1/simple-lambda1.txtar b/lang/interpret_test/TestAstFunc1/simple-lambda1.txtar index aa81961c..9c5ac569 100644 --- a/lang/interpret_test/TestAstFunc1/simple-lambda1.txtar +++ b/lang/interpret_test/TestAstFunc1/simple-lambda1.txtar @@ -10,4 +10,8 @@ $out = $answer() test "${out}" {} -- OUTPUT -- -Vertex: const: str("the answer is 42") +Edge: FuncValue -> call # fn +Edge: call -> callSubgraphOutput # dummy +Vertex: FuncValue +Vertex: call +Vertex: callSubgraphOutput diff --git a/lang/interpret_test/TestAstFunc1/simple-lambda2.txtar b/lang/interpret_test/TestAstFunc1/simple-lambda2.txtar index 31f94bfd..08f5289c 100644 --- a/lang/interpret_test/TestAstFunc1/simple-lambda2.txtar +++ b/lang/interpret_test/TestAstFunc1/simple-lambda2.txtar @@ -11,12 +11,19 @@ $out2 = $answer() test [$out1 + $out2,] {} -- OUTPUT -- +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn Edge: _operator -> composite: []str # 0 +Edge: call -> callSubgraphOutput # dummy +Edge: call -> callSubgraphOutput # dummy +Edge: callSubgraphOutput -> _operator # a +Edge: callSubgraphOutput -> _operator # b Edge: const: str("+") -> _operator # op -Edge: const: str("the answer is 42") -> _operator # a -Edge: const: str("the answer is 42") -> _operator # b +Vertex: FuncValue Vertex: _operator +Vertex: call +Vertex: call +Vertex: callSubgraphOutput +Vertex: callSubgraphOutput Vertex: composite: []str Vertex: const: str("+") -Vertex: const: str("the answer is 42") -Vertex: const: str("the answer is 42") diff --git a/lang/interpret_test/TestAstFunc1/slow_unification0.txtar b/lang/interpret_test/TestAstFunc1/slow_unification0.txtar index 0313f0da..b6454b55 100644 --- a/lang/interpret_test/TestAstFunc1/slow_unification0.txtar +++ b/lang/interpret_test/TestAstFunc1/slow_unification0.txtar @@ -52,12 +52,18 @@ if $state == "three" { Exec["timer"] -> Kv["${ns}"] } -- OUTPUT -- +Edge: FuncValue -> call # fn Edge: _lookup_default -> _operator # a Edge: _lookup_default -> _operator # a Edge: _lookup_default -> _operator # a Edge: _lookup_default -> _operator # a Edge: _operator -> _operator # a Edge: _operator -> _operator # b +Edge: _operator -> stmtif # condition +Edge: _operator -> stmtif # condition +Edge: _operator -> stmtif # condition +Edge: call -> callSubgraphOutput # dummy +Edge: callSubgraphOutput -> _lookup_default # listormap Edge: const: str("") -> _lookup_default # indexorkey Edge: const: str("==") -> _operator # op Edge: const: str("==") -> _operator # op @@ -65,25 +71,20 @@ Edge: const: str("==") -> _operator # op Edge: const: str("==") -> _operator # op Edge: const: str("default") -> _lookup_default # default Edge: const: str("default") -> _operator # b -Edge: const: str("estate") -> kvlookup # namespace Edge: const: str("one") -> _operator # b Edge: const: str("or") -> _operator # op Edge: const: str("three") -> _operator # b Edge: const: str("two") -> _operator # b -Edge: kvlookup -> _lookup_default # listormap +Vertex: FuncValue Vertex: _lookup_default Vertex: _operator Vertex: _operator Vertex: _operator Vertex: _operator Vertex: _operator +Vertex: call +Vertex: callSubgraphOutput Vertex: const: str("") -Vertex: const: str("/tmp/mgmt/state") -Vertex: const: str("/tmp/mgmt/state") -Vertex: const: str("/tmp/mgmt/state") -Vertex: const: str("/usr/bin/sleep 1s") -Vertex: const: str("/usr/bin/sleep 1s") -Vertex: const: str("/usr/bin/sleep 1s") Vertex: const: str("==") Vertex: const: str("==") Vertex: const: str("==") @@ -92,19 +93,9 @@ Vertex: const: str("default") Vertex: const: str("default") Vertex: const: str("estate") Vertex: const: str("one") -Vertex: const: str("one") Vertex: const: str("or") -Vertex: const: str("state: one\n") -Vertex: const: str("state: three\n") -Vertex: const: str("state: two\n") Vertex: const: str("three") -Vertex: const: str("three") -Vertex: const: str("timer") -Vertex: const: str("timer") -Vertex: const: str("timer") -Vertex: const: str("timer") -Vertex: const: str("timer") -Vertex: const: str("timer") Vertex: const: str("two") -Vertex: const: str("two") -Vertex: kvlookup +Vertex: stmtif +Vertex: stmtif +Vertex: stmtif diff --git a/lang/interpret_test/TestAstFunc1/static-function0.txtar b/lang/interpret_test/TestAstFunc1/static-function0.txtar index a7935028..edd99c74 100644 --- a/lang/interpret_test/TestAstFunc1/static-function0.txtar +++ b/lang/interpret_test/TestAstFunc1/static-function0.txtar @@ -16,9 +16,19 @@ test "greeting3" { anotherstr => $fn(), } -- OUTPUT -- +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: call -> callSubgraphOutput # dummy +Edge: call -> callSubgraphOutput # dummy +Edge: call -> callSubgraphOutput # dummy +Vertex: FuncValue +Vertex: call +Vertex: call +Vertex: call +Vertex: callSubgraphOutput +Vertex: callSubgraphOutput +Vertex: callSubgraphOutput Vertex: const: str("greeting1") Vertex: const: str("greeting2") Vertex: const: str("greeting3") -Vertex: const: str("hello world") -Vertex: const: str("hello world") -Vertex: const: str("hello world") diff --git a/lang/interpret_test/TestAstFunc2/class-include-as-vars-simple.txtar b/lang/interpret_test/TestAstFunc2/class-include-as-vars-simple.txtar new file mode 100644 index 00000000..a848cf75 --- /dev/null +++ b/lang/interpret_test/TestAstFunc2/class-include-as-vars-simple.txtar @@ -0,0 +1,14 @@ +-- main.mcl -- +class c1($b) { + if $b { + test "t2" {} + } else { + test "t3" {} + } +} +include c1(true) +include c1(false) + +-- OUTPUT -- +Vertex: test[t2] +Vertex: test[t3] diff --git a/lang/interpret_test/TestAstFunc2/consistency0.txtar b/lang/interpret_test/TestAstFunc2/consistency0.txtar new file mode 100644 index 00000000..09d12c96 --- /dev/null +++ b/lang/interpret_test/TestAstFunc2/consistency0.txtar @@ -0,0 +1,13 @@ +-- main.mcl -- +# This test checks that we propagate consistently and without glitches. We would +# fail with: test[-1] or test[1] or similar if we weren't consistent. It is not +# necessarily guaranteed by the function engine to work this way, but we aim to. +import "datetime" +import "fmt" + +$now = datetime.now() +$zero = $now - $now + +test [fmt.printf("%d", $zero),] {} +-- OUTPUT -- +Vertex: test[0] diff --git a/lang/interpret_test/TestAstFunc2/map-ignore-arg.txtar b/lang/interpret_test/TestAstFunc2/map-ignore-arg.txtar index d78a3034..1702be7b 100644 --- a/lang/interpret_test/TestAstFunc2/map-ignore-arg.txtar +++ b/lang/interpret_test/TestAstFunc2/map-ignore-arg.txtar @@ -2,6 +2,8 @@ import "iter" $fn = func($x) { # ignore arg + # XXX: copy() bug (performance) in the function graph, we see this "hey" + # str node three times instead of just seeing it only once. Is it a bug? "hey" } diff --git a/lang/lang.go b/lang/lang.go index b4676fc8..1f97ac3a 100644 --- a/lang/lang.go +++ b/lang/lang.go @@ -106,10 +106,14 @@ type Lang struct { funcs *dage.Engine // function event engine graph *pgraph.Graph // function graph - streamChan <-chan error // signals a new graph can be created or problem + streamChan chan *pgraph.Graph // stream of new graphs //streamBurst bool // should we try and be bursty with the stream events? - wg *sync.WaitGroup + interpreter *interpret.Interpreter + + wg *sync.WaitGroup + err error + errMutex *sync.Mutex // guards err } // Init initializes the lang struct, and starts up the initial input parsing. @@ -338,51 +342,46 @@ func (obj *Lang) Init(ctx context.Context) error { return errwrap.Wrapf(err, "init error with func engine") } - obj.streamChan = obj.funcs.Stream() // after obj.funcs.Setup runs + //obj.Logf("function engine validating...") + //if err := obj.funcs.Validate(); err != nil { + // return errwrap.Wrapf(err, "validate error with func engine") + //} + + obj.streamChan = make(chan *pgraph.Graph) + + obj.interpreter = &interpret.Interpreter{ + Debug: obj.Debug, + Logf: func(format string, v ...interface{}) { + // TODO: is this a sane prefix to use here? + obj.Logf("interpret: "+format, v...) + }, + } + + obj.wg = &sync.WaitGroup{} + obj.errMutex = &sync.Mutex{} return nil } // Run kicks off the function engine. Use the context to shut it down. func (obj *Lang) Run(ctx context.Context) (reterr error) { - wg := &sync.WaitGroup{} - defer wg.Wait() + defer obj.wg.Wait() - runCtx, cancel := context.WithCancel(context.Background()) // Don't inherit from parent + ctx, cancel := context.WithCancel(ctx) // wrap parent defer cancel() - var timing time.Time - - //obj.Logf("function engine validating...") - //if err := obj.funcs.Validate(); err != nil { - // return errwrap.Wrapf(err, "validate error with func engine") - //} - - obj.Logf("function engine starting...") - timing = time.Now() - wg.Add(1) - go func() { - defer wg.Done() - if err := obj.funcs.Run(runCtx); err == nil { - reterr = errwrap.Append(reterr, err) - } - // Run() should only error if not a dag I think... - }() - - <-obj.funcs.Started() // wait for startup (will not block forever) + //<-obj.funcs.Started() // wait for startup (will not block forever) // Sanity checks for graph size. - if count := obj.funcs.NumVertices(); count != 0 { - return fmt.Errorf("expected empty graph on start, got %d vertices", count) - } - defer func() { - if count := obj.funcs.NumVertices(); count != 0 { - err := fmt.Errorf("expected empty graph on exit, got %d vertices", count) - reterr = errwrap.Append(reterr, err) - } - }() - defer wg.Wait() - defer cancel() // now cancel Run only after Reverse and Free are done! + //if count := obj.funcs.NumVertices(); count != 0 { + // return fmt.Errorf("expected empty graph on start, got %d vertices", count) + //} + //defer func() { + // if count := obj.funcs.NumVertices(); count != 0 { + // err := fmt.Errorf("expected empty graph on exit, got %d vertices", count) + // reterr = errwrap.Append(reterr, err) + // } + //}() txn := obj.funcs.Txn() defer txn.Free() // remember to call Free() @@ -396,70 +395,98 @@ func (obj *Lang) Run(ctx context.Context) (reterr error) { } }() - obj.Logf("function engine starting took: %s", time.Since(timing)) + //obj.Logf("function engine starting took: %s", time.Since(timing)) // wait for some activity obj.Logf("stream...") - // print some stats if the engine takes too long to startup - if EngineStartupStatsTimeout > 0 { - wg.Add(1) - go func() { - defer wg.Done() + tableChan := obj.funcs.Stream() // after obj.funcs.Setup runs + + obj.wg.Add(1) + go func() { + defer obj.wg.Done() + defer close(obj.streamChan) + defer cancel() // if this loop errors, it should cancel and err + + var table interfaces.Table + var ok bool + for { select { - case <-obj.funcs.Loaded(): // funcs are now loaded! - case <-time.After(time.Duration(EngineStartupStatsTimeout) * time.Second): - obj.Logf("stats...") - obj.Logf("%s", obj.funcs.Stats()) + case table, ok = <-tableChan: + if !ok { + return + } + case <-ctx.Done(): + obj.errAppend(ctx.Err()) + return } - }() - } - select { - case <-ctx.Done(): - } + // this call returns the graph + // XXX: add a ctx? + graph, err := obj.interpreter.Interpret(obj.ast, table) + if err != nil { + e := errwrap.Wrapf(err, "could not interpret") + obj.errAppend(e) + return + } - return nil + select { + case obj.streamChan <- graph: + + case <-ctx.Done(): + obj.errAppend(ctx.Err()) + return + } + } + }() + + // print some stats if the engine takes too long to startup + //if EngineStartupStatsTimeout > 0 { + // wg.Add(1) + // go func() { + // defer wg.Done() + // select { + // case <-obj.funcs.Loaded(): // funcs are now loaded! + // case <-time.After(time.Duration(EngineStartupStatsTimeout) * time.Second): + // obj.Logf("stats...") + // obj.Logf("%s", obj.funcs.Stats()) + // case <-ctx.Done(): + // } + // }() + //} + + obj.Logf("function engine starting...") + + err := obj.funcs.Run(ctx) + // When run terminates, inspect the "official" error first. + if err := obj.funcs.Err(); err != nil { + return err + } + return err // If we got this far, return whatever Run did. } -// Stream returns a channel of graph change requests or errors. These are -// usually sent when a func output changes. -func (obj *Lang) Stream() <-chan error { +// Stream returns a channel of resource graphs. This changes when a func output +// changes. +func (obj *Lang) Stream(ctx context.Context) <-chan *pgraph.Graph { return obj.streamChan } -// Interpret runs the interpreter and returns a graph and corresponding error. -func (obj *Lang) Interpret() (*pgraph.Graph, error) { - select { - case <-obj.funcs.Loaded(): // funcs are now loaded! - // pass - default: - // if this is hit, someone probably called this too early! - // it should only be called in response to a stream event! - return nil, fmt.Errorf("funcs aren't loaded yet") - } - - obj.Logf("running interpret...") - table := obj.funcs.Table() // map[pgraph.Vertex]types.Value - - interpreter := &interpret.Interpreter{ - Debug: obj.Debug, - Logf: func(format string, v ...interface{}) { - // TODO: is this a sane prefix to use here? - obj.Logf("interpret: "+format, v...) - }, - } - - // this call returns the graph - graph, err := interpreter.Interpret(obj.ast, table) - if err != nil { - return nil, errwrap.Wrapf(err, "could not interpret") - } - - return graph, nil // return a graph +// Err will contain the last error when Stream shuts down. It waits for all the +// running processes to exit before it returns. +func (obj *Lang) Err() error { + obj.wg.Wait() + return obj.err } // Cleanup cleans up and frees memory and resources after everything is done. func (obj *Lang) Cleanup() error { - return obj.funcs.Cleanup() + //return obj.funcs.Cleanup() // not implemented atm + return nil +} + +// errAppend is a simple helper function. +func (obj *Lang) errAppend(err error) { + obj.errMutex.Lock() + obj.err = errwrap.Append(obj.err, err) + obj.errMutex.Unlock() } diff --git a/lang/lang_test.go b/lang/lang_test.go index 0a77b696..42d62edc 100644 --- a/lang/lang_test.go +++ b/lang/lang_test.go @@ -151,27 +151,20 @@ func runInterpret(t *testing.T, code string) (_ *pgraph.Graph, reterr error) { wg.Add(1) go func() { defer wg.Done() - if err := lang.Run(ctx); err != nil { + if err := lang.Run(ctx); err != nil && err != context.Canceled { reterr = errwrap.Append(reterr, err) } }() defer cancel() // shutdown the Run // we only wait for the first event, instead of the continuous stream + var graph *pgraph.Graph + var ok bool select { - case err, ok := <-lang.Stream(): + case graph, ok = <-lang.Stream(ctx): if !ok { return nil, fmt.Errorf("stream closed without event") } - if err != nil { - return nil, errwrap.Wrapf(err, "stream failed") - } - } - - // run artificially without the entire GAPI loop - graph, err := lang.Interpret() - if err != nil { - return nil, errwrap.Wrapf(err, "interpret failed") } return graph, nil diff --git a/lang/types/full/full.go b/lang/types/full/full.go index 97966d07..216421a2 100644 --- a/lang/types/full/full.go +++ b/lang/types/full/full.go @@ -69,7 +69,7 @@ func NewFunc(t *types.Type) *FuncValue { if t.Kind != types.KindFunc { return nil // sanity check } - v := func(interfaces.Txn, []interfaces.Func) (interfaces.Func, error) { + v := func(interfaces.Txn, []interfaces.Func, interfaces.Func) (interfaces.Func, error) { return nil, fmt.Errorf("nil function") // TODO: is this correct? } f := func(context.Context, []types.Value) (types.Value, error) { @@ -170,6 +170,6 @@ func (obj *FuncValue) CallWithValues(ctx context.Context, args []types.Value) (t } // CallWithFuncs calls the function with the provided txn and func args. -func (obj *FuncValue) CallWithFuncs(txn interfaces.Txn, args []interfaces.Func) (interfaces.Func, error) { - return obj.V(txn, args) +func (obj *FuncValue) CallWithFuncs(txn interfaces.Txn, args []interfaces.Func, out interfaces.Func) (interfaces.Func, error) { + return obj.V(txn, args, out) } diff --git a/lib/main.go b/lib/main.go index 26e07452..f9cbe150 100644 --- a/lib/main.go +++ b/lib/main.go @@ -783,6 +783,9 @@ func (obj *Main) Run() error { Logf("gapi exited") } gapiChan = nil // disable it + if err := gapiImpl.Err(); err != nil { + obj.Logf("gapi exited with error: %+v", errwrap.WithoutContext(err)) + } continue } diff --git a/puppet/gapi.go b/puppet/gapi.go index f796848e..9922def5 100644 --- a/puppet/gapi.go +++ b/puppet/gapi.go @@ -79,6 +79,7 @@ type GAPI struct { initialized bool wg sync.WaitGroup err error + errMutex *sync.Mutex // guards err } // Cli takes an *Info struct, and returns our deploy if activated, and if there @@ -240,6 +241,7 @@ func (obj *GAPI) Init(data *gapi.Data) error { } } + obj.errMutex = &sync.Mutex{} obj.initialized = true return nil } @@ -287,10 +289,10 @@ func (obj *GAPI) Next(ctx context.Context) chan gapi.Next { select { case ch <- next: case <-ctx.Done(): - obj.err = ctx.Err() + obj.errAppend(ctx.Err()) return } - obj.err = err + obj.errAppend(err) return } startChan := make(chan struct{}) // start signal @@ -313,7 +315,7 @@ func (obj *GAPI) Next(ctx context.Context) chan gapi.Next { return } case <-ctx.Done(): - obj.err = ctx.Err() + obj.errAppend(ctx.Err()) return } @@ -326,7 +328,7 @@ func (obj *GAPI) Next(ctx context.Context) chan gapi.Next { obj.data.Logf("generating new graph...") g, err := obj.graph() if err != nil { - obj.err = err + obj.errAppend(err) return } @@ -339,7 +341,7 @@ func (obj *GAPI) Next(ctx context.Context) chan gapi.Next { case ch <- next: // trigger a run (send a msg) // unblock if we exit while waiting to send! case <-ctx.Done(): - obj.err = ctx.Err() + obj.errAppend(ctx.Err()) return } } @@ -376,3 +378,10 @@ func (obj *GAPI) cleanup() error { obj.initialized = false // closed = true return nil } + +// errAppend is a simple helper function. +func (obj *GAPI) errAppend(err error) { + obj.errMutex.Lock() + obj.err = errwrap.Append(obj.err, err) + obj.errMutex.Unlock() +} diff --git a/test/shell/exchange.sh b/test/shell/exchange.sh deleted file mode 100755 index 83228d98..00000000 --- a/test/shell/exchange.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash - -. "$(dirname "$0")/../util.sh" - -if in_env github; then - # TODO: consider debugging this - echo "This is failing in github, skipping test!" - exit -fi - -set -o errexit -set -o pipefail - -$TIMEOUT "$MGMT" run --hostname h1 --tmp-prefix --no-pgp empty & -pid1=$! -sleep 10s -$TIMEOUT "$MGMT" run --hostname h2 --seeds=http://127.0.0.1:2379 --client-urls=http://127.0.0.1:2381 --server-urls=http://127.0.0.1:2382 --tmp-prefix --no-pgp empty & -pid2=$! -sleep 10s -$TIMEOUT "$MGMT" run --hostname h3 --seeds=http://127.0.0.1:2379 --client-urls=http://127.0.0.1:2383 --server-urls=http://127.0.0.1:2384 --tmp-prefix --no-pgp empty & -pid3=$! -sleep 10s -$TIMEOUT "$MGMT" run --hostname h4 --seeds=http://127.0.0.1:2379 --client-urls=http://127.0.0.1:2385 --server-urls=http://127.0.0.1:2386 --tmp-prefix --no-pgp empty & -pid4=$! -sleep 10s -$TIMEOUT "$MGMT" deploy --no-git --seeds=http://127.0.0.1:2379 lang exchange0.mcl - -# kill servers on error/exit -#trap 'pkill -9 mgmt' EXIT - -# wait for everything to converge -sleep 15s - -# debug -tail /tmp/mgmt/exchange-* - -test "$(cat /tmp/mgmt/exchange-* | grep -c h1)" -eq 4 -test "$(cat /tmp/mgmt/exchange-* | grep -c h2)" -eq 4 -test "$(cat /tmp/mgmt/exchange-* | grep -c h3)" -eq 4 -test "$(cat /tmp/mgmt/exchange-* | grep -c h4)" -eq 4 - -$(sleep 15s && kill -SIGINT $pid4)& # send ^C to exit mgmt... -wait $pid4 -e=$? -if [ $e -ne 0 ]; then - exit $e -fi - -$(sleep 15s && kill -SIGINT $pid3)& # send ^C to exit mgmt... -wait $pid3 -e=$? -if [ $e -ne 0 ]; then - exit $e -fi - -$(sleep 15s && kill -SIGINT $pid2)& # send ^C to exit mgmt... -wait $pid2 -e=$? -if [ $e -ne 0 ]; then - exit $e -fi - -$(sleep 15s && kill -SIGINT $pid1)& # send ^C to exit mgmt... -wait $pid1 -e=$? -if [ $e -ne 0 ]; then - exit $e -fi diff --git a/test/shell/exchange0.mcl b/test/shell/exchange0.mcl deleted file mode 100644 index b7c45144..00000000 --- a/test/shell/exchange0.mcl +++ /dev/null @@ -1,20 +0,0 @@ -# run this example with these commands -# watch -n 0.1 'tail *' # run this in /tmp/mgmt/ -# time ./mgmt run --hostname h1 --tmp-prefix --no-pgp empty -# time ./mgmt run --hostname h2 --seeds=http://127.0.0.1:2379 --client-urls=http://127.0.0.1:2381 --server-urls=http://127.0.0.1:2382 --tmp-prefix --no-pgp empty -# time ./mgmt run --hostname h3 --seeds=http://127.0.0.1:2379 --client-urls=http://127.0.0.1:2383 --server-urls=http://127.0.0.1:2384 --tmp-prefix --no-pgp empty -# time ./mgmt run --hostname h4 --seeds=http://127.0.0.1:2379 --client-urls=http://127.0.0.1:2385 --server-urls=http://127.0.0.1:2386 --tmp-prefix --no-pgp empty -# time ./mgmt deploy --no-git --seeds=http://127.0.0.1:2379 lang examples/lang/exchange0.mcl - -import "golang" -import "sys" -import "world" - -$rand = random1(8) -$exchanged = world.exchange("keyns", $rand) -$host = sys.hostname() - -file "/tmp/mgmt/exchange-${host}" { - state => $const.res.file.state.exists, - content => golang.template("Found: {{ . }}\n", $exchanged), -} diff --git a/yamlgraph/gapi.go b/yamlgraph/gapi.go index c9b4667a..a5b6fa17 100644 --- a/yamlgraph/gapi.go +++ b/yamlgraph/gapi.go @@ -62,6 +62,7 @@ type GAPI struct { closeChan chan struct{} wg sync.WaitGroup // sync group for tunnel go routines err error + errMutex *sync.Mutex // guards err } // Cli takes an *Info struct, and returns our deploy if activated, and if there @@ -111,6 +112,7 @@ func (obj *GAPI) Init(data *gapi.Data) error { } obj.data = data // store for later obj.closeChan = make(chan struct{}) + obj.errMutex = &sync.Mutex{} obj.initialized = true return nil } @@ -171,10 +173,10 @@ func (obj *GAPI) Next(ctx context.Context) chan gapi.Next { select { case ch <- next: case <-ctx.Done(): - obj.err = ctx.Err() + obj.errAppend(ctx.Err()) return } - obj.err = err + obj.errAppend(err) return } @@ -216,7 +218,7 @@ func (obj *GAPI) Next(ctx context.Context) chan gapi.Next { obj.data.Logf("generating new graph...") g, err := obj.graph() if err != nil { - obj.err = err + obj.errAppend(err) return } @@ -246,3 +248,10 @@ func (obj *GAPI) Err() error { obj.wg.Wait() return obj.err } + +// errAppend is a simple helper function. +func (obj *GAPI) errAppend(err error) { + obj.errMutex.Lock() + obj.err = errwrap.Append(obj.err, err) + obj.errMutex.Unlock() +}