// Mgmt // Copyright (C) James Shubin and the project contributors // Written by James Shubin and the project contributors // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with this program. If not, see . // // Additional permission under GNU GPL version 3 section 7 // // If you modify this program, or any covered work, by linking or combining it // with embedded mcl code and modules (and that the embedded mcl code and // modules which link with this program, contain a copy of their source code in // the authoritative form) containing parts covered by the terms of any other // license, the licensors of this program grant you additional permission to // convey the resulting work. Furthermore, the licensors of this program grant // the original author, James Shubin, additional permission to update this // additional permission if he deems it necessary to achieve the goals of this // additional permission. package core import ( "context" "fmt" "github.com/purpleidea/mgmt/engine" "github.com/purpleidea/mgmt/lang/funcs" "github.com/purpleidea/mgmt/lang/interfaces" "github.com/purpleidea/mgmt/lang/types" "github.com/purpleidea/mgmt/util/errwrap" ) const ( // CollectFuncName is the name this function is registered as. This // starts with an underscore so that it cannot be used from the lexer. CollectFuncName = funcs.CollectFuncName // arg names... collectArgNameKind = "kind" collectArgNameNames = "names" //collectFuncInType = "[]struct{kind str; name str; host str}" //collectFuncInFieldKind = "kind" // must match above struct field collectFuncInFieldName = funcs.CollectFuncInFieldName collectFuncInFieldHost = funcs.CollectFuncInFieldHost // collectFuncInType is the most complex of the three possible input // types. The other two possible ones are str or []str. collectFuncInType = funcs.CollectFuncInType // "[]struct{name str; host str}" collectFuncOutFieldName = funcs.CollectFuncOutFieldName collectFuncOutFieldHost = funcs.CollectFuncOutFieldHost collectFuncOutFieldData = funcs.CollectFuncOutFieldData // collectFuncOutStruct is the struct type that we return a list of. collectFuncOutStruct = funcs.CollectFuncOutStruct // collectFuncOutType is the expected return type, the data field is an // encoded resource blob. // XXX: Once structs can be real map keys in mcl, could this instead be: // map{struct{name str; host str}: str} // key => $data (efficiency!) collectFuncOutType = funcs.CollectFuncOutType // "[]struct{name str; host str; data str}" ) func init() { funcs.Register(CollectFuncName, func() interfaces.Func { return &CollectFunc{} }) // must register the func and name } var _ interfaces.InferableFunc = &CollectFunc{} // ensure it meets this expectation // CollectFunc is a special internal function which gets given information about // incoming resource collection data. For example, to collect, that "pseudo // resource" will need to know what resource "kind" it's collecting, the names // of those resources, and the corresponding hostnames that they are getting the // data from. With that three-tuple of data, it can pull all of that from etcd // and pass it into a hidden resource body field so that the collect "pseudo // resource" can use it to build the exported resource! // // The "kind" comes in as the first arg. The second arg (in its complex form) is // []struct{name str; host str} is what the end user is _asking_ this function // for. // TODO: We could have a second version of this collect function which takes a // single arg which receives []struct{kind str; name str; host str} which would // let us write a truly dynamic collector. It's unlikely we want to allow this // in most cases because it lets you play type games since the field name in one // resource kind might be a different type in another. type CollectFunc struct { // Type is the type of the second arg that we receive. (When known.) Type *types.Type init *interfaces.Init last types.Value // last value received to use for diff args []types.Value kind string result types.Value // last calculated output watchChan chan error } // String returns a simple name for this function. This is needed so this struct // can satisfy the pgraph.Vertex interface. func (obj *CollectFunc) String() string { return CollectFuncName } // ArgGen returns the Nth arg name for this function. func (obj *CollectFunc) ArgGen(index int) (string, error) { seq := []string{collectArgNameKind, collectArgNameNames} if l := len(seq); index >= l { return "", fmt.Errorf("index %d exceeds arg length of %d", index, l) } return seq[index], nil } // helper func (obj *CollectFunc) sig() *types.Type { arg := "?1" if obj.Type != nil { arg = obj.Type.String() } return types.NewType(fmt.Sprintf( "func(%s str, %s %s) %s", collectArgNameKind, collectArgNameNames, arg, collectFuncOutType, )) } // check determines if our arg type is valid. func (obj *CollectFunc) check(typ *types.Type) error { if typ.Cmp(types.TypeStr) == nil { return nil } if typ.Cmp(types.TypeListStr) == nil { return nil } if typ.Cmp(types.NewType(collectFuncInType)) == nil { return nil } return fmt.Errorf("unexpected type: %s", typ.String()) } // FuncInfer takes partial type and value information from the call site of this // function so that it can build an appropriate type signature for it. The type // signature may include unification variables. func (obj *CollectFunc) FuncInfer(partialType *types.Type, partialValues []types.Value) (*types.Type, []*interfaces.UnificationInvariant, error) { // There are many variants which we could allow... These variants are // what the user specifies in the $name field when they collect. They // will often get the third form from helper functions that filter the // data from the world graph, so that they can programmatically match // using our mcl language rather than hard-coding a mini matcher lang. // // XXX: Do we want to allow all these variants? // // func(str, str) out # matches all hostnames // OR // func(str, []str) out # matches all hostnames // OR // func(str, []struct{name str; host str} ) out # matches exact tuples or all hostnames if host is "" // SO // func(str, ?1) out // AND // out = []struct{name str; host str; data str} # it could have kind too, but not needed right now // // NOTE: map[str]str (name => host) is NOT a good choice because even // though we nominally have one host exporting a given name, it's valid // to have that same name come from more than one host and for them to // be compatible, almost like an "exported resources redundancy". // // NOTE map[str][]str (name => []host) is sensible, BUT it makes it // harder to express that we want "every host", which we can do with the // struct variant above by having host be the empty string. It's also // easier for the mcl programmer to understand that variant. if l := 2; len(partialValues) != l { return nil, nil, fmt.Errorf("function must have %d args", l) } if err := partialValues[0].Type().Cmp(types.TypeStr); err != nil { return nil, nil, errwrap.Wrapf(err, "function arg kind must be a str") } kind := partialValues[0].Str() // must not panic if kind == "" { return nil, nil, fmt.Errorf("function must not have an empty kind arg") } if !engine.IsKind(kind) { return nil, nil, fmt.Errorf("invalid resource kind: %s", kind) } // If second arg is one of what we're expecting, then we are solved! if len(partialType.Ord) == 2 && partialType.Map[partialType.Ord[1]] != nil { typ := partialType.Map[partialType.Ord[1]] if err := obj.check(typ); err == nil { obj.Type = typ // success! } } return obj.sig(), []*interfaces.UnificationInvariant{}, nil } // Build is run to turn the polymorphic, undetermined function, into the // specific statically typed version. It is usually run after Unify completes, // and must be run before Info() and any of the other Func interface methods are // used. This function is idempotent, as long as the arg isn't changed between // runs. func (obj *CollectFunc) Build(typ *types.Type) (*types.Type, error) { // typ is the KindFunc signature we're trying to build... if typ.Kind != types.KindFunc { return nil, fmt.Errorf("input type must be of kind func") } if len(typ.Ord) != 2 { return nil, fmt.Errorf("the collect function needs two args") } tStr, exists := typ.Map[typ.Ord[0]] if !exists || tStr == nil { return nil, fmt.Errorf("first arg must be specified") } if tStr.Cmp(types.TypeStr) != nil { return nil, fmt.Errorf("first arg must be a str") } tArg, exists := typ.Map[typ.Ord[1]] if !exists || tArg == nil { return nil, fmt.Errorf("second arg must be specified") } if err := obj.check(tArg); err != nil { return nil, err } obj.Type = tArg // store it! return obj.sig(), nil } // Copy is implemented so that the obj.Type value is not lost if we copy this // function. That value is learned during FuncInfer, and previously would have // been lost by the time we used it in Build. func (obj *CollectFunc) Copy() interfaces.Func { return &CollectFunc{ Type: obj.Type, // don't copy because we use this after unification init: obj.init, // likely gets overwritten anyways } } // Validate tells us if the input struct takes a valid form. func (obj *CollectFunc) Validate() error { if obj.Type == nil { return fmt.Errorf("the Type is unknown") } if err := obj.check(obj.Type); err != nil { return err } return nil } // Info returns some static info about itself. Build must be called before this // will return correct data. func (obj *CollectFunc) Info() *interfaces.Info { // Since this function implements FuncInfer we want sig to return nil to // avoid an accidental return of unification variables when we should be // getting them from FuncInfer, and not from here. (During unification!) var sig *types.Type if obj.Type != nil && obj.check(obj.Type) == nil { sig = obj.sig() // helper } return &interfaces.Info{ Pure: false, Memo: false, Fast: false, Spec: false, Sig: sig, Err: obj.Validate(), } } // Init runs some startup code for this function. func (obj *CollectFunc) Init(init *interfaces.Init) error { obj.init = init obj.watchChan = make(chan error) // XXX: sender should close this, but did I implement that part yet??? return nil } // Stream returns the changing values that this func has over time. func (obj *CollectFunc) Stream(ctx context.Context) error { defer close(obj.init.Output) // the sender closes ctx, cancel := context.WithCancel(ctx) defer cancel() // important so that we cleanup the watch when exiting for { select { // TODO: should this first chan be run as a priority channel to // avoid some sort of glitch? is that even possible? can our // hostname check with reality (below) fix that? case input, ok := <-obj.init.Input: if !ok { obj.init.Input = nil // don't infinite loop back continue // no more inputs, but don't return! } //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { // return errwrap.Wrapf(err, "wrong function input") //} if obj.last != nil && input.Cmp(obj.last) == nil { continue // value didn't change, skip it } obj.last = input // store for next args, err := interfaces.StructToCallableArgs(input) // []types.Value, error) if err != nil { return err } obj.args = args kind := args[0].Str() if kind == "" { return fmt.Errorf("can't use an empty kind") } if obj.init.Debug { obj.init.Logf("kind: %s", kind) } // TODO: support changing the key over time? if obj.kind == "" { obj.kind = kind // store it var err error // Don't send a value right away, wait for the // first Watch startup event to get one! obj.watchChan, err = obj.init.World.ResWatch(ctx, obj.kind) // watch for var changes if err != nil { return err } } else if obj.kind != kind { return fmt.Errorf("can't change kind, previously: `%s`", obj.kind) } continue // we get values on the watch chan, not here! case err, ok := <-obj.watchChan: if !ok { // closed // XXX: if we close, perhaps the engine is // switching etcd hosts and we should retry? // maybe instead we should get an "etcd // reconnect" signal, and the lang will restart? return nil } if err != nil { return errwrap.Wrapf(err, "channel watch failed on `%s`", obj.kind) } result, err := obj.Call(ctx, obj.args) // get the value... if err != nil { return err } // if the result is still the same, skip sending an update... if obj.result != nil && result.Cmp(obj.result) == nil { continue // result didn't change } obj.result = result // store new result case <-ctx.Done(): return nil } select { case obj.init.Output <- obj.result: // send // pass case <-ctx.Done(): return nil } } } // Call this function with the input args and return the value if it is possible // to do so at this time. This was previously getValue which gets the value // we're looking for. func (obj *CollectFunc) Call(ctx context.Context, args []types.Value) (types.Value, error) { if len(args) < 2 { return nil, fmt.Errorf("not enough args") } kind := args[0].Str() if kind == "" { return nil, fmt.Errorf("resource kind is empty") } if !engine.IsKind(kind) { return nil, fmt.Errorf("invalid resource kind: %s", kind) } filters := []*engine.ResFilter{} arg := args[1] typ := arg.Type() // Can be one of: str, []str, []struct{name str; host str} for matching. if typ.Cmp(types.TypeStr) == nil { // it must be a name only filter := &engine.ResFilter{ Kind: kind, Name: arg.Str(), Host: "", // any } filters = append(filters, filter) } if typ.Cmp(types.TypeListStr) == nil { for _, x := range arg.List() { filter := &engine.ResFilter{ Kind: kind, Name: x.Str(), Host: "", // any } filters = append(filters, filter) } } if typ.Cmp(types.NewType(collectFuncInType)) == nil { for _, x := range arg.List() { st, ok := x.(*types.StructValue) if !ok { // programming error return nil, fmt.Errorf("value is not a struct") } name, exists := st.Lookup(collectFuncInFieldName) if !exists { // programming error? return nil, fmt.Errorf("name field is missing") } host, exists := st.Lookup(collectFuncInFieldHost) if !exists { // programming error? return nil, fmt.Errorf("host field is missing") } filter := &engine.ResFilter{ Kind: kind, Name: name.Str(), Host: host.Str(), } filters = append(filters, filter) } } if obj.init == nil { return nil, funcs.ErrCantSpeculate } list := types.NewList(obj.Info().Sig.Out) // collectFuncOutType if len(filters) == 0 { // If we have no filters, it means we're matching on nothing, // which happens if we've pre-filtered away all the resources // that we'd want to collect, so here we return absolutely zero! return list, nil } resOutput, err := obj.init.World.ResCollect(ctx, filters) if err != nil { return nil, err } for _, x := range resOutput { // programming error if any of these error... if x.Kind != kind { return nil, fmt.Errorf("unexpected kind: %s", x.Kind) } if x.Name == "" { return nil, fmt.Errorf("unexpected empty name") } if x.Host == "" { return nil, fmt.Errorf("unexpected empty host") } if x.Host == "*" { // safety check return nil, fmt.Errorf("unexpected star host") } if x.Data == "" { return nil, fmt.Errorf("unexpected empty data") } name := &types.StrValue{V: x.Name} host := &types.StrValue{V: x.Host} // from data := &types.StrValue{V: x.Data} st := types.NewStruct(types.NewType(collectFuncOutStruct)) if err := st.Set(collectFuncOutFieldName, name); err != nil { return nil, errwrap.Wrapf(err, "struct could not add field `%s`, val: `%s`", collectFuncOutFieldName, name) } if err := st.Set(collectFuncOutFieldHost, host); err != nil { return nil, errwrap.Wrapf(err, "struct could not add field `%s`, val: `%s`", collectFuncOutFieldHost, host) } if err := st.Set(collectFuncOutFieldData, data); err != nil { return nil, errwrap.Wrapf(err, "struct could not add field `%s`, val: `%s`", collectFuncOutFieldData, data) } if err := list.Add(st); err != nil { // XXX: improve perf of Add return nil, err } } return list, nil // put struct into interface type }