engine, lang: Modern exported resources
I've been waiting to write this patch for a long time. I firmly believe that the idea of "exported resources" was truly a brilliant one, but which was never even properly understood by its original inventors! This patch set aims to show how it should have been done. The main differences are: * Real-time modelling, since "once per run" makes no sense. * Filter with code/functions not language syntax. * Directed exporting to limit the intended recipients. The next step is to add more "World" reading and filtering functions to make it easy and expressive to make your selection of resources to collect!
This commit is contained in:
James Shubin
@@ -35,11 +35,12 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/purpleidea/mgmt/engine"
|
||||
engineUtil "github.com/purpleidea/mgmt/engine/util"
|
||||
"github.com/purpleidea/mgmt/etcd/interfaces"
|
||||
"github.com/purpleidea/mgmt/util"
|
||||
"github.com/purpleidea/mgmt/util/errwrap"
|
||||
|
||||
etcd "go.etcd.io/etcd/client/v3"
|
||||
clientv3Util "go.etcd.io/etcd/client/v3/clientv3util"
|
||||
//pb "go.etcd.io/etcd/api/v3/etcdserverpb"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -50,94 +51,26 @@ const (
|
||||
// change.
|
||||
// TODO: Filter our watch (on the server side if possible) based on the
|
||||
// collection prefixes and filters that we care about...
|
||||
func WatchResources(ctx context.Context, client interfaces.Client) (chan error, error) {
|
||||
path := fmt.Sprintf("%s/exported/", ns)
|
||||
// XXX: filter based on kind as well, we don't do that currently... See:
|
||||
// https://github.com/etcd-io/etcd/issues/19667
|
||||
func WatchResources(ctx context.Context, client interfaces.Client, hostname, kind string) (chan error, error) {
|
||||
// key structure is $NS/exported/$hostname:to/$hostname:from/$kind/$name = $data
|
||||
// TODO: support the star (*) hostname matching catch-all?
|
||||
path := fmt.Sprintf("%s/exported/%s/", ns, hostname)
|
||||
return client.Watcher(ctx, path, etcd.WithPrefix())
|
||||
}
|
||||
|
||||
// SetResources exports all of the resources which we pass in to etcd.
|
||||
func SetResources(ctx context.Context, client interfaces.Client, hostname string, resourceList []engine.Res) error {
|
||||
// key structure is $NS/exported/$hostname/resources/$uid = $data
|
||||
|
||||
var kindFilter []string // empty to get from everyone
|
||||
hostnameFilter := []string{hostname}
|
||||
// this is not a race because we should only be reading keys which we
|
||||
// set, and there should not be any contention with other hosts here!
|
||||
originals, err := GetResources(ctx, client, hostnameFilter, kindFilter)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(originals) == 0 && len(resourceList) == 0 { // special case of no add or del
|
||||
return nil
|
||||
}
|
||||
|
||||
ifs := []etcd.Cmp{} // list matching the desired state
|
||||
ops := []etcd.Op{} // list of ops in this transaction
|
||||
for _, res := range resourceList {
|
||||
if res.Kind() == "" {
|
||||
return fmt.Errorf("empty kind: %s", res.Name())
|
||||
}
|
||||
uid := fmt.Sprintf("%s/%s", res.Kind(), res.Name())
|
||||
path := fmt.Sprintf("%s/exported/%s/resources/%s", ns, hostname, uid)
|
||||
if data, err := engineUtil.ResToB64(res); err == nil {
|
||||
ifs = append(ifs, etcd.Compare(etcd.Value(path), "=", data)) // desired state
|
||||
ops = append(ops, etcd.OpPut(path, data))
|
||||
} else {
|
||||
return fmt.Errorf("can't convert to B64: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
match := func(res engine.Res, resourceList []engine.Res) bool { // helper lambda
|
||||
for _, x := range resourceList {
|
||||
if res.Kind() == x.Kind() && res.Name() == x.Name() {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
hasDeletes := false
|
||||
// delete old, now unused resources here...
|
||||
for _, res := range originals {
|
||||
if res.Kind() == "" {
|
||||
return fmt.Errorf("empty kind: %s", res.Name())
|
||||
}
|
||||
uid := fmt.Sprintf("%s/%s", res.Kind(), res.Name())
|
||||
path := fmt.Sprintf("%s/exported/%s/resources/%s", ns, hostname, uid)
|
||||
|
||||
if match(res, resourceList) { // if we match, no need to delete!
|
||||
continue
|
||||
}
|
||||
|
||||
ops = append(ops, etcd.OpDelete(path))
|
||||
|
||||
hasDeletes = true
|
||||
}
|
||||
|
||||
// if everything is already correct, do nothing, otherwise, run the ops!
|
||||
// it's important to do this in one transaction, and atomically, because
|
||||
// this way, we only generate one watch event, and only when it's needed
|
||||
if hasDeletes { // always run, ifs don't matter
|
||||
_, err = client.Txn(ctx, nil, ops, nil) // TODO: does this run? it should!
|
||||
} else {
|
||||
_, err = client.Txn(ctx, ifs, nil, ops) // TODO: do we need to look at response?
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// GetResources collects all of the resources which match a filter from etcd. If
|
||||
// the kindfilter or hostnameFilter is empty, then it assumes no filtering...
|
||||
// TODO: Expand this with a more powerful filter based on what we eventually
|
||||
// support in our collect DSL. Ideally a server side filter like WithFilter()
|
||||
// could do this if the pattern was $NS/exported/$kind/$hostname/$uid = $data.
|
||||
func GetResources(ctx context.Context, client interfaces.Client, hostnameFilter, kindFilter []string) ([]engine.Res, error) {
|
||||
// key structure is $NS/exported/$hostname/resources/$uid = $data
|
||||
// GetResources reads the resources sent to the input hostname, and also applies
|
||||
// the filters to ensure we get a limited selection.
|
||||
// XXX: We'd much rather filter server side if etcd had better filtering API's.
|
||||
// See: https://github.com/etcd-io/etcd/issues/19667
|
||||
func GetResources(ctx context.Context, client interfaces.Client, hostname string, filters []*engine.ResFilter) ([]*engine.ResOutput, error) {
|
||||
// key structure is $NS/exported/$hostname:to/$hostname:from/$kind/$name = $data
|
||||
path := fmt.Sprintf("%s/exported/", ns)
|
||||
resourceList := []engine.Res{}
|
||||
output := []*engine.ResOutput{}
|
||||
keyMap, err := client.Get(ctx, path, etcd.WithPrefix(), etcd.WithSort(etcd.SortByKey, etcd.SortAscend))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not get resources: %v", err)
|
||||
return nil, errwrap.Wrapf(err, "could not get resources")
|
||||
}
|
||||
for key, val := range keyMap {
|
||||
if !strings.HasPrefix(key, path) { // sanity check
|
||||
@@ -145,35 +78,118 @@ func GetResources(ctx context.Context, client interfaces.Client, hostnameFilter,
|
||||
}
|
||||
|
||||
str := strings.Split(key[len(path):], "/")
|
||||
if len(str) != 4 {
|
||||
return nil, fmt.Errorf("unexpected chunk count")
|
||||
if len(str) < 4 {
|
||||
return nil, fmt.Errorf("unexpected chunk count of: %d", len(str))
|
||||
}
|
||||
hostname, r, kind, name := str[0], str[1], str[2], str[3]
|
||||
if r != "resources" {
|
||||
return nil, fmt.Errorf("unexpected chunk pattern")
|
||||
// The name may contain slashes, so join all those pieces back!
|
||||
hostnameTo, hostnameFrom, kind, name := str[0], str[1], str[2], strings.Join(str[3:], "/")
|
||||
if hostnameTo == "" || hostnameFrom == "" {
|
||||
return nil, fmt.Errorf("unexpected empty hostname")
|
||||
}
|
||||
if kind == "" {
|
||||
return nil, fmt.Errorf("unexpected kind chunk")
|
||||
return nil, fmt.Errorf("unexpected empty kind")
|
||||
}
|
||||
if name == "" { // TODO: should I check this?
|
||||
if name == "" {
|
||||
return nil, fmt.Errorf("unexpected empty name")
|
||||
}
|
||||
// FIXME: ideally this would be a server side filter instead!
|
||||
if len(hostnameFilter) > 0 && !util.StrInList(hostname, hostnameFilter) {
|
||||
|
||||
// XXX: Do we want to include this catch-all match?
|
||||
if hostnameTo != hostname && hostnameTo != "*" { // star is any
|
||||
continue
|
||||
}
|
||||
|
||||
// FIXME: ideally this would be a server side filter instead!
|
||||
if len(kindFilter) > 0 && !util.StrInList(kind, kindFilter) {
|
||||
continue
|
||||
// TODO: I'd love to avoid this O(N^2) matching if possible...
|
||||
for _, filter := range filters {
|
||||
if err := filter.Match(kind, name, hostnameFrom); err != nil {
|
||||
continue // did not match
|
||||
}
|
||||
}
|
||||
|
||||
if res, err := engineUtil.B64ToRes(val); err == nil {
|
||||
//obj.Logf("Get: (Hostname, Kind, Name): (%s, %s, %s)", hostname, kind, name)
|
||||
resourceList = append(resourceList, res)
|
||||
} else {
|
||||
return nil, fmt.Errorf("can't convert from B64: %v", err)
|
||||
ro := &engine.ResOutput{
|
||||
Kind: kind,
|
||||
Name: name,
|
||||
Host: hostnameFrom, // from this host
|
||||
Data: val, // encoded res data
|
||||
}
|
||||
output = append(output, ro)
|
||||
}
|
||||
return resourceList, nil
|
||||
|
||||
return output, nil
|
||||
}
|
||||
|
||||
// SetResources stores some resource data for export in etcd. It returns an
|
||||
// error if anything goes wrong. If it didn't need to make a changes because the
|
||||
// data was already correct in the database, it returns (true, nil). Otherwise
|
||||
// it returns (false, nil).
|
||||
func SetResources(ctx context.Context, client interfaces.Client, hostname string, resourceExports []*engine.ResExport) (bool, error) {
|
||||
// key structure is $NS/exported/$hostname:to/$hostname:from/$kind/$name = $data
|
||||
|
||||
// XXX: We run each export one at a time, because there's a bug if we
|
||||
// group them, See: https://github.com/etcd-io/etcd/issues/19678
|
||||
b := true
|
||||
for _, re := range resourceExports {
|
||||
ifs := []etcd.Cmp{} // list matching the desired state
|
||||
thn := []etcd.Op{} // list of ops in this transaction (then)
|
||||
els := []etcd.Op{} // list of ops in this transaction (else)
|
||||
|
||||
host := re.Host
|
||||
if host == "" {
|
||||
host = "*" // XXX: use whatever means "all"
|
||||
}
|
||||
|
||||
path := fmt.Sprintf("%s/exported/%s/%s/%s/%s", ns, host, hostname, re.Kind, re.Name)
|
||||
ifs = append(ifs, etcd.Compare(etcd.Value(path), "=", re.Data))
|
||||
els = append(els, etcd.OpPut(path, re.Data))
|
||||
|
||||
// it's important to do this in one transaction, and atomically, because
|
||||
// this way, we only generate one watch event, and only when it's needed
|
||||
out, err := client.Txn(ctx, ifs, thn, els)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
b = b && out.Succeeded // collect the true/false responses...
|
||||
}
|
||||
|
||||
// false means something changed
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// DelResources deletes some exported resource data from etcd. It returns an
|
||||
// error if anything goes wrong. If it didn't need to make a changes because the
|
||||
// data was already correct in the database, it returns (true, nil). Otherwise
|
||||
// it returns (false, nil).
|
||||
func DelResources(ctx context.Context, client interfaces.Client, hostname string, resourceDeletes []*engine.ResDelete) (bool, error) {
|
||||
// key structure is $NS/exported/$hostname:to/$hostname:from/$kind/$name = $data
|
||||
|
||||
// XXX: We run each delete one at a time, because there's a bug if we
|
||||
// group them, See: https://github.com/etcd-io/etcd/issues/19678
|
||||
b := true
|
||||
for _, rd := range resourceDeletes {
|
||||
|
||||
ifs := []etcd.Cmp{} // list matching the desired state
|
||||
thn := []etcd.Op{} // list of ops in this transaction (then)
|
||||
els := []etcd.Op{} // list of ops in this transaction (else)
|
||||
|
||||
host := rd.Host
|
||||
if host == "" {
|
||||
host = "*" // XXX: use whatever means "all"
|
||||
}
|
||||
|
||||
path := fmt.Sprintf("%s/exported/%s/%s/%s/%s", ns, host, hostname, rd.Kind, rd.Name)
|
||||
ifs = append(ifs, clientv3Util.KeyExists(path))
|
||||
thn = append(thn, etcd.OpDelete(path))
|
||||
|
||||
// it's important to do this in one transaction, and atomically, because
|
||||
// this way, we only generate one watch event, and only when it's needed
|
||||
out, err := client.Txn(ctx, ifs, thn, els)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
b = b && out.Succeeded // collect the true/false responses...
|
||||
}
|
||||
|
||||
// false means something changed
|
||||
return b, nil
|
||||
}
|
||||
|
||||
@@ -167,23 +167,32 @@ func (obj *World) AddDeploy(ctx context.Context, id uint64, hash, pHash string,
|
||||
|
||||
// ResWatch returns a channel which spits out events on possible exported
|
||||
// resource changes.
|
||||
func (obj *World) ResWatch(ctx context.Context) (chan error, error) {
|
||||
return resources.WatchResources(ctx, obj.client)
|
||||
func (obj *World) ResWatch(ctx context.Context, kind string) (chan error, error) {
|
||||
return resources.WatchResources(ctx, obj.client, obj.init.Hostname, kind)
|
||||
}
|
||||
|
||||
// ResExport exports a list of resources under our hostname namespace.
|
||||
// Subsequent calls replace the previously set collection atomically.
|
||||
func (obj *World) ResExport(ctx context.Context, resourceList []engine.Res) error {
|
||||
return resources.SetResources(ctx, obj.client, obj.init.Hostname, resourceList)
|
||||
}
|
||||
|
||||
// ResCollect gets the collection of exported resources which match the filter.
|
||||
// ResCollect gets the collection of exported resources which match the filters.
|
||||
// It does this atomically so that a call always returns a complete collection.
|
||||
func (obj *World) ResCollect(ctx context.Context, hostnameFilter, kindFilter []string) ([]engine.Res, error) {
|
||||
// XXX: should we be restricted to retrieving resources that were
|
||||
// exported with a tag that allows or restricts our hostname? We could
|
||||
// enforce that here if the underlying API supported it... Add this?
|
||||
return resources.GetResources(ctx, obj.client, hostnameFilter, kindFilter)
|
||||
func (obj *World) ResCollect(ctx context.Context, filters []*engine.ResFilter) ([]*engine.ResOutput, error) {
|
||||
return resources.GetResources(ctx, obj.client, obj.init.Hostname, filters)
|
||||
}
|
||||
|
||||
// ResExport stores a number of resources in the world storage system. The
|
||||
// individual records should not be updated if they are identical to what is
|
||||
// already present. (This is to prevent unnecessary events.) If this makes no
|
||||
// changes, it returns (true, nil). If it makes a change, then it returns
|
||||
// (false, nil). On any error we return (false, err). It stores the exports
|
||||
// under our hostname namespace. Subsequent calls do NOT replace the previously
|
||||
// set collection.
|
||||
func (obj *World) ResExport(ctx context.Context, resourceExports []*engine.ResExport) (bool, error) {
|
||||
return resources.SetResources(ctx, obj.client, obj.init.Hostname, resourceExports)
|
||||
}
|
||||
|
||||
// ResDelete deletes a number of resources in the world storage system. If this
|
||||
// doesn't delete, it returns (true, nil). If it makes a delete, then it returns
|
||||
// (false, nil). On any error we return (false, err).
|
||||
func (obj *World) ResDelete(ctx context.Context, resourceDeletes []*engine.ResDelete) (bool, error) {
|
||||
return resources.DelResources(ctx, obj.client, obj.init.Hostname, resourceDeletes)
|
||||
}
|
||||
|
||||
// IdealClusterSizeWatch returns a stream of errors anytime the cluster-wide
|
||||
|
||||
Reference in New Issue
Block a user