diff --git a/docs/documentation.md b/docs/documentation.md index af02ee14..e44e4588 100644 --- a/docs/documentation.md +++ b/docs/documentation.md @@ -182,6 +182,7 @@ parameter with the [Noop](#Noop) resource. * [Exec](#Exec): Execute shell commands on the system. * [File](#File): Manage files and directories. * [Hostname](#Hostname): Manages the hostname on the system. +* [KV](#KV): Set a key value pair in our shared world database. * [Msg](#Msg): Send log messages. * [Noop](#Noop): A simple resource that does nothing. * [Nspawn](#Nspawn): Manage systemd-machined nspawn containers. @@ -268,6 +269,30 @@ The pretty hostname is a free-form UTF8 host name for presentation to the user. Hostname is the fallback value for all 3 fields above, if only `hostname` is specified, it will set all 3 fields to this value. +### KV + +The KV resource sets a key and value pair in the global world database. This is +quite useful for setting a flag after a number of resources have run. It will +ignore database updates to the value that are greater in compare order than the +requested key if the `SkipLessThan` parameter is set to true. If we receive a +refresh, then the stored value will be reset to the requested value even if the +stored value is greater. + +#### Key +The string key used to store the key. + +#### Value +The string value to set. This can also be set via Send/Recv. + +#### SkipLessThan +If this parameter is set to `true`, then it will ignore updating the value as +long as the database versions are greater than the requested value. The compare +operation used is based on the `SkipCmpStyle` parameter. + +#### SkipCmpStyle +By default this converts the string values to integers and compares them as you +would expect. + ### Msg The msg resource sends messages to the main log, or an external service such diff --git a/examples/kv1.yaml b/examples/kv1.yaml new file mode 100644 index 00000000..a97bafbd --- /dev/null +++ b/examples/kv1.yaml @@ -0,0 +1,8 @@ +--- +graph: mygraph +resources: + kv: + - name: kv1 + key: "hello" + value: "world" +edges: [] diff --git a/examples/kv2.yaml b/examples/kv2.yaml new file mode 100644 index 00000000..ec0e052f --- /dev/null +++ b/examples/kv2.yaml @@ -0,0 +1,7 @@ +--- +graph: mygraph +resources: + kv: + - name: kv1 + key: "iamdeleted" +edges: [] diff --git a/examples/kv3.yaml b/examples/kv3.yaml new file mode 100644 index 00000000..265df5b3 --- /dev/null +++ b/examples/kv3.yaml @@ -0,0 +1,9 @@ +--- +graph: mygraph +resources: + kv: + - name: kv1 + key: "stage" + value: "3" + skiplessthan: true +edges: [] diff --git a/examples/kv4.yaml b/examples/kv4.yaml new file mode 100644 index 00000000..3cc7f186 --- /dev/null +++ b/examples/kv4.yaml @@ -0,0 +1,31 @@ +--- +graph: mygraph +resources: + kv: + - name: kv1 + key: "stage" + value: "1" + skiplessthan: true + - name: kv2 + key: "stage" + value: "2" + skiplessthan: true + - name: kv3 + key: "stage" + value: "3" + skiplessthan: true +edges: +- name: e1 + from: + kind: kv + name: kv1 + to: + kind: kv + name: kv2 +- name: e2 + from: + kind: kv + name: kv2 + to: + kind: kv + name: kv3 diff --git a/resources/kv.go b/resources/kv.go new file mode 100644 index 00000000..0a438438 --- /dev/null +++ b/resources/kv.go @@ -0,0 +1,304 @@ +// Mgmt +// Copyright (C) 2013-2017+ James Shubin and the project contributors +// Written by James Shubin and the project contributors +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package resources + +import ( + "encoding/gob" + "fmt" + "log" + "strconv" + + errwrap "github.com/pkg/errors" +) + +func init() { + gob.Register(&KVRes{}) +} + +// KVResSkipCmpStyle represents the different styles of comparison when using SkipLessThan. +type KVResSkipCmpStyle int + +// These are the different allowed comparison styles. Most folks will want SkipCmpStyleInt. +const ( + SkipCmpStyleInt KVResSkipCmpStyle = iota + SkipCmpStyleString +) + +// KVRes is a resource which writes a key/value pair into cluster wide storage. +// It will ensure that the key is set to the requested value. The one exception +// is that if you use the SkipLessThan parameter, then it will only replace the +// stored value with the requested value if it is greater than that stored one. +// This allows the KV resource to be used in fast acting, finite state machines +// which have monotonically increasing state values that represent progression. +// The one exception is that when this resource receives a refresh signal, then +// it will set the value to be the exact one if they are not identical already. +type KVRes struct { + BaseRes `yaml:",inline"` + Key string `yaml:"key"` // key to set + Value *string `yaml:"value"` // value to set (nil to delete) + SkipLessThan bool `yaml:"skiplessthan"` // skip updates as long as stored value is greater + SkipCmpStyle KVResSkipCmpStyle `yaml:"skipcmpstyle"` // how to do the less than cmp + // TODO: does it make sense to have different backends here? (eg: local) +} + +// Default returns some sensible defaults for this resource. +func (obj *KVRes) Default() Res { + return &KVRes{ + BaseRes: BaseRes{ + MetaParams: DefaultMetaParams, // force a default + }, + } +} + +// Validate if the params passed in are valid data. +// FIXME: This will catch most issues unless data is passed in after Init with +// the Send/Recv mechanism. Should the engine re-call Validate after Send/Recv? +func (obj *KVRes) Validate() error { + if obj.Key == "" { + return fmt.Errorf("key must not be empty") + } + if obj.SkipLessThan { + if obj.SkipCmpStyle != SkipCmpStyleInt && obj.SkipCmpStyle != SkipCmpStyleString { + return fmt.Errorf("the SkipCmpStyle of %v is invalid", obj.SkipCmpStyle) + } + + if v := obj.Value; obj.SkipCmpStyle == SkipCmpStyleInt && v != nil { + if _, err := strconv.Atoi(*v); err != nil { + return fmt.Errorf("the set value of %v can't convert to int", v) + } + } + } + return obj.BaseRes.Validate() +} + +// Init initializes the resource. +func (obj *KVRes) Init() error { + obj.BaseRes.kind = "KV" + return obj.BaseRes.Init() // call base init, b/c we're overriding +} + +// Watch is the primary listener for this resource and it outputs events. +func (obj *KVRes) Watch() error { + + // notify engine that we're running + if err := obj.Running(); err != nil { + return err // bubble up a NACK... + } + + ch := obj.Data().World.StrWatch(obj.Key) // get possible events! + + var send = false // send event? + var exit *error + for { + select { + // NOTE: this part is very similar to the file resource code + case err, ok := <-ch: + if !ok { // channel shutdown + return nil + } + if err != nil { + return errwrap.Wrapf(err, "unknown %s[%s] watcher error", obj.Kind(), obj.GetName()) + } + if obj.Data().Debug { + log.Printf("%s[%s]: Event!", obj.Kind(), obj.GetName()) + } + send = true + obj.StateOK(false) // dirty + + case event := <-obj.Events(): + // we avoid sending events on unpause + if exit, send = obj.ReadEvent(event); exit != nil { + return *exit // exit + } + } + + // do all our event sending all together to avoid duplicate msgs + if send { + send = false + obj.Event() + } + } +} + +// lessThanCheck checks for less than validity. +func (obj *KVRes) lessThanCheck(value string) (checkOK bool, err error) { + + v := *obj.Value + if value == v { // redundant check for safety + return true, nil + } + + var refresh = obj.Refresh() // do we have a pending reload to apply? + if !obj.SkipLessThan || refresh { // update lessthan on refresh + return false, nil + } + + switch obj.SkipCmpStyle { + case SkipCmpStyleInt: + intValue, err := strconv.Atoi(value) + if err != nil { + // NOTE: We don't error here since we're going to write + // over the value anyways. It could be from an old run! + return false, nil // value is bad (old/corrupt), fix it + } + if vint, err := strconv.Atoi(v); err != nil { + return false, errwrap.Wrapf(err, "can't convert %v to int", v) + } else if vint < intValue { + return true, nil + } + + case SkipCmpStyleString: + if v < value { // weird way to cmp, but valid + return true, nil + } + + default: + return false, fmt.Errorf("unmatches SkipCmpStyle style %v", obj.SkipCmpStyle) + } + + return false, nil +} + +// CheckApply method for Password resource. Does nothing, returns happy! +func (obj *KVRes) CheckApply(apply bool) (checkOK bool, err error) { + log.Printf("%s[%s]: CheckApply(%t)", obj.Kind(), obj.GetName(), apply) + + if val, exists := obj.Recv["Value"]; exists && val.Changed { + // if we received on Value, and it changed, wooo, nothing to do. + log.Printf("CheckApply: `Value` was updated!") + } + + hostname := obj.Data().Hostname // me + keyMap, err := obj.Data().World.StrGet(obj.Key) + if err != nil { + return false, errwrap.Wrapf(err, "check error during StrGet") + } + + if value, ok := keyMap[hostname]; ok && obj.Value != nil { + if value == *obj.Value { + return true, nil + } + + if c, err := obj.lessThanCheck(value); err != nil { + return false, err + } else if c { + return true, nil + } + + } else if !ok && obj.Value == nil { + return true, nil // nothing to delete, we're good! + + } else if ok && obj.Value == nil { // delete + err := obj.Data().World.StrDel(obj.Key) + return false, errwrap.Wrapf(err, "apply error during StrDel") + } + + if !apply { + return false, nil + } + + if err := obj.Data().World.StrSet(obj.Key, *obj.Value); err != nil { + return false, errwrap.Wrapf(err, "apply error during StrSet") + } + + return false, nil +} + +// KVUID is the UID struct for KVRes. +type KVUID struct { + BaseUID + name string +} + +// AutoEdges returns the AutoEdge interface. In this case no autoedges are used. +func (obj *KVRes) AutoEdges() AutoEdge { + return nil +} + +// UIDs includes all params to make a unique identification of this object. +// Most resources only return one, although some resources can return multiple. +func (obj *KVRes) UIDs() []ResUID { + x := &KVUID{ + BaseUID: BaseUID{name: obj.GetName(), kind: obj.Kind()}, + name: obj.Name, + } + return []ResUID{x} +} + +// GroupCmp returns whether two resources can be grouped together or not. +func (obj *KVRes) GroupCmp(r Res) bool { + _, ok := r.(*KVRes) + if !ok { + return false + } + return false // TODO: this is doable! + // TODO: it could be useful to group our writes and watches! +} + +// Compare two resources and return if they are equivalent. +func (obj *KVRes) Compare(res Res) bool { + switch res.(type) { + // we can only compare KVRes to others of the same resource + case *KVRes: + res := res.(*KVRes) + if !obj.BaseRes.Compare(res) { // call base Compare + return false + } + + if obj.Key != res.Key { + return false + } + if (obj.Value == nil) != (res.Value == nil) { // xor + return false + } + if obj.Value != nil && res.Value != nil { + if *obj.Value != *res.Value { // compare the strings + return false + } + } + if obj.SkipLessThan != res.SkipLessThan { + return false + } + if obj.SkipCmpStyle != res.SkipCmpStyle { + return false + } + default: + return false + } + return true +} + +// UnmarshalYAML is the custom unmarshal handler for this struct. +// It is primarily useful for setting the defaults. +func (obj *KVRes) UnmarshalYAML(unmarshal func(interface{}) error) error { + type rawRes KVRes // indirection to avoid infinite recursion + + def := obj.Default() // get the default + res, ok := def.(*KVRes) // put in the right format + if !ok { + return fmt.Errorf("could not convert to KVRes") + } + raw := rawRes(*res) // convert; the defaults go here + + if err := unmarshal(&raw); err != nil { + return err + } + + *obj = KVRes(raw) // restore from indirection with type conversion! + return nil +} diff --git a/yamlgraph/gconfig.go b/yamlgraph/gconfig.go index 95e55674..94904a8b 100644 --- a/yamlgraph/gconfig.go +++ b/yamlgraph/gconfig.go @@ -59,6 +59,7 @@ type Resources struct { Exec []*resources.ExecRes `yaml:"exec"` File []*resources.FileRes `yaml:"file"` Hostname []*resources.HostnameRes `yaml:"hostname"` + KV []*resources.KVRes `yaml:"kv"` Msg []*resources.MsgRes `yaml:"msg"` Noop []*resources.NoopRes `yaml:"noop"` Nspawn []*resources.NspawnRes `yaml:"nspawn"`