diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index a9b0d799..67dfbd0c 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -306,7 +306,8 @@ The nspawn resource is used to manage systemd-machined style containers. ###Password -The password resource can generate a random string to be used as a password. +The password resource can generate a random string to be used as a password. It +will re-generate the password if it receives a refresh notification. ###Pkg diff --git a/examples/lib/libmgmt3.go b/examples/lib/libmgmt3.go index b0eb190f..0e98ca93 100644 --- a/examples/lib/libmgmt3.go +++ b/examples/lib/libmgmt3.go @@ -58,11 +58,25 @@ func (obj *MyGAPI) Graph() (*pgraph.Graph, error) { g := pgraph.NewGraph(obj.Name) + content := "Delete me to trigger a notification!\n" + f0 := &resources.FileRes{ + BaseRes: resources.BaseRes{ + Name: "README", + }, + Path: "/tmp/mgmt/README", + Content: &content, + State: "present", + } + + v0 := pgraph.NewVertex(f0) + g.AddVertex(v0) + p1 := &resources.PasswordRes{ BaseRes: resources.BaseRes{ Name: "password1", }, - Length: 8, // generated string will have this many characters + Length: 8, // generated string will have this many characters + Saved: true, // this causes passwords to be stored in plain text! } v1 := pgraph.NewVertex(p1) g.AddVertex(v1) @@ -71,11 +85,11 @@ func (obj *MyGAPI) Graph() (*pgraph.Graph, error) { BaseRes: resources.BaseRes{ Name: "file1", // send->recv! - Recv: map[string]resources.Send{ - "Content": resources.Send{Res: p1, Key: "Password"}, + Recv: map[string]*resources.Send{ + "Content": &resources.Send{Res: p1, Key: "Password"}, }, }, - Path: "/tmp/mgmt/f1", + Path: "/tmp/mgmt/secret", //Content: p1.Password, // won't work State: "present", } @@ -83,17 +97,21 @@ func (obj *MyGAPI) Graph() (*pgraph.Graph, error) { v2 := pgraph.NewVertex(f1) g.AddVertex(v2) - s1 := &resources.SvcRes{ + n1 := &resources.NoopRes{ BaseRes: resources.BaseRes{ - Name: "purpleidea", + Name: "noop1", }, - State: "stopped", } - v3 := pgraph.NewVertex(s1) + v3 := pgraph.NewVertex(n1) g.AddVertex(v3) + e0 := pgraph.NewEdge("e0") + e0.Notify = true // send a notification from v0 to v1 + g.AddEdge(v0, v1, e0) + g.AddEdge(v1, v2, pgraph.NewEdge("e1")) + e2 := pgraph.NewEdge("e2") e2.Notify = true // send a notification from v2 to v3 g.AddEdge(v2, v3, e2) @@ -150,7 +168,9 @@ func Run() error { obj := &mgmt.Main{} obj.Program = "libmgmt" // TODO: set on compilation obj.Version = "0.0.1" // TODO: set on compilation - obj.TmpPrefix = true + obj.TmpPrefix = true // disable for easy debugging + //prefix := "/tmp/testprefix/" + //obj.Prefix = &p // enable for easy debugging obj.IdealClusterSize = -1 obj.ConvergedTimeout = -1 obj.Noop = false // FIXME: careful! diff --git a/pgraph/actions.go b/pgraph/actions.go index 37f91750..06b0e69e 100644 --- a/pgraph/actions.go +++ b/pgraph/actions.go @@ -174,9 +174,9 @@ func (g *Graph) Process(v *Vertex) error { obj.SetState(resources.ResStateCheckApply) // connect any senders to receivers and detect if values changed - if changed, err := obj.SendRecv(obj); err != nil { + if updated, err := obj.SendRecv(obj); err != nil { return errwrap.Wrapf(err, "could not SendRecv in Process") - } else if changed { + } else if len(updated) > 0 { obj.StateOK(false) // invalidate cache, mark as dirty } diff --git a/resources/file.go b/resources/file.go index f73e3ca3..34931326 100644 --- a/resources/file.go +++ b/resources/file.go @@ -640,6 +640,18 @@ func (obj *FileRes) contentCheckApply(apply bool) (checkOK bool, _ error) { // input is true. It returns error info and if the state check passed or not. func (obj *FileRes) CheckApply(apply bool) (checkOK bool, _ error) { + // NOTE: all send/recv change notifications *must* be processed before + // there is a possibility of failure in CheckApply. This is because if + // we fail (and possibly run again) the subsequent send->recv transfer + // might not have a new value to copy, and therefore we won't see this + // notification of change. Therefore, it is important to process these + // promptly, if they must not be lost, such as for cache invalidation. + if val, exists := obj.Recv["Content"]; exists && val.Changed { + // if we received on Content, and it changed, invalidate the cache! + log.Printf("contentCheckApply: Invalidating sha256sum of `Content`") + obj.sha256sum = "" // invalidate!! + } + checkOK = true if c, err := obj.contentCheckApply(apply); err != nil { diff --git a/resources/noop.go b/resources/noop.go index 1c6f33be..ed17a8ff 100644 --- a/resources/noop.go +++ b/resources/noop.go @@ -19,6 +19,7 @@ package resources import ( "encoding/gob" + "log" "time" "github.com/purpleidea/mgmt/event" @@ -110,6 +111,9 @@ func (obj *NoopRes) Watch(processChan chan event.Event) error { // CheckApply method for Noop resource. Does nothing, returns happy! func (obj *NoopRes) CheckApply(apply bool) (checkOK bool, err error) { + if obj.Refresh() { + log.Printf("%s[%s]: Received a notification!", obj.Kind(), obj.GetName()) + } return true, nil // state is always okay } diff --git a/resources/password.go b/resources/password.go index d0412bcd..589f6ca5 100644 --- a/resources/password.go +++ b/resources/password.go @@ -21,6 +21,8 @@ import ( "crypto/rand" "encoding/gob" "fmt" + "io/ioutil" + "log" "math/big" "os" "path" @@ -28,6 +30,7 @@ import ( "time" "github.com/purpleidea/mgmt/event" + "github.com/purpleidea/mgmt/recwatch" errwrap "github.com/pkg/errors" ) @@ -45,10 +48,13 @@ const ( type PasswordRes struct { BaseRes `yaml:",inline"` // FIXME: is uint16 too big? - Length uint16 `yaml:"length"` // number of characters to return - Password *string // the generated password + Length uint16 `yaml:"length"` // number of characters to return + Saved bool // this caches the password in the clear locally + CheckRecovery bool // recovery from integrity checks by re-generating + Password *string // the generated password, read only, do not set! - path string // the path to local storage + path string // the path to local storage + recWatcher *recwatch.RecWatcher } // NewPasswordRes is a constructor for this resource. It also calls Init() for you. @@ -62,14 +68,34 @@ func NewPasswordRes(name string, length uint16) (*PasswordRes, error) { return obj, obj.Init() } +// Init generates a new password for this resource if one was not provided. It +// will save this into a local file. It will load it back in from previous runs. +func (obj *PasswordRes) Init() error { + obj.BaseRes.kind = "Password" // must be set before using VarDir + + dir, err := obj.VarDir("") + if err != nil { + return errwrap.Wrapf(err, "could not get VarDir in Init()") + } + obj.path = path.Join(dir, "password") // return a unique file + + return obj.BaseRes.Init() // call base init, b/c we're overriding +} + +// Validate if the params passed in are valid data. +// FIXME: where should this get called ? +func (obj *PasswordRes) Validate() error { + return nil +} + func (obj *PasswordRes) read() (string, error) { file, err := os.Open(obj.path) // open a handle to read the file if err != nil { - return "", errwrap.Wrapf(err, "could not read password") + return "", err } defer file.Close() - data := make([]byte, obj.Length+uint16(len(newline))) // data + newline - if _, err := file.Read(data); err != nil { + data, err := ioutil.ReadAll(file) + if err != nil { return "", errwrap.Wrapf(err, "could not read from file") } return strings.TrimSpace(string(data)), nil @@ -81,7 +107,11 @@ func (obj *PasswordRes) write(password string) (int, error) { return -1, errwrap.Wrapf(err, "can't create file") } defer file.Close() - return file.Write([]byte(password + newline)) + var c int + if c, err = file.Write([]byte(password + newline)); err != nil { + return c, errwrap.Wrapf(err, "can't write file") + } + return c, file.Sync() } // generate generates a new password. @@ -113,6 +143,14 @@ func (obj *PasswordRes) generate() (string, error) { // check validates a stored password string func (obj *PasswordRes) check(value string) error { length := uint16(len(value)) + + if !obj.Saved && length == 0 { // expecting an empty string + return nil + } + if !obj.Saved && length != 0 { // should have no stored password + return fmt.Errorf("Expected empty token only!") + } + if length != obj.Length { return fmt.Errorf("String length is not %d", obj.Length) } @@ -129,71 +167,6 @@ Loop: return nil } -// Init generates a new password for this resource if one was not provided. It -// will save this into a local file. It will load it back in from previous runs. -func (obj *PasswordRes) Init() error { - // XXX: eventually store a hash instead of the plain text! we might want - // to generate a new value on fresh run if the downstream resource needs - // an update (triggers a backpoke?) this is a POC for send/recv for now. - obj.BaseRes.kind = "Password" // must be set before using VarDir - - dir, err := obj.VarDir("") - if err != nil { - return errwrap.Wrapf(err, "could not get VarDir in Init()") - } - - obj.path = path.Join(dir, "password") // return a unique file - password := "" - if _, err := os.Stat(obj.path); err != nil { // probably doesn't exist - if !os.IsNotExist(err) { - return errwrap.Wrapf(err, "unknown stat error") - } - - // generate password and store it in the file - if obj.Password != nil { - password = *obj.Password // reuse what we've got - } else { - var err error - if password, err = obj.generate(); err != nil { // generate one! - return errwrap.Wrapf(err, "could not init password") - } - } - - // store it to disk - if _, err := obj.write(password); err != nil { - return errwrap.Wrapf(err, "can't write to file") - } - - } else { // must exist already! - - password, err := obj.read() - if err != nil { - return errwrap.Wrapf(err, "could not read password") - } - if err := obj.check(password); err != nil { - return errwrap.Wrapf(err, "check failed") - } - - if p := obj.Password; p != nil && *p != password { - // stored password isn't consistent with memory - if _, err := obj.write(*p); err != nil { - return errwrap.Wrapf(err, "consistency overwrite failed") - } - password = *p // use the copy from the resource - } - } - - obj.Password = &password // save in memory - - return obj.BaseRes.Init() // call base init, b/c we're overriding -} - -// Validate if the params passed in are valid data. -// FIXME: where should this get called ? -func (obj *PasswordRes) Validate() error { - return nil -} - // Watch is the primary listener for this resource and it outputs events. func (obj *PasswordRes) Watch(processChan chan event.Event) error { if obj.IsWatching() { @@ -213,11 +186,30 @@ func (obj *PasswordRes) Watch(processChan chan event.Event) error { return time.After(time.Duration(500) * time.Millisecond) // 1/2 the resolution of converged timeout } + var err error + obj.recWatcher, err = recwatch.NewRecWatcher(obj.path, false) + if err != nil { + return err + } + defer obj.recWatcher.Close() + var send = false // send event? var exit = false for { obj.SetState(ResStateWatching) // reset select { + // NOTE: this part is very similar to the file resource code + case event, ok := <-obj.recWatcher.Events(): + if !ok { // channel shutdown + return nil + } + cuid.SetConverged(false) + if err := event.Error; err != nil { + return errwrap.Wrapf(err, "Unknown %s[%s] watcher error", obj.Kind(), obj.GetName()) + } + send = true + obj.StateOK(false) // dirty + case event := <-obj.Events(): cuid.SetConverged(false) // we avoid sending events on unpause @@ -247,7 +239,83 @@ func (obj *PasswordRes) Watch(processChan chan event.Event) error { // CheckApply method for Password resource. Does nothing, returns happy! func (obj *PasswordRes) CheckApply(apply bool) (checkOK bool, err error) { - return true, nil + + var refresh = obj.Refresh() // do we have a pending reload to apply? + var exists = true // does the file (aka the token) exist? + var generate bool // do we need to generate a new password? + var write bool // do we need to write out to disk? + + password, err := obj.read() // password might be empty if just a token + if err != nil { + if !os.IsNotExist(err) { + return false, errwrap.Wrapf(err, "unknown read error") + } + exists = false + } + + if exists { + if err := obj.check(password); err != nil { + if !obj.CheckRecovery { + return false, errwrap.Wrapf(err, "check failed") + } + log.Printf("%s[%s]: Integrity check failed", obj.Kind(), obj.GetName()) + generate = true // okay to build a new one + write = true // make sure to write over the old one + } + } else { // doesn't exist, write one + write = true + } + + // if we previously had !obj.Saved, and now we want it, we re-generate! + if refresh || !exists || (obj.Saved && password == "") { + generate = true + } + + // stored password isn't consistent with memory + if p := obj.Password; obj.Saved && (p != nil && *p != password) { + write = true + } + + if !refresh && exists && !generate && !write { // nothing to do, done! + return true, nil + } + // a refresh was requested, the token doesn't exist, or the check failed + + if !apply { + return false, nil + } + + if generate { + // we'll need to write this out... + if obj.Saved || (!obj.Saved && password != "") { + write = true + } + // generate the actual password + var err error + log.Printf("%s[%s]: Generating new password...", obj.Kind(), obj.GetName()) + if password, err = obj.generate(); err != nil { // generate one! + return false, errwrap.Wrapf(err, "could not generate password") + } + } + + obj.Password = &password // save in memory + + var output string // the string to write out + + // if memory value != value on disk, save it + if write { + if obj.Saved { // save password as clear text + // TODO: would it make sense to encrypt this password? + output = password + } + // write either an empty token, or the password + log.Printf("%s[%s]: Writing password token...", obj.Kind(), obj.GetName()) + if _, err := obj.write(output); err != nil { + return false, errwrap.Wrapf(err, "can't write to file") + } + } + + return false, nil } // PasswordUID is the UID struct for PasswordRes. @@ -275,13 +343,11 @@ func (obj *PasswordRes) GetUIDs() []ResUID { func (obj *PasswordRes) GroupCmp(r Res) bool { _, ok := r.(*PasswordRes) if !ok { - // NOTE: technically we could group a noop into any other - // resource, if that resource knew how to handle it, although, - // since the mechanics of inter-kind resource grouping are - // tricky, avoid doing this until there's a good reason. return false } - return true // noop resources can always be grouped together! + return false // TODO: this is doable, but probably not very useful + // TODO: it could be useful to group our tokens into a single write, and + // as a result, we save inotify watches too! } // Compare two resources and return if they are equivalent. @@ -297,6 +363,17 @@ func (obj *PasswordRes) Compare(res Res) bool { if obj.Name != res.Name { return false } + if obj.Length != res.Length { + return false + } + // TODO: we *could* optimize by allowing CheckApply to move from + // saved->!saved, by removing the file, but not likely worth it! + if obj.Saved != res.Saved { + return false + } + if obj.CheckRecovery != res.CheckRecovery { + return false + } default: return false } diff --git a/resources/refresh.go b/resources/refresh.go index 640de553..74696620 100644 --- a/resources/refresh.go +++ b/resources/refresh.go @@ -19,6 +19,7 @@ package resources import ( "fmt" + "io/ioutil" "os" "strings" @@ -68,12 +69,11 @@ func (obj *DiskBool) Get() (bool, error) { return false, errwrap.Wrapf(err, "could not read token") } defer file.Close() - str := obj.str() - data := make([]byte, len(str)) // data + newline - if _, err := file.Read(data); err != nil { + data, err := ioutil.ReadAll(file) + if err != nil { return false, errwrap.Wrapf(err, "could not read from file") } - return strings.TrimSpace(string(data)) == strings.TrimSpace(str), nil + return strings.TrimSpace(string(data)) == strings.TrimSpace(obj.str()), nil } // Set stores the true boolean value, if no error setting the value occurs. diff --git a/resources/resources.go b/resources/resources.go index 0ceb9d2e..9ce6d0ce 100644 --- a/resources/resources.go +++ b/resources/resources.go @@ -134,10 +134,10 @@ type Base interface { SetState(ResState) DoSend(chan event.Event, string) (bool, error) SendEvent(event.EventName, bool, bool) bool - ReadEvent(*event.Event) (bool, bool) // TODO: optional here? - Refresh() bool // is there a pending refresh to run? - SetRefresh(bool) // set the refresh state of this resource - SendRecv(Res) (bool, error) // send->recv data passing function + ReadEvent(*event.Event) (bool, bool) // TODO: optional here? + Refresh() bool // is there a pending refresh to run? + SetRefresh(bool) // set the refresh state of this resource + SendRecv(Res) (map[string]bool, error) // send->recv data passing function IsStateOK() bool StateOK(b bool) GroupCmp(Res) bool // TODO: is there a better name for this? @@ -164,9 +164,9 @@ type Res interface { // BaseRes is the base struct that gets used in every resource. type BaseRes struct { - Name string `yaml:"name"` - MetaParams MetaParams `yaml:"meta"` // struct of all the metaparams - Recv map[string]Send // mapping of key to receive on from value + Name string `yaml:"name"` + MetaParams MetaParams `yaml:"meta"` // struct of all the metaparams + Recv map[string]*Send // mapping of key to receive on from value kind string events chan event.Event diff --git a/resources/sendrecv.go b/resources/sendrecv.go index 8f209202..38de0d5a 100644 --- a/resources/sendrecv.go +++ b/resources/sendrecv.go @@ -113,20 +113,22 @@ func (obj *BaseRes) ReadEvent(ev *event.Event) (exit, send bool) { type Send struct { Res Res // a handle to the resource which is sending a value Key string // the key in the resource that we're sending + + Changed bool // set to true if this key was updated, read only! } // SendRecv pulls in the sent values into the receive slots. It is called by the // receiver and must be given as input the full resource struct to receive on. -func (obj *BaseRes) SendRecv(res Res) (bool, error) { +func (obj *BaseRes) SendRecv(res Res) (map[string]bool, error) { if global.DEBUG { // NOTE: this could expose private resource data like passwords log.Printf("%s[%s]: SendRecv: %+v", obj.Kind(), obj.GetName(), obj.Recv) } - var changed bool // did we update a value? + var updated = make(map[string]bool) // list of updated keys var err error for k, v := range obj.Recv { - log.Printf("SendRecv: %s[%s].%s <- %s[%s].%s", obj.Kind(), obj.GetName(), k, v.Res.Kind(), v.Res.GetName(), v.Key) - + updated[k] = false // default + v.Changed = false // reset to the default // send obj1 := reflect.Indirect(reflect.ValueOf(v.Res)) type1 := obj1.Type() @@ -177,10 +179,12 @@ func (obj *BaseRes) SendRecv(res Res) (bool, error) { if !reflect.DeepEqual(value1.Interface(), value2.Interface()) { // TODO: can we catch the panics here in case they happen? value2.Set(value1) // do it for all types that match - changed = true + updated[k] = true // we updated this key! + v.Changed = true // tag this key as updated! + log.Printf("SendRecv: %s[%s].%s <- %s[%s].%s", obj.Kind(), obj.GetName(), k, v.Res.Kind(), v.Res.GetName(), v.Key) } } - return changed, err + return updated, err } // TypeCmp compares two reflect values to see if they are the same Kind. It can