golint: Fix some golint issues

Let tip builds pass in travis too!

.ackrc

@@ -1,2 +1,3 @@
 --ignore-dir=old/
 --ignore-dir=tmp/
+--ignore-dir=vendor/

.gitignore

@@ -1,10 +1,11 @@
+.idea/
 .omv/
 .ssh/
 .vagrant/
-mgmt-documentation.pdf
 old/
 tmp/
 *_stringer.go
 mgmt
 mgmt.static
+mgmt.iml
 rpmbuild/

.gitmodules

@@ -10,3 +10,6 @@
 [submodule "vendor/gopkg.in/fsnotify.v1"]
 	path = vendor/gopkg.in/fsnotify.v1
 	url = https://gopkg.in/fsnotify.v1
+[submodule "vendor/github.com/purpleidea/go-systemd"]
+	path = vendor/github.com/purpleidea/go-systemd
+	url = https://github.com/purpleidea/go-systemd

Makefile

@@ -138,8 +138,8 @@ format: gofmt yamlfmt
 
 docs: $(PROGRAM)-documentation.pdf
 
-$(PROGRAM)-documentation.pdf: DOCUMENTATION.md
+$(PROGRAM)-documentation.pdf: docs/documentation.md
-	pandoc DOCUMENTATION.md -o '$(PROGRAM)-documentation.pdf'
+	pandoc docs/documentation.md -o docs/'$(PROGRAM)-documentation.pdf'
 
 #
 #	build aliases

README.md

@@ -4,7 +4,7 @@
 
 [![Go Report Card](https://goreportcard.com/badge/github.com/purpleidea/mgmt)](https://goreportcard.com/report/github.com/purpleidea/mgmt)
 [![Build Status](https://secure.travis-ci.org/purpleidea/mgmt.png?branch=master)](http://travis-ci.org/purpleidea/mgmt)
-[![Documentation](https://img.shields.io/docs/markdown.png)](DOCUMENTATION.md)
+[![Documentation](https://img.shields.io/docs/markdown.png)](docs/documentation.md)
 [![GoDoc](https://godoc.org/github.com/purpleidea/mgmt?status.svg)](https://godoc.org/github.com/purpleidea/mgmt)
 [![IRC](https://img.shields.io/irc/%23mgmtconfig.png)](https://webchat.freenode.net/?channels=#mgmtconfig)
 [![Jenkins](https://img.shields.io/jenkins/status.png)](https://ci.centos.org/job/purpleidea-mgmt/)
@@ -23,7 +23,7 @@ With your help you'll be able to influence our design and get us there sooner!
 
 ## Questions:
 Please join the [#mgmtconfig](https://webchat.freenode.net/?channels=#mgmtconfig) IRC community!
-If you have a well phrased question that might benefit others, consider asking it by sending a patch to the documentation [FAQ](https://github.com/purpleidea/mgmt/blob/master/DOCUMENTATION.md#usage-and-frequently-asked-questions) section. I'll merge your question, and a patch with the answer!
+If you have a well phrased question that might benefit others, consider asking it by sending a patch to the documentation [FAQ](https://github.com/purpleidea/mgmt/blob/master/docs/documentation.md#usage-and-frequently-asked-questions) section. I'll merge your question, and a patch with the answer!
 
 ## Quick start:
 * Make sure you have golang version 1.6 or greater installed.
@@ -49,7 +49,7 @@ cd $GOPATH/src/github.com/purpleidea/mgmt
 Please look in the [examples/](examples/) folder for more examples!
 
 ## Documentation:
-Please see: the manually created [DOCUMENTATION.md](DOCUMENTATION.md) (also available as [PDF](https://pdfdoc-purpleidea.rhcloud.com/pdf/https://github.com/purpleidea/mgmt/blob/master/DOCUMENTATION.md)) and the automatically generated [GoDoc documentation](https://godoc.org/github.com/purpleidea/mgmt).
+Please see: the manually created [documentation.md](docs/documentation.md) (also available as [PDF](https://pdfdoc-purpleidea.rhcloud.com/pdf/https://github.com/purpleidea/mgmt/blob/master/docs/documentation.md)) and the automatically generated [GoDoc documentation](https://godoc.org/github.com/purpleidea/mgmt).
 
 ## Roadmap:
 Please see: [TODO.md](TODO.md) for a list of upcoming work and TODO items.
@@ -104,6 +104,9 @@ We'd love to have your patches! Please send them by email, or as a pull request.
 * Felix Frank; blog: [Translating All The Things (puppet to mgmt translation warnings)](https://ffrank.github.io/features/2016/08/19/translating-all-the-things/)
 * James Shubin; video: [Recording from systemd.conf 2016](https://www.youtube.com/watch?v=jB992Zb3nH0&html5=1)
 * James Shubin; blog: [Remote execution in mgmt](https://ttboj.wordpress.com/2016/10/07/remote-execution-in-mgmt/)
+* James Shubin; video: [Recording from High Load Strategy 2016](https://vimeo.com/191493409)
+* James Shubin; video: [Recording from NLUUG 2016](https://www.youtube.com/watch?v=MmpwOQAb_SE&html5=1)
+* James Shubin; blog: [Send/Recv in mgmt](https://ttboj.wordpress.com/2016/12/07/sendrecv-in-mgmt/)
 
 ##
 

docs/.gitignore

@@ -0,0 +1 @@
+mgmt-documentation.pdf

docs/documentation.md

@@ -23,7 +23,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 ####Available from:
 ####[https://github.com/purpleidea/mgmt/](https://github.com/purpleidea/mgmt/)
 
-####This documentation is available in: [Markdown](https://github.com/purpleidea/mgmt/blob/master/DOCUMENTATION.md) or [PDF](https://pdfdoc-purpleidea.rhcloud.com/pdf/https://github.com/purpleidea/mgmt/blob/master/DOCUMENTATION.md) format.
+####This documentation is available in: [Markdown](https://github.com/purpleidea/mgmt/blob/master/docs/documentation.md) or [PDF](https://pdfdoc-purpleidea.rhcloud.com/pdf/https://github.com/purpleidea/mgmt/blob/master/docs/documentation.md) format.
 
 ####Table of Contents
 
@@ -218,8 +218,11 @@ parameter with the [Noop](#Noop) resource.
 
 * [Exec](#Exec): Execute shell commands on the system.
 * [File](#File): Manage files and directories.
+* [Hostname](#Hostname): Manages the hostname on the system.
 * [Msg](#Msg): Send log messages.
 * [Noop](#Noop): A simple resource that does nothing.
+* [Nspawn](#Nspawn): Manage systemd-machined nspawn containers.
+* [Password](#Password): Create random password strings.
 * [Pkg](#Pkg):  Manage system packages with PackageKit.
 * [Svc](#Svc): Manage system systemd services.
 * [Timer](#Timer): Manage system systemd services.
@@ -263,6 +266,30 @@ The force property is required if we want the file resource to be able to change
 a file into a directory or vice-versa. If such a change is needed, but the force
 property is not set to `true`, then this file resource will error.
 
+###Hostname
+
+The hostname resource manages static, transient/dynamic and pretty hostnames
+on the system and watches them for changes.
+
+#### static_hostname
+The static hostname is the one configured in /etc/hostname or a similar
+file.
+It is chosen by the local user. It is not always in sync with the current
+host name as returned by the gethostname() system call.
+
+#### transient_hostname
+The transient / dynamic hostname is the one configured via the kernel's
+sethostbyname().
+It can be different from the static hostname in case DHCP or mDNS have been
+configured to change the name based on network information.
+
+#### pretty_hostname
+The pretty hostname is a free-form UTF8 host name for presentation to the user.
+
+#### hostname
+Hostname is the fallback value for all 3 fields above, if only `hostname` is
+specified, it will set all 3 fields to this value.
+
 ###Msg
 
 The msg resource sends messages to the main log, or an external service such
@@ -273,6 +300,15 @@ as systemd's journal.
 The noop resource does absolutely nothing. It does have some utility in testing
 `mgmt` and also as a placeholder in the resource graph.
 
+###Nspawn
+
+The nspawn resource is used to manage systemd-machined style containers.
+
+###Password
+
+The password resource can generate a random string to be used as a password. It
+will re-generate the password if it receives a refresh notification.
+
 ###Pkg
 
 The pkg resource is used to manage system packages. This resource works on many

docs/puppet.md

@@ -14,7 +14,7 @@ This document goes into detail on how this works, and lists
 some pitfalls and limitations.
 
 For basic instructions on how to use the Puppet support, see
-the [main documentation](DOCUMENTATION.md#puppet-support).
+the [main documentation](documentation.md#puppet-support).
 
 ##Prerequisites
 

docs/resource-guide.md

@@ -0,0 +1,534 @@
+#mgmt
+
+<!--
+Mgmt
+Copyright (C) 2013-2016+ James Shubin and the project contributors
+Written by James Shubin <james@shubin.ca> and the project contributors
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+-->
+
+##mgmt resource guide by [James](https://ttboj.wordpress.com/)
+####Available from:
+####[https://github.com/purpleidea/mgmt/](https://github.com/purpleidea/mgmt/)
+
+####This documentation is available in: [Markdown](https://github.com/purpleidea/mgmt/blob/master/docs/resource-guide.md) or [PDF](https://pdfdoc-purpleidea.rhcloud.com/pdf/https://github.com/purpleidea/mgmt/blob/master/docs/resource-guide.md) format.
+
+####Table of Contents
+
+1. [Overview](#overview)
+2. [Theory - Resource theory in mgmt](#theory)
+3. [Resource API - Getting started with mgmt](#resource-api)
+	* [Init - Initialize the resource](#init)
+	* [CheckApply - Check and apply resource state](#checkapply)
+	* [Watch - Detect resource changes](#watch)
+	* [Compare - Compare resource with another](#compare)
+4. [Further considerations - More information about resource writing](#further-considerations)
+5. [Automatic edges - Adding automatic resources dependencies](#automatic-edges)
+6. [Automatic grouping - Grouping multiple resources into one](#automatic-grouping)
+7. [Send/Recv - Communication between resources](#send-recv)
+8. [Composite resources - Importing code from one resource into another](#composite-resources)
+9. [FAQ - Frequently asked questions](#frequently-asked-questions)
+10. [Suggestions - API change suggestions](#suggestions)
+11. [Authors - Authors and contact information](#authors)
+
+##Overview
+
+The `mgmt` tool has built-in resource primitives which make up the building
+blocks of any configuration. Each instance of a resource is mapped to a single
+vertex in the resource [graph](https://en.wikipedia.org/wiki/Directed_acyclic_graph).
+This guide is meant to instruct developers on how to write a brand new resource.
+Since `mgmt` and the core resources are written in golang, some prior golang
+knowledge is assumed.
+
+##Theory
+
+Resources in `mgmt` are similar to resources in other systems in that they are
+[idempotent](https://en.wikipedia.org/wiki/Idempotence). Our resources are
+uniquely different in that they can detect when their state has changed, and as
+a result can run to revert or repair this change instantly. For some background
+on this design, please read the
+[original article](https://ttboj.wordpress.com/2016/01/18/next-generation-configuration-mgmt/)
+on the subject.
+
+##Resource API
+
+To implement a resource in `mgmt` it must satisfy the
+[`Res`](https://github.com/purpleidea/mgmt/blob/master/resources/resources.go)
+interface. What follows are each of the method signatures and a description of
+each.
+
+###Init
+```golang
+Init() error
+```
+
+This is called to initialize the resource. If something goes wrong, it should
+return an error. It should set the resource `kind`, do any resource specific
+work, and finish by calling the `Init` method of the base resource.
+
+####Example
+```golang
+// Init initializes the Foo resource.
+func (obj *FooRes) Init() error {
+	obj.BaseRes.kind = "Foo" // must set capitalized resource kind
+	// run the resource specific initialization, and error if anything fails
+	if some_error {
+		return err // something went wrong!
+	}
+	return obj.BaseRes.Init() // call the base resource init
+}
+```
+
+###CheckApply
+```golang
+CheckApply(apply bool) (checkOK bool, err error)
+```
+
+`CheckApply` is where the real _work_ is done. Under normal circumstances, this
+function should check if the state of this resource is correct, and if so, it
+should return: `(true, nil)`. If the `apply` variable is set to `true`, then
+this means that we should then proceed to run the changes required to bring the
+resource into the correct state. If the `apply` variable is set to `false`, then
+the resource is operating in _noop_ mode and _no operations_ should be executed!
+
+After having executed the necessary operations to bring the resource back into
+the desired state, or after having detected that the state was incorrect, but
+that changes can't be made because `apply` is `false`, you should then return
+`(false, nil)`.
+
+You must cause the resource to converge during a single execution of this
+function. If you cannot, then you must return an error! The exception to this
+rule is that if an external force changes the state of the resource while it is
+being remedied, it is possible to return from this function even though the
+resource isn't now converged. This is not a bug, as the resources `Watch`
+facility will detect the change, ultimately resulting in a subsequent call to
+`CheckApply`.
+
+####Example
+```golang
+// CheckApply does the idempotent work of checking and applying resource state.
+func (obj *FooRes) CheckApply(apply bool) (bool, error) {
+	// check the state
+	if state_is_okay { return true, nil } // done early! :)
+	// state was bad
+	if !apply { return false, nil } // don't apply; !stateok, nil
+	// do the apply!
+	return false, nil // after success applying
+	if any_error { return false, err } // anytime there's an err!
+}
+```
+
+The `CheckApply` function is called by the `mgmt` engine when it believes a call
+is necessary. Under certain conditions when a `Watch` call does not invalidate
+the state of the resource, and no refresh call was sent, its execution might be
+skipped. This is an engine optimization, and not a bug. It is mentioned here in
+the documentation in case you are confused as to why a debug message you've
+added to the code isn't always printed.
+
+####Refresh notifications
+Some resources may choose to support receiving refresh notifications. In general
+these should be avoided if possible, but nevertheless, they do make sense in
+certain situations. Resources that support these need to verify if one was sent
+during the CheckApply phase of execution. This is accomplished by calling the
+`Refresh() bool` method of the resource, and inspecting the return value. This
+is only necessary if you plan to perform a refresh action. Refresh actions
+should still respect the `apply` variable, and no system changes should be made
+if it is `false`. Refresh notifications are generated by any resource when an
+action is applied by that resource and are transmitted through graph edges which
+have enabled their propagation. Resources that currently perform some refresh
+action include `svc`, `timer`, and `password`.
+
+####Paired execution
+For many resources it is not uncommon to see `CheckApply` run twice in rapid
+succession. This is usually not a pathological occurrence, but rather a healthy
+pattern which is a consequence of the event system. When the state of the
+resource is incorrect, `CheckApply` will run to remedy the state. In response to
+having just changed the state, it is usually the case that this repair will
+trigger the `Watch` code! In response, a second `CheckApply` is triggered, which
+will likely find the state to now be correct.
+
+####Summary
+* Anytime an error occurs during `CheckApply`, you should return `(false, err)`.
+* If the state is correct and no changes are needed, return `(true, nil)`.
+* You should only make changes to the system if `apply` is set to `true`.
+* After checking the state and possibly applying the fix, return `(false, nil)`.
+* Returning `(true, err)` is a programming error and will cause a `Fatal`.
+
+###Watch
+```golang
+Watch(chan Event) error
+```
+
+`Watch` is a main loop that runs and sends messages when it detects that the
+state of the resource might have changed. To send a message you should write to
+the input `Event` channel using the `DoSend` helper method. The Watch function
+should run continuously until a shutdown message is received. If at any time
+something goes wrong, you should return an error, and the `mgmt` engine will
+handle possibly restarting the main loop based on the `retry` meta parameters.
+
+It is better to send an event notification which turns out to be spurious, than
+to miss a possible event. Resources which can miss events are incorrect and need
+to be re-engineered so that this isn't the case. If you have an idea for a
+resource which would fit this criteria, but you can't find a solution, please
+contact the `mgmt` maintainers so that this problem can be investigated and a
+possible system level engineering fix can be found.
+
+You may have trouble deciding how much resource state checking should happen in
+the `Watch` loop versus deferring it all to the `CheckApply` method. You may
+want to put some simple fast path checking in `Watch` to avoid generating
+obviously spurious events, but in general it's best to keep the `Watch` method
+as simple as possible. Contact the `mgmt` maintainers if you're not sure.
+
+If the resource is activated in `polling` mode, the `Watch` method will not get
+executed. As a result, the resource must still work even if the main loop is not
+running.
+
+####Select
+The lifetime of most resources `Watch` method should be spent in an infinite
+loop that is bounded by a `select` call. The `select` call is the point where
+our method hands back control to the engine (and the kernel) so that we can
+sleep until something of interest wakes us up. In this loop we must process
+events from the engine via the `<-obj.Events()` call, wait for the converged
+timeout with `<-cuid.ConvergedTimer()`, and receive events for our resource
+itself!
+
+####Events
+If we receive an internal event from the `<-obj.Events()` method, we can read it
+with the ReadEvent helper function. This function tells us if we should shutdown
+our resource, and if we should generate an event. When we want to send an event,
+we use the `DoSend` helper function. It is also important to mark the resource
+state as `dirty` if we believe it might have changed. We do this with the
+`StateOK(false)` function.
+
+####Startup
+Once the `Watch` function has finished starting up successfully, it is important
+to generate one event to notify the `mgmt` engine that we're now listening
+successfully, so that it can run an initial `CheckApply` to ensure we're safely
+tracking a healthy state and that we didn't miss anything when `Watch` was down
+or from before `mgmt` was running. It does this by calling the `Running` method.
+
+####Converged
+The engine might be asked to shutdown when the entire state of the system has
+not seen any changes for some duration of time. In order for the engine to be
+able to make this determination, each resource must report its converged state.
+To do this, the `Watch` method should get the `ConvergedUID` handle that has
+been prepared for it by the engine. This is done by calling the `Converger`
+method on the resource object. The result can be used to set the converged
+status with `SetConverged`, and to notify when the particular timeout has been
+reached by waiting on `ConvergedTimer`.
+
+Instead of interacting with the `ConvergedUID` with these two methods, we can
+instead use the `StartTimer` and `ResetTimer` methods which accomplish the same
+thing, but provide a `select`-free interface for different coding situations.
+
+####Example
+```golang
+// Watch is the listener and main loop for this resource.
+func (obj *FooRes) Watch(processChan chan event.Event) error {
+	cuid := obj.Converger() // get the converger uid used to report status
+
+	// setup the Foo resource
+	var err error
+	if err, obj.foo = OpenFoo(); err != nil {
+		return err // we couldn't startup
+	}
+	defer obj.whatever.CloseFoo() // shutdown our
+
+	// notify engine that we're running
+	if err := obj.Running(processChan); err != nil {
+		return err // bubble up a NACK...
+	}
+
+	var send = false // send event?
+	var exit = false
+	for {
+		obj.SetState(ResStateWatching) // reset
+		select {
+		case event := <-obj.Events():
+			cuid.SetConverged(false)
+			// we avoid sending events on unpause
+			if exit, send = obj.ReadEvent(&event); exit {
+				return nil // exit
+			}
+
+		// the actual events!
+		case event := <-obj.foo.Events:
+			if is_an_event {
+				send = true // used below
+				cuid.SetConverged(false)
+				obj.StateOK(false) // dirty
+			}
+
+		// event errors
+		case err := <-obj.foo.Errors:
+			cuuid.SetConverged(false)
+			return err // will cause a retry or permanent failure
+
+		case <-cuid.ConvergedTimer():
+			cuid.SetConverged(true) // converged!
+			continue
+		}
+
+		// do all our event sending all together to avoid duplicate msgs
+		if send {
+			send = false
+			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
+				return err // we exit or bubble up a NACK...
+			}
+		}
+	}
+}
+```
+
+####Summary
+* Remember to call the appropriate `converger` methods throughout the resource.
+* Remember to call `Startup` when the `Watch` is running successfully.
+* Remember to process internal events and shutdown promptly if asked to.
+* Ensure the design of your resource is well thought out.
+* Have a look at the existing resources for a rough idea of how this all works.
+
+###Compare
+```golang
+Compare(Res) bool
+```
+
+Each resource must have a `Compare` method. This takes as input another resource
+and must return whether they are identical or not. This is used for identifying
+if an existing resource can be used in place of a new one with a similar set of
+parameters. In particular, when switching from one graph to a new (possibly
+identical) graph, this avoids recomputing the state for resources which don't
+change or that are sufficiently similar that they don't need to be swapped out.
+
+In general if all the resource properties are identical, then they usually don't
+need to be changed. On occasion, not all of them need to be compared, in
+particular if they store some generated state, or if they aren't significant in
+some way.
+
+####Example
+```golang
+// Compare two resources and return if they are equivalent.
+func (obj *FooRes) Compare(res Res) bool {
+	switch res.(type) {
+	case *FooRes: // only compare to other resources of the Foo kind!
+		res := res.(*FileRes)
+		if !obj.BaseRes.Compare(res) { // call base Compare
+			return false
+		}
+		if obj.Name != res.Name {
+			return false
+		}
+		if obj.whatever != res.whatever {
+			return false
+		}
+		if obj.Flag != res.Flag {
+			return false
+		}
+	default:
+		return false // different kind of resource
+	}
+	return true // they must match!
+}
+```
+
+###Validate
+```golang
+Validate() error
+```
+
+This method is used to validate if the populated resource struct is a valid
+representation of the resource kind. If it does not conform to the resource
+specifications, it should generate an error. If you notice that this method is
+quite large, it might be an indication that you might want to reconsider the
+parameter list and interface to this resource.
+
+###GetUIDs
+```golang
+GetUIDs() []ResUID
+```
+
+The `GetUIDs` method returns a list of `ResUID` interfaces that represent the
+particular resource uniquely. This is used with the AutoEdges API to determine
+if another resource can match a dependency to this one.
+
+###AutoEdges
+```golang
+AutoEdges() AutoEdge
+```
+
+This returns a struct that implements the `AutoEdge` interface. This struct
+is used to match other resources that might be relevant dependencies for this
+resource.
+
+###CollectPattern
+```golang
+CollectPattern() string
+```
+
+This is currently a stub and will be updated once the DSL is further along.
+
+##Further considerations
+There is some additional information that any resource writer will need to know.
+Each issue is listed separately below!
+
+###Resource struct
+Each resource will implement methods as pointer receivers on a resource struct.
+The resource struct must include an anonymous reference to the `BaseRes` struct.
+The naming convention for resources is that they end with a `Res` suffix. If
+you'd like your resource to be accessible by the `YAML` graph API (GAPI), then
+you'll need to include the appropriate YAML fields as shown below.
+
+####Example
+```golang
+type FooRes struct {
+	BaseRes `yaml:",inline"` // base properties
+
+	Whatever string `yaml:"whatever"` // you pick!
+	Bar int // no yaml, used as public output value for send/recv
+	Baz bool `yaml:"baz"` // something else
+
+	something string // some private field
+}
+```
+
+###YAML
+In addition to labelling your resource struct with YAML fields, you must also
+add an entry to the internal `GraphConfig` struct. It is a fairly straight
+forward one line patch.
+
+```golang
+type GraphConfig struct {
+// [snip...]
+	Resources struct {
+		Noop []*resources.NoopRes `yaml:"noop"`
+		File []*resources.FileRes `yaml:"file"`
+		// [snip...]
+		Foo []*resources.FooRes `yaml:"foo"` // tada :)
+	}
+}
+```
+
+###Gob registration
+All resources must be registered with the `golang` _gob_ module so that they can
+be encoded and decoded. Make sure to include the following code snippet for this
+to work.
+
+```golang
+import "encoding/gob"
+func init() { // special golang method that runs once
+	gob.Register(&FooRes{}) // substitude your resource here
+}
+```
+
+##Automatic edges
+Automatic edges in `mgmt` are well described in [this article](https://ttboj.wordpress.com/2016/03/14/automatic-edges-in-mgmt/).
+The best example of this technique can be seen in the `svc` resource.
+Unfortunately no further documentation about this subject has been written. To
+expand this section, please send a patch! Please contact us if you'd like to
+work on a resource that uses this feature, or to add it to an existing one!
+
+##Automatic grouping
+Automatic grouping in `mgmt` is well described in [this article](https://ttboj.wordpress.com/2016/03/30/automatic-grouping-in-mgmt/).
+The best example of this technique can be seen in the `pkg` resource.
+Unfortunately no further documentation about this subject has been written. To
+expand this section, please send a patch! Please contact us if you'd like to
+work on a resource that uses this feature, or to add it to an existing one!
+
+
+##Send/Recv
+In `mgmt` there is a novel concept called _Send/Recv_. For some background,
+please [read the introductory article](https://ttboj.wordpress.com/2016/12/07/sendrecv-in-mgmt/).
+When using this feature, the engine will automatically send the user specified
+value to the intended destination without requiring any resource specific code.
+Any time that one of the destination values is changed, the engine automatically
+marks the resource state as `dirty`. To detect if a particular value was
+received, and if it changed (during this invocation of CheckApply) from the
+previous value, you can query the Recv parameter. It will contain a `map` of all
+the keys which can be received on, and the value has a `Changed` property which
+will indicate whether the value was updated on this particular `CheckApply`
+invocation. The type of the sending key must match that of the receiving one.
+This can _only_ be done inside of the `CheckApply` function!
+
+```golang
+// inside CheckApply, probably near the top
+if val, exists := obj.Recv["SomeKey"]; exists {
+	log.Printf("SomeKey was sent to us from: %s[%s].%s", val.Res.Kind(), val.Res.GetName(), val.Key)
+	if val.Changed {
+		log.Printf("SomeKey was just updated!")
+		// you may want to invalidate some local cache
+	}
+}
+```
+
+Astute readers will note that there isn't anything that prevents a user from
+sending an identically typed value to some arbitrary (public) key that the
+resource author hadn't considered! While this is true, resources should probably
+work within this problem space anyways. The rule of thumb is that any public
+parameter which is normally used in a resource can be used safely.
+
+One subtle scenario is that if a resource creates a local cache or stores a
+computation that depends on the value of a public parameter and will require
+invalidation should that public parameter change, then you must detect that
+scenario and invalidate the cache when it occurs. This *must* be processed
+before there is a possibility of failure in CheckApply, because if we fail (and
+possibly run again) the subsequent send->recv transfer might not have a new
+value to copy, and therefore we won't see this notification of change.
+Therefore, it is important to process these promptly, if they must not be lost,
+such as for cache invalidation.
+
+Remember, `Send/Recv` only changes your resource code if you cache state.
+
+##Composite resources
+Composite resources are resources which embed one or more existing resources.
+This is useful to prevent code duplication in higher level resource scenarios.
+The best example of this technique can be seen in the `nspawn` resource which
+can be seen to partially embed a `svc` resource, but without its `Watch`.
+Unfortunately no further documentation about this subject has been written. To
+expand this section, please send a patch! Please contact us if you'd like to
+work on a resource that uses this feature, or to add it to an existing one!
+
+##Frequently asked questions
+(Send your questions as a patch to this FAQ! I'll review it, merge it, and
+respond by commit with the answer.)
+
+###Can I write resources in a different language?
+Currently `golang` is the only supported language for built-in resources. We
+might consider allowing external resources to be imported in the future. This
+will likely require a language that can expose a C-like API, such as `python` or
+`ruby`. Custom `golang` resources are already possible when using mgmt as a lib.
+Higher level resource collections will be possible once the `mgmt` DSL is ready.
+
+###What new resource primitives need writing?
+There are still many ideas for new resources that haven't been written yet. If
+you'd like to contribute one, please contact us and tell us about your idea!
+
+###Where can I find more information about mgmt?
+Additional blog posts, videos and other material [is available!](https://github.com/purpleidea/mgmt/#on-the-web).
+
+##Suggestions
+If you have any ideas for API changes or other improvements to resource writing,
+please let us know! We're still pre 1.0 and pre 0.1 and happy to break API in
+order to get it right!
+
+##Authors
+
+Copyright (C) 2013-2016+ James Shubin and the project contributors
+
+Please see the
+[AUTHORS](https://github.com/purpleidea/mgmt/tree/master/AUTHORS) file
+for more information.
+
+* [github](https://github.com/purpleidea/)
+* [&#64;purpleidea](https://twitter.com/#!/purpleidea)
+* [https://ttboj.wordpress.com/](https://ttboj.wordpress.com/)

etcd/etcd.go

@@ -64,7 +64,6 @@ import (
 
 	"github.com/purpleidea/mgmt/converger"
 	"github.com/purpleidea/mgmt/event"
-	"github.com/purpleidea/mgmt/global"
 	"github.com/purpleidea/mgmt/resources"
 	"github.com/purpleidea/mgmt/util"
 
@@ -154,6 +153,13 @@ type TN struct {
 	data    *etcd.TxnResponse
 }
 
+// Flags are some constant flags which are used throughout the program.
+type Flags struct {
+	Debug   bool // add additional log messages
+	Trace   bool // add execution flow log messages
+	Verbose bool // add extra log message output
+}
+
 // EmbdEtcd provides the embedded server and client etcd functionality
 type EmbdEtcd struct { // EMBeddeD etcd
 	// etcd client connection related
@@ -190,6 +196,7 @@ type EmbdEtcd struct { // EMBeddeD etcd
 	delq    chan *DL // delete queue
 	txnq    chan *TN // txn queue
 
+	flags     Flags
 	prefix    string              // folder prefix to use for misc storage
 	converger converger.Converger // converged tracking
 
@@ -200,7 +207,7 @@ type EmbdEtcd struct { // EMBeddeD etcd
 }
 
 // NewEmbdEtcd creates the top level embedded etcd struct client and server obj
-func NewEmbdEtcd(hostname string, seeds, clientURLs, serverURLs etcdtypes.URLs, noServer bool, idealClusterSize uint16, prefix string, converger converger.Converger) *EmbdEtcd {
+func NewEmbdEtcd(hostname string, seeds, clientURLs, serverURLs etcdtypes.URLs, noServer bool, idealClusterSize uint16, flags Flags, prefix string, converger converger.Converger) *EmbdEtcd {
 	endpoints := make(etcdtypes.URLsMap)
 	if hostname == seedSentinel { // safety
 		return nil
@@ -229,6 +236,7 @@ func NewEmbdEtcd(hostname string, seeds, clientURLs, serverURLs etcdtypes.URLs,
 
 		idealClusterSize: idealClusterSize,
 		converger:        converger,
+		flags:            flags,
 		prefix:           prefix,
 		dataDir:          path.Join(prefix, "etcd"),
 	}
@@ -273,7 +281,7 @@ func (obj *EmbdEtcd) GetConfig() etcd.Config {
 // Connect connects the client to a server, and then builds the *API structs.
 // If reconnect is true, it will force a reconnect with new config endpoints.
 func (obj *EmbdEtcd) Connect(reconnect bool) error {
-	if global.DEBUG {
+	if obj.flags.Debug {
 		log.Println("Etcd: Connect...")
 	}
 	obj.cLock.Lock()
@@ -308,7 +316,7 @@ func (obj *EmbdEtcd) Connect(reconnect bool) error {
 			if emax > maxClientConnectRetries {
 				log.Printf("Etcd: The dataDir (%s) might be inconsistent or corrupt.", obj.dataDir)
 				log.Printf("Etcd: Please see: %s", "https://github.com/purpleidea/mgmt/blob/master/DOCUMENTATION.md#what-does-the-error-message-about-an-inconsistent-datadir-mean")
-				obj.cError = fmt.Errorf("Can't find an available endpoint.")
+				obj.cError = fmt.Errorf("can't find an available endpoint")
 				return obj.cError
 			}
 			err = &CtxDelayErr{time.Duration(emax) * time.Second, "No endpoints available yet!"} // retry with backoff...
@@ -529,29 +537,29 @@ func (obj *EmbdEtcd) CtxError(ctx context.Context, err error) (context.Context,
 	var isTimeout = false
 	var iter int // = 0
 	if ctxerr, ok := ctx.Value(ctxErr).(error); ok {
-		if global.DEBUG {
+		if obj.flags.Debug {
 			log.Printf("Etcd: CtxError: err(%v), ctxerr(%v)", err, ctxerr)
 		}
 		if i, ok := ctx.Value(ctxIter).(int); ok {
 			iter = i + 1 // load and increment
-			if global.DEBUG {
+			if obj.flags.Debug {
 				log.Printf("Etcd: CtxError: Iter: %v", iter)
 			}
 		}
 		isTimeout = err == context.DeadlineExceeded
-		if global.DEBUG {
+		if obj.flags.Debug {
 			log.Printf("Etcd: CtxError: isTimeout: %v", isTimeout)
 		}
 		if !isTimeout {
 			iter = 0 // reset timer
 		}
 		err = ctxerr // restore error
-	} else if global.DEBUG {
+	} else if obj.flags.Debug {
 		log.Printf("Etcd: CtxError: No value found")
 	}
 	ctxHelper := func(tmin, texp, tmax int) context.Context {
 		t := expBackoff(tmin, texp, iter, tmax)
-		if global.DEBUG {
+		if obj.flags.Debug {
 			log.Printf("Etcd: CtxError: Timeout: %v", t)
 		}
 
@@ -638,13 +646,13 @@ func (obj *EmbdEtcd) CtxError(ctx context.Context, err error) (context.Context,
 		fallthrough
 	case isGrpc(grpc.ErrClientConnClosing):
 
-		if global.DEBUG {
+		if obj.flags.Debug {
 			log.Printf("Etcd: CtxError: Error(%T): %+v", err, err)
 			log.Printf("Etcd: Endpoints are: %v", obj.client.Endpoints())
 			log.Printf("Etcd: Client endpoints are: %v", obj.endpoints)
 		}
 
-		if global.DEBUG {
+		if obj.flags.Debug {
 			log.Printf("Etcd: CtxError: Locking...")
 		}
 		obj.rLock.Lock()
@@ -665,7 +673,7 @@ func (obj *EmbdEtcd) CtxError(ctx context.Context, err error) (context.Context,
 			obj.ctxErr = fmt.Errorf("Etcd: Permanent connect error: %v", err)
 			return ctx, obj.ctxErr
 		}
-		if global.DEBUG {
+		if obj.flags.Debug {
 			log.Printf("Etcd: CtxError: Unlocking...")
 		}
 		obj.rLock.Unlock()
@@ -709,7 +717,7 @@ func (obj *EmbdEtcd) CbLoop() {
 			if !re.skipConv { // if we want to count it...
 				cuid.ResetTimer() // activity!
 			}
-			if global.TRACE {
+			if obj.flags.Trace {
 				log.Printf("Trace: Etcd: CbLoop: Event: StartLoop")
 			}
 			for {
@@ -717,11 +725,11 @@ func (obj *EmbdEtcd) CbLoop() {
 					//re.resp.NACK() // nope!
 					break
 				}
-				if global.TRACE {
+				if obj.flags.Trace {
 					log.Printf("Trace: Etcd: CbLoop: rawCallback()")
 				}
 				err := rawCallback(ctx, re)
-				if global.TRACE {
+				if obj.flags.Trace {
 					log.Printf("Trace: Etcd: CbLoop: rawCallback(): %v", err)
 				}
 				if err == nil {
@@ -733,7 +741,7 @@ func (obj *EmbdEtcd) CbLoop() {
 					break // TODO: it's bad, break or return?
 				}
 			}
-			if global.TRACE {
+			if obj.flags.Trace {
 				log.Printf("Trace: Etcd: CbLoop: Event: FinishLoop")
 			}
 
@@ -761,11 +769,11 @@ func (obj *EmbdEtcd) Loop() {
 		select {
 		case aw := <-obj.awq:
 			cuid.ResetTimer() // activity!
-			if global.TRACE {
+			if obj.flags.Trace {
 				log.Printf("Trace: Etcd: Loop: PriorityAW: StartLoop")
 			}
 			obj.loopProcessAW(ctx, aw)
-			if global.TRACE {
+			if obj.flags.Trace {
 				log.Printf("Trace: Etcd: Loop: PriorityAW: FinishLoop")
 			}
 			continue // loop to drain the priority channel first!
@@ -777,18 +785,18 @@ func (obj *EmbdEtcd) Loop() {
 		// add watcher
 		case aw := <-obj.awq:
 			cuid.ResetTimer() // activity!
-			if global.TRACE {
+			if obj.flags.Trace {
 				log.Printf("Trace: Etcd: Loop: AW: StartLoop")
 			}
 			obj.loopProcessAW(ctx, aw)
-			if global.TRACE {
+			if obj.flags.Trace {
 				log.Printf("Trace: Etcd: Loop: AW: FinishLoop")
 			}
 
 		// set kv pair
 		case kv := <-obj.setq:
 			cuid.ResetTimer() // activity!
-			if global.TRACE {
+			if obj.flags.Trace {
 				log.Printf("Trace: Etcd: Loop: Set: StartLoop")
 			}
 			for {
@@ -805,7 +813,7 @@ func (obj *EmbdEtcd) Loop() {
 					break // TODO: it's bad, break or return?
 				}
 			}
-			if global.TRACE {
+			if obj.flags.Trace {
 				log.Printf("Trace: Etcd: Loop: Set: FinishLoop")
 			}
 
@@ -814,7 +822,7 @@ func (obj *EmbdEtcd) Loop() {
 			if !gq.skipConv {
 				cuid.ResetTimer() // activity!
 			}
-			if global.TRACE {
+			if obj.flags.Trace {
 				log.Printf("Trace: Etcd: Loop: Get: StartLoop")
 			}
 			for {
@@ -832,14 +840,14 @@ func (obj *EmbdEtcd) Loop() {
 					break // TODO: it's bad, break or return?
 				}
 			}
-			if global.TRACE {
+			if obj.flags.Trace {
 				log.Printf("Trace: Etcd: Loop: Get: FinishLoop")
 			}
 
 		// delete value
 		case dl := <-obj.delq:
 			cuid.ResetTimer() // activity!
-			if global.TRACE {
+			if obj.flags.Trace {
 				log.Printf("Trace: Etcd: Loop: Delete: StartLoop")
 			}
 			for {
@@ -857,14 +865,14 @@ func (obj *EmbdEtcd) Loop() {
 					break // TODO: it's bad, break or return?
 				}
 			}
-			if global.TRACE {
+			if obj.flags.Trace {
 				log.Printf("Trace: Etcd: Loop: Delete: FinishLoop")
 			}
 
 		// run txn
 		case tn := <-obj.txnq:
 			cuid.ResetTimer() // activity!
-			if global.TRACE {
+			if obj.flags.Trace {
 				log.Printf("Trace: Etcd: Loop: Txn: StartLoop")
 			}
 			for {
@@ -882,7 +890,7 @@ func (obj *EmbdEtcd) Loop() {
 					break // TODO: it's bad, break or return?
 				}
 			}
-			if global.TRACE {
+			if obj.flags.Trace {
 				log.Printf("Trace: Etcd: Loop: Txn: FinishLoop")
 			}
 
@@ -936,7 +944,7 @@ func (obj *EmbdEtcd) Set(key, value string, opts ...etcd.OpOption) error {
 
 // rawSet actually implements the key set operation
 func (obj *EmbdEtcd) rawSet(ctx context.Context, kv *KV) error {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: rawSet()")
 	}
 	// key is the full key path
@@ -945,7 +953,7 @@ func (obj *EmbdEtcd) rawSet(ctx context.Context, kv *KV) error {
 	response, err := obj.client.KV.Put(ctx, kv.key, kv.value, kv.opts...)
 	obj.rLock.RUnlock()
 	log.Printf("Etcd: Set(%s): %v", kv.key, response) // w00t... bonus
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: rawSet(): %v", err)
 	}
 	return err
@@ -970,7 +978,7 @@ func (obj *EmbdEtcd) ComplexGet(path string, skipConv bool, opts ...etcd.OpOptio
 }
 
 func (obj *EmbdEtcd) rawGet(ctx context.Context, gq *GQ) (result map[string]string, err error) {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: rawGet()")
 	}
 	obj.rLock.RLock()
@@ -986,7 +994,7 @@ func (obj *EmbdEtcd) rawGet(ctx context.Context, gq *GQ) (result map[string]stri
 		result[bytes.NewBuffer(x.Key).String()] = bytes.NewBuffer(x.Value).String()
 	}
 
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: rawGet(): %v", result)
 	}
 	return
@@ -1004,7 +1012,7 @@ func (obj *EmbdEtcd) Delete(path string, opts ...etcd.OpOption) (int64, error) {
 }
 
 func (obj *EmbdEtcd) rawDelete(ctx context.Context, dl *DL) (count int64, err error) {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: rawDelete()")
 	}
 	count = -1
@@ -1014,7 +1022,7 @@ func (obj *EmbdEtcd) rawDelete(ctx context.Context, dl *DL) (count int64, err er
 	if err == nil {
 		count = response.Deleted
 	}
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: rawDelete(): %v", err)
 	}
 	return
@@ -1032,13 +1040,13 @@ func (obj *EmbdEtcd) Txn(ifcmps []etcd.Cmp, thenops, elseops []etcd.Op) (*etcd.T
 }
 
 func (obj *EmbdEtcd) rawTxn(ctx context.Context, tn *TN) (*etcd.TxnResponse, error) {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: rawTxn()")
 	}
 	obj.rLock.RLock()
 	response, err := obj.client.KV.Txn(ctx).If(tn.ifcmps...).Then(tn.thenops...).Else(tn.elseops...).Commit()
 	obj.rLock.RUnlock()
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: rawTxn(): %v, %v", response, err)
 	}
 	return response, err
@@ -1072,7 +1080,7 @@ func (obj *EmbdEtcd) rawAddWatcher(ctx context.Context, aw *AW) (func(), error)
 			err := response.Err()
 			isCanceled := response.Canceled || err == context.Canceled
 			if response.Header.Revision == 0 { // by inspection
-				if global.DEBUG {
+				if obj.flags.Debug {
 					log.Printf("Etcd: Watch: Received empty message!") // switched client connection
 				}
 				isCanceled = true
@@ -1093,7 +1101,7 @@ func (obj *EmbdEtcd) rawAddWatcher(ctx context.Context, aw *AW) (func(), error)
 				}
 				locked = false
 			} else {
-				if global.DEBUG {
+				if obj.flags.Debug {
 					log.Printf("Etcd: Watch: Error: %v", err) // probably fixable
 				}
 				// this new context is the fix for a tricky set
@@ -1142,9 +1150,6 @@ func rawCallback(ctx context.Context, re *RE) error {
 			// NOTE: the callback must *not* block!
 			// FIXME: do we need to pass ctx in via *RE, or in the callback signature ?
 			err = callback(re) // run the callback
-			if global.TRACE {
-				log.Printf("Trace: Etcd: rawCallback(): %v", err)
-			}
 			if !re.errCheck || err == nil {
 				return nil
 			}
@@ -1160,7 +1165,7 @@ func rawCallback(ctx context.Context, re *RE) error {
 // FIXME: we might need to respond to member change/disconnect/shutdown events,
 // see: https://github.com/coreos/etcd/issues/5277
 func (obj *EmbdEtcd) volunteerCallback(re *RE) error {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: volunteerCallback()")
 		defer log.Printf("Trace: Etcd: volunteerCallback(): Finished!")
 	}
@@ -1348,7 +1353,7 @@ func (obj *EmbdEtcd) volunteerCallback(re *RE) error {
 // nominateCallback runs to respond to the nomination list change events
 // functionally, it controls the starting and stopping of the server process
 func (obj *EmbdEtcd) nominateCallback(re *RE) error {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: nominateCallback()")
 		defer log.Printf("Trace: Etcd: nominateCallback(): Finished!")
 	}
@@ -1397,7 +1402,7 @@ func (obj *EmbdEtcd) nominateCallback(re *RE) error {
 	_, exists := obj.nominated[obj.hostname]
 	// FIXME: can we get rid of the len(obj.nominated) == 0 ?
 	newCluster := len(obj.nominated) == 0 || (len(obj.nominated) == 1 && exists)
-	if global.DEBUG {
+	if obj.flags.Debug {
 		log.Printf("Etcd: nominateCallback(): newCluster: %v; exists: %v; obj.server == nil: %t", newCluster, exists, obj.server == nil)
 	}
 	// XXX: check if i have actually volunteered first of all...
@@ -1500,7 +1505,7 @@ func (obj *EmbdEtcd) nominateCallback(re *RE) error {
 
 // endpointCallback runs to respond to the endpoint list change events
 func (obj *EmbdEtcd) endpointCallback(re *RE) error {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: endpointCallback()")
 		defer log.Printf("Trace: Etcd: endpointCallback(): Finished!")
 	}
@@ -1566,7 +1571,7 @@ func (obj *EmbdEtcd) endpointCallback(re *RE) error {
 
 // idealClusterSizeCallback runs to respond to the ideal cluster size changes
 func (obj *EmbdEtcd) idealClusterSizeCallback(re *RE) error {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: idealClusterSizeCallback()")
 		defer log.Printf("Trace: Etcd: idealClusterSizeCallback(): Finished!")
 	}
@@ -1672,6 +1677,14 @@ func (obj *EmbdEtcd) StartServer(newCluster bool, peerURLsMap etcdtypes.URLsMap)
 		obj.serverwg.Add(1)      // add for the DestroyServer()
 		obj.DestroyServer()
 		return e
+	// TODO: should we wait for this notification elsewhere?
+	case <-obj.server.Server.StopNotify(): // it's going down now...
+		e := fmt.Errorf("Etcd: StartServer: Received stop notification.")
+		log.Printf(e.Error())
+		obj.server.Server.Stop() // trigger a shutdown
+		obj.serverwg.Add(1)      // add for the DestroyServer()
+		obj.DestroyServer()
+		return e
 	}
 	//log.Fatal(<-obj.server.Err())	XXX
 	log.Printf("Etcd: StartServer: Server running...")
@@ -1705,7 +1718,7 @@ func (obj *EmbdEtcd) DestroyServer() error {
 
 // EtcdNominate nominates a particular client to be a server (peer)
 func EtcdNominate(obj *EmbdEtcd, hostname string, urls etcdtypes.URLs) error {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: EtcdNominate(%v): %v", hostname, urls.String())
 		defer log.Printf("Trace: Etcd: EtcdNominate(%v): Finished!", hostname)
 	}
@@ -1747,7 +1760,7 @@ func EtcdNominated(obj *EmbdEtcd) (etcdtypes.URLsMap, error) {
 			return nil, fmt.Errorf("Etcd: Nominated: Data format error!: %v", err)
 		}
 		nominated[name] = urls // add to map
-		if global.DEBUG {
+		if obj.flags.Debug {
 			log.Printf("Etcd: Nominated(%v): %v", name, val)
 		}
 	}
@@ -1756,7 +1769,7 @@ func EtcdNominated(obj *EmbdEtcd) (etcdtypes.URLsMap, error) {
 
 // EtcdVolunteer offers yourself up to be a server if needed
 func EtcdVolunteer(obj *EmbdEtcd, urls etcdtypes.URLs) error {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: EtcdVolunteer(%v): %v", obj.hostname, urls.String())
 		defer log.Printf("Trace: Etcd: EtcdVolunteer(%v): Finished!", obj.hostname)
 	}
@@ -1779,7 +1792,7 @@ func EtcdVolunteer(obj *EmbdEtcd, urls etcdtypes.URLs) error {
 
 // EtcdVolunteers returns a urls map of available etcd server volunteers
 func EtcdVolunteers(obj *EmbdEtcd) (etcdtypes.URLsMap, error) {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: EtcdVolunteers()")
 		defer log.Printf("Trace: Etcd: EtcdVolunteers(): Finished!")
 	}
@@ -1802,7 +1815,7 @@ func EtcdVolunteers(obj *EmbdEtcd) (etcdtypes.URLsMap, error) {
 			return nil, fmt.Errorf("Etcd: Volunteers: Data format error!: %v", err)
 		}
 		volunteers[name] = urls // add to map
-		if global.DEBUG {
+		if obj.flags.Debug {
 			log.Printf("Etcd: Volunteer(%v): %v", name, val)
 		}
 	}
@@ -1811,7 +1824,7 @@ func EtcdVolunteers(obj *EmbdEtcd) (etcdtypes.URLsMap, error) {
 
 // EtcdAdvertiseEndpoints advertises the list of available client endpoints
 func EtcdAdvertiseEndpoints(obj *EmbdEtcd, urls etcdtypes.URLs) error {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: EtcdAdvertiseEndpoints(%v): %v", obj.hostname, urls.String())
 		defer log.Printf("Trace: Etcd: EtcdAdvertiseEndpoints(%v): Finished!", obj.hostname)
 	}
@@ -1834,7 +1847,7 @@ func EtcdAdvertiseEndpoints(obj *EmbdEtcd, urls etcdtypes.URLs) error {
 
 // EtcdEndpoints returns a urls map of available etcd server endpoints
 func EtcdEndpoints(obj *EmbdEtcd) (etcdtypes.URLsMap, error) {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: EtcdEndpoints()")
 		defer log.Printf("Trace: Etcd: EtcdEndpoints(): Finished!")
 	}
@@ -1857,7 +1870,7 @@ func EtcdEndpoints(obj *EmbdEtcd) (etcdtypes.URLsMap, error) {
 			return nil, fmt.Errorf("Etcd: Endpoints: Data format error!: %v", err)
 		}
 		endpoints[name] = urls // add to map
-		if global.DEBUG {
+		if obj.flags.Debug {
 			log.Printf("Etcd: Endpoint(%v): %v", name, val)
 		}
 	}
@@ -1866,7 +1879,7 @@ func EtcdEndpoints(obj *EmbdEtcd) (etcdtypes.URLsMap, error) {
 
 // EtcdSetHostnameConverged sets whether a specific hostname is converged.
 func EtcdSetHostnameConverged(obj *EmbdEtcd, hostname string, isConverged bool) error {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: EtcdSetHostnameConverged(%s): %v", hostname, isConverged)
 		defer log.Printf("Trace: Etcd: EtcdSetHostnameConverged(%v): Finished!", hostname)
 	}
@@ -1880,7 +1893,7 @@ func EtcdSetHostnameConverged(obj *EmbdEtcd, hostname string, isConverged bool)
 
 // EtcdHostnameConverged returns a map of every hostname's converged state.
 func EtcdHostnameConverged(obj *EmbdEtcd) (map[string]bool, error) {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: EtcdHostnameConverged()")
 		defer log.Printf("Trace: Etcd: EtcdHostnameConverged(): Finished!")
 	}
@@ -1925,7 +1938,7 @@ func EtcdAddHostnameConvergedWatcher(obj *EmbdEtcd, callbackFn func(map[string]b
 
 // EtcdSetClusterSize sets the ideal target cluster size of etcd peers
 func EtcdSetClusterSize(obj *EmbdEtcd, value uint16) error {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: EtcdSetClusterSize(): %v", value)
 		defer log.Printf("Trace: Etcd: EtcdSetClusterSize(): Finished!")
 	}
@@ -2019,7 +2032,7 @@ func EtcdMembers(obj *EmbdEtcd) (map[uint64]string, error) {
 			return nil, fmt.Errorf("Exiting...")
 		}
 		obj.rLock.RLock()
-		if global.TRACE {
+		if obj.flags.Trace {
 			log.Printf("Trace: Etcd: EtcdMembers(): Endpoints are: %v", obj.client.Endpoints())
 		}
 		response, err = obj.client.MemberList(ctx)
@@ -2271,9 +2284,7 @@ func ApplyDeltaEvents(re *RE, urlsmap etcdtypes.URLsMap) (etcdtypes.URLsMap, err
 			if _, exists := urlsmap[key]; !exists {
 				// this can happen if we retry an operation b/w
 				// a reconnect so ignore if we are reconnecting
-				if global.DEBUG {
+				log.Printf("Etcd: ApplyDeltaEvents: Inconsistent key: %v", key)
-					log.Printf("Etcd: ApplyDeltaEvents: Inconsistent key: %v", key)
-				}
 				return nil, errApplyDeltaEventsInconsistent
 			}
 			delete(urlsmap, key)

etcd/world.go

@@ -0,0 +1,43 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package etcd
+
+import (
+	"github.com/purpleidea/mgmt/resources"
+)
+
+// World is an etcd backed implementation of the World interface.
+type World struct {
+	Hostname string // uuid for the consumer of these
+	EmbdEtcd *EmbdEtcd
+}
+
+// ResExport exports a list of resources under our hostname namespace.
+// Subsequent calls replace the previously set collection atomically.
+func (obj *World) ResExport(resourceList []resources.Res) error {
+	return EtcdSetResources(obj.EmbdEtcd, obj.Hostname, resourceList)
+}
+
+// ResCollect gets the collection of exported resources which match the filter.
+// It does this atomically so that a call always returns a complete collection.
+func (obj *World) ResCollect(hostnameFilter, kindFilter []string) ([]resources.Res, error) {
+	// XXX: should we be restricted to retrieving resources that were
+	// exported with a tag that allows or restricts our hostname? We could
+	// enforce that here if the underlying API supported it... Add this?
+	return EtcdGetResources(obj.EmbdEtcd, hostnameFilter, kindFilter)
+}

examples/hostname.yaml

@@ -0,0 +1,7 @@
+---
+graph: mygraph
+resources:
+  hostname:
+    - name: Hostname Watcher @ TestHost
+      hostname: test.hostname.example.com
+edges: []

examples/lib/libmgmt1.go

@@ -11,7 +11,7 @@ import (
 	"time"
 
 	"github.com/purpleidea/mgmt/gapi"
-	mgmt "github.com/purpleidea/mgmt/mgmtmain"
+	mgmt "github.com/purpleidea/mgmt/lib"
 	"github.com/purpleidea/mgmt/pgraph"
 	"github.com/purpleidea/mgmt/resources"
 	"github.com/purpleidea/mgmt/yamlgraph"
@@ -81,12 +81,12 @@ func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
 		Comment: "comment!",
 	}
 
-	g, err := gc.NewGraphFromConfig(obj.data.Hostname, obj.data.EmbdEtcd, obj.data.Noop)
+	g, err := gc.NewGraphFromConfig(obj.data.Hostname, obj.data.World, obj.data.Noop)
 	return g, err
 }
 
-// SwitchStream returns nil errors every time there could be a new graph.
+// Next returns nil errors every time there could be a new graph.
-func (obj *MyGAPI) SwitchStream() chan error {
+func (obj *MyGAPI) Next() chan error {
 	if obj.data.NoWatch || obj.Interval <= 0 {
 		return nil
 	}

examples/lib/libmgmt2.go

@@ -12,7 +12,7 @@ import (
 	"time"
 
 	"github.com/purpleidea/mgmt/gapi"
-	mgmt "github.com/purpleidea/mgmt/mgmtmain"
+	mgmt "github.com/purpleidea/mgmt/lib"
 	"github.com/purpleidea/mgmt/pgraph"
 	"github.com/purpleidea/mgmt/resources"
 )
@@ -74,12 +74,12 @@ func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
 		vertex = v // save
 	}
 
-	//g, err := config.NewGraphFromConfig(obj.data.Hostname, obj.data.EmbdEtcd, obj.data.Noop)
+	//g, err := config.NewGraphFromConfig(obj.data.Hostname, obj.data.World, obj.data.Noop)
 	return g, nil
 }
 
-// SwitchStream returns nil errors every time there could be a new graph.
+// Next returns nil errors every time there could be a new graph.
-func (obj *MyGAPI) SwitchStream() chan error {
+func (obj *MyGAPI) Next() chan error {
 	if obj.data.NoWatch || obj.Interval <= 0 {
 		return nil
 	}

examples/lib/libmgmt3.go

@@ -0,0 +1,225 @@
+// libmgmt example of send->recv
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/purpleidea/mgmt/gapi"
+	mgmt "github.com/purpleidea/mgmt/lib"
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/resources"
+)
+
+// MyGAPI implements the main GAPI interface.
+type MyGAPI struct {
+	Name     string // graph name
+	Interval uint   // refresh interval, 0 to never refresh
+
+	data        gapi.Data
+	initialized bool
+	closeChan   chan struct{}
+	wg          sync.WaitGroup // sync group for tunnel go routines
+}
+
+// NewMyGAPI creates a new MyGAPI struct and calls Init().
+func NewMyGAPI(data gapi.Data, name string, interval uint) (*MyGAPI, error) {
+	obj := &MyGAPI{
+		Name:     name,
+		Interval: interval,
+	}
+	return obj, obj.Init(data)
+}
+
+// Init initializes the MyGAPI struct.
+func (obj *MyGAPI) Init(data gapi.Data) error {
+	if obj.initialized {
+		return fmt.Errorf("Already initialized!")
+	}
+	if obj.Name == "" {
+		return fmt.Errorf("The graph name must be specified!")
+	}
+	obj.data = data // store for later
+	obj.closeChan = make(chan struct{})
+	obj.initialized = true
+	return nil
+}
+
+// Graph returns a current Graph.
+func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
+	if !obj.initialized {
+		return nil, fmt.Errorf("libmgmt: MyGAPI is not initialized")
+	}
+
+	g := pgraph.NewGraph(obj.Name)
+
+	content := "Delete me to trigger a notification!\n"
+	f0 := &resources.FileRes{
+		BaseRes: resources.BaseRes{
+			Name: "README",
+		},
+		Path:    "/tmp/mgmt/README",
+		Content: &content,
+		State:   "present",
+	}
+
+	v0 := pgraph.NewVertex(f0)
+	g.AddVertex(v0)
+
+	p1 := &resources.PasswordRes{
+		BaseRes: resources.BaseRes{
+			Name: "password1",
+		},
+		Length: 8,    // generated string will have this many characters
+		Saved:  true, // this causes passwords to be stored in plain text!
+	}
+	v1 := pgraph.NewVertex(p1)
+	g.AddVertex(v1)
+
+	f1 := &resources.FileRes{
+		BaseRes: resources.BaseRes{
+			Name: "file1",
+			// send->recv!
+			Recv: map[string]*resources.Send{
+				"Content": {Res: p1, Key: "Password"},
+			},
+		},
+		Path: "/tmp/mgmt/secret",
+		//Content:  p1.Password, // won't work
+		State: "present",
+	}
+
+	v2 := pgraph.NewVertex(f1)
+	g.AddVertex(v2)
+
+	n1 := &resources.NoopRes{
+		BaseRes: resources.BaseRes{
+			Name: "noop1",
+		},
+	}
+
+	v3 := pgraph.NewVertex(n1)
+	g.AddVertex(v3)
+
+	e0 := pgraph.NewEdge("e0")
+	e0.Notify = true // send a notification from v0 to v1
+	g.AddEdge(v0, v1, e0)
+
+	g.AddEdge(v1, v2, pgraph.NewEdge("e1"))
+
+	e2 := pgraph.NewEdge("e2")
+	e2.Notify = true // send a notification from v2 to v3
+	g.AddEdge(v2, v3, e2)
+
+	//g, err := config.NewGraphFromConfig(obj.data.Hostname, obj.data.World, obj.data.Noop)
+	return g, nil
+}
+
+// Next returns nil errors every time there could be a new graph.
+func (obj *MyGAPI) Next() chan error {
+	if obj.data.NoWatch || obj.Interval <= 0 {
+		return nil
+	}
+	ch := make(chan error)
+	obj.wg.Add(1)
+	go func() {
+		defer obj.wg.Done()
+		defer close(ch) // this will run before the obj.wg.Done()
+		if !obj.initialized {
+			ch <- fmt.Errorf("libmgmt: MyGAPI is not initialized")
+			return
+		}
+
+		// arbitrarily change graph every interval seconds
+		ticker := time.NewTicker(time.Duration(obj.Interval) * time.Second)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ticker.C:
+				log.Printf("libmgmt: Generating new graph...")
+				ch <- nil // trigger a run
+			case <-obj.closeChan:
+				return
+			}
+		}
+	}()
+	return ch
+}
+
+// Close shuts down the MyGAPI.
+func (obj *MyGAPI) Close() error {
+	if !obj.initialized {
+		return fmt.Errorf("libmgmt: MyGAPI is not initialized")
+	}
+	close(obj.closeChan)
+	obj.wg.Wait()
+	obj.initialized = false // closed = true
+	return nil
+}
+
+// Run runs an embedded mgmt server.
+func Run() error {
+
+	obj := &mgmt.Main{}
+	obj.Program = "libmgmt" // TODO: set on compilation
+	obj.Version = "0.0.1"   // TODO: set on compilation
+	obj.TmpPrefix = true    // disable for easy debugging
+	//prefix := "/tmp/testprefix/"
+	//obj.Prefix = &p // enable for easy debugging
+	obj.IdealClusterSize = -1
+	obj.ConvergedTimeout = -1
+	obj.Noop = false // FIXME: careful!
+
+	obj.GAPI = &MyGAPI{ // graph API
+		Name:     "libmgmt", // TODO: set on compilation
+		Interval: 60 * 10,   // arbitrarily change graph every 15 seconds
+	}
+
+	if err := obj.Init(); err != nil {
+		return err
+	}
+
+	// install the exit signal handler
+	exit := make(chan struct{})
+	defer close(exit)
+	go func() {
+		signals := make(chan os.Signal, 1)
+		signal.Notify(signals, os.Interrupt) // catch ^C
+		//signal.Notify(signals, os.Kill) // catch signals
+		signal.Notify(signals, syscall.SIGTERM)
+
+		select {
+		case sig := <-signals: // any signal will do
+			if sig == os.Interrupt {
+				log.Println("Interrupted by ^C")
+				obj.Exit(nil)
+				return
+			}
+			log.Println("Interrupted by signal")
+			obj.Exit(fmt.Errorf("Killed by %v", sig))
+			return
+		case <-exit:
+			return
+		}
+	}()
+
+	if err := obj.Run(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func main() {
+	log.Printf("Hello!")
+	if err := Run(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+		return
+	}
+	log.Printf("Goodbye!")
+}

examples/nspawn1.yaml

@@ -0,0 +1,7 @@
+---
+graph: mygraph
+resources:
+  nspawn:
+  - name: mgmt-nspawn1
+    state: running
+edges: []

examples/nspawn2.yaml

@@ -0,0 +1,7 @@
+---
+graph: mygraph
+resources:
+  nspawn:
+  - name: mgmt-nspawn2
+    state: stopped
+edges: []

gapi/gapi.go

@@ -19,14 +19,23 @@
 package gapi
 
 import (
-	"github.com/purpleidea/mgmt/etcd"
 	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/resources"
 )
 
+// World is an interface to the rest of the different graph state. It allows
+// the GAPI to store state and exchange information throughout the cluster. It
+// is the interface each machine uses to communicate with the rest of the world.
+type World interface { // TODO: is there a better name for this interface?
+	ResExport([]resources.Res) error
+	// FIXME: should this method take a "filter" data struct instead of many args?
+	ResCollect(hostnameFilter, kindFilter []string) ([]resources.Res, error)
+}
+
 // Data is the set of input values passed into the GAPI structs via Init.
 type Data struct {
 	Hostname string // uuid for the host, required for GAPI
-	EmbdEtcd *etcd.EmbdEtcd
+	World    World
 	Noop     bool
 	NoWatch  bool
 	// NOTE: we can add more fields here if needed by GAPI endpoints
@@ -36,6 +45,6 @@ type Data struct {
 type GAPI interface {
 	Init(Data) error               // initializes the GAPI and passes in useful data
 	Graph() (*pgraph.Graph, error) // returns the most recent pgraph
-	SwitchStream() chan error      // returns a stream of switch events
+	Next() chan error              // returns a stream of switch events
 	Close() error                  // shutdown the GAPI
 }

lib/cli.go

@@ -15,7 +15,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package mgmtmain
+package lib
 
 import (
 	"fmt"
@@ -37,6 +37,11 @@ func run(c *cli.Context) error {
 
 	obj.Program = c.App.Name
 	obj.Version = c.App.Version
+	if val, exists := c.App.Metadata["flags"]; exists {
+		if flags, ok := val.(Flags); ok {
+			obj.Flags = flags
+		}
+	}
 
 	if h := c.String("hostname"); c.IsSet("hostname") && h != "" {
 		obj.Hostname = &h
@@ -96,6 +101,16 @@ func run(c *cli.Context) error {
 	obj.NoCaching = c.Bool("no-caching")
 	obj.Depth = uint16(c.Int("depth"))
 
+	obj.NoPgp = c.Bool("no-pgp")
+
+	if kp := c.String("pgp-key-path"); c.IsSet("pgp-key-path") {
+		obj.PgpKeyPath = &kp
+	}
+
+	if us := c.String("pgp-identity"); c.IsSet("pgp-identity") {
+		obj.PgpIdentity = &us
+	}
+
 	if err := obj.Init(); err != nil {
 		return err
 	}
@@ -133,7 +148,7 @@ func run(c *cli.Context) error {
 }
 
 // CLI is the entry point for using mgmt normally from the CLI.
-func CLI(program, version string) error {
+func CLI(program, version string, flags Flags) error {
 
 	// test for sanity
 	if program == "" || version == "" {
@@ -143,6 +158,9 @@ func CLI(program, version string) error {
 	app.Name = program // App.name and App.version pass these values through
 	app.Version = version
 	app.Usage = "next generation config management"
+	app.Metadata = map[string]interface{}{ // additional flags
+		"flags": flags,
+	}
 	//app.Action = ... // without a default action, help runs
 
 	app.Commands = []cli.Command{
@@ -288,6 +306,20 @@ func CLI(program, version string) error {
 					Value:  0,
 					Usage:  "specify depth in remote hierarchy",
 				},
+				cli.BoolFlag{
+					Name:  "no-pgp",
+					Usage: "don't create pgp keys",
+				},
+				cli.StringFlag{
+					Name:  "pgp-key-path",
+					Value: "",
+					Usage: "path for instance key pair",
+				},
+				cli.StringFlag{
+					Name:  "pgp-identity",
+					Value: "",
+					Usage: "default identity used for generation",
+				},
 			},
 		},
 	}

lib/main.go

@@ -15,22 +15,24 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package mgmtmain
+package lib
 
 import (
 	"fmt"
 	"io/ioutil"
 	"log"
 	"os"
-	"sync"
+	"path"
 	"time"
 
 	"github.com/purpleidea/mgmt/converger"
 	"github.com/purpleidea/mgmt/etcd"
 	"github.com/purpleidea/mgmt/gapi"
+	"github.com/purpleidea/mgmt/pgp"
 	"github.com/purpleidea/mgmt/pgraph"
 	"github.com/purpleidea/mgmt/recwatch"
 	"github.com/purpleidea/mgmt/remote"
+	"github.com/purpleidea/mgmt/resources"
 	"github.com/purpleidea/mgmt/util"
 
 	etcdtypes "github.com/coreos/etcd/pkg/types"
@@ -39,11 +41,20 @@ import (
 	errwrap "github.com/pkg/errors"
 )
 
+// Flags are some constant flags which are used throughout the program.
+type Flags struct {
+	Debug   bool // add additional log messages
+	Trace   bool // add execution flow log messages
+	Verbose bool // add extra log message output
+}
+
 // Main is the main struct for running the mgmt logic.
 type Main struct {
 	Program string // the name of this program, usually set at compile time
 	Version string // the version of this program, usually set at compile time
 
+	Flags Flags // static global flags that are set at compile time
+
 	Hostname *string // hostname to use; nil if undefined
 
 	Prefix         *string // prefix passed in; nil if undefined
@@ -72,14 +83,16 @@ type Main struct {
 	NoCaching        bool   // don't allow remote caching of remote execution binary
 	Depth            uint16 // depth in remote hierarchy; for internal use only
 
-	DEBUG   bool
-	VERBOSE bool
-
 	seeds            etcdtypes.URLs // processed seeds value
 	clientURLs       etcdtypes.URLs // processed client urls value
 	serverURLs       etcdtypes.URLs // processed server urls value
 	idealClusterSize uint16         // processed ideal cluster size value
 
+	NoPgp       bool    // disallow pgp functionality
+	PgpKeyPath  *string // import a pre-made key pair
+	PgpIdentity *string
+	pgpKeys     *pgp.PGP // agent key pair
+
 	exit chan error // exit signal
 }
 
@@ -159,7 +172,7 @@ func (obj *Main) Run() error {
 	var start = time.Now().UnixNano()
 
 	var flags int
-	if obj.DEBUG || true { // TODO: remove || true
+	if obj.Flags.Debug || true { // TODO: remove || true
 		flags = log.LstdFlags | log.Lshortfile
 	}
 	flags = (flags - log.Ldate) // remove the date for now
@@ -167,7 +180,7 @@ func (obj *Main) Run() error {
 
 	// un-hijack from capnslog...
 	log.SetOutput(os.Stderr)
-	if obj.VERBOSE {
+	if obj.Flags.Verbose {
 		capnslog.SetFormatter(capnslog.NewLogFormatter(os.Stderr, "(etcd) ", flags))
 	} else {
 		capnslog.SetFormatter(capnslog.NewNilFormatter())
@@ -205,8 +218,54 @@ func (obj *Main) Run() error {
 		}
 	}
 	log.Printf("Main: Working prefix is: %s", prefix)
+	pgraphPrefix := fmt.Sprintf("%s/", path.Join(prefix, "pgraph")) // pgraph namespace
+	if err := os.MkdirAll(pgraphPrefix, 0770); err != nil {
+		return errwrap.Wrapf(err, "Can't create pgraph prefix")
+	}
+
+	if !obj.NoPgp {
+		pgpPrefix := fmt.Sprintf("%s/", path.Join(prefix, "pgp"))
+		if err := os.MkdirAll(pgpPrefix, 0770); err != nil {
+			return errwrap.Wrapf(err, "Can't create pgp prefix")
+		}
+
+		pgpKeyringPath := path.Join(pgpPrefix, pgp.DefaultKeyringFile) // default path
+
+		if p := obj.PgpKeyPath; p != nil {
+			pgpKeyringPath = *p
+		}
+
+		var err error
+		if obj.pgpKeys, err = pgp.Import(pgpKeyringPath); err != nil && !os.IsNotExist(err) {
+			return errwrap.Wrapf(err, "Can't import pgp key")
+		}
+
+		if obj.pgpKeys == nil {
+
+			identity := fmt.Sprintf("%s <%s> %s", obj.Program, "root@"+hostname, "generated by "+obj.Program)
+			if p := obj.PgpIdentity; p != nil {
+				identity = *p
+			}
+
+			name, comment, email, err := pgp.ParseIdentity(identity)
+			if err != nil {
+				return errwrap.Wrapf(err, "Can't parse user string")
+
+			}
+
+			// TODO: Make hash configurable
+			if obj.pgpKeys, err = pgp.Generate(name, comment, email, nil); err != nil {
+				return errwrap.Wrapf(err, "Can't creating pgp key")
+			}
+
+			if err := obj.pgpKeys.SaveKey(pgpKeyringPath); err != nil {
+				return errwrap.Wrapf(err, "Can't save pgp key")
+			}
+		}
+
+		// TODO: Import admin key
+	}
 
-	var wg sync.WaitGroup
 	var G, oldGraph *pgraph.Graph
 
 	// exit after `max-runtime` seconds for no reason at all...
@@ -237,6 +296,11 @@ func (obj *Main) Run() error {
 		obj.serverURLs,
 		obj.NoServer,
 		obj.idealClusterSize,
+		etcd.Flags{
+			Debug:   obj.Flags.Debug,
+			Trace:   obj.Flags.Trace,
+			Verbose: obj.Flags.Verbose,
+		},
 		prefix,
 		converger,
 	)
@@ -269,32 +333,36 @@ func (obj *Main) Run() error {
 	if obj.GAPI != nil {
 		data := gapi.Data{
 			Hostname: hostname,
-			EmbdEtcd: EmbdEtcd,
+			// NOTE: alternate implementations can be substituted in
-			Noop:     obj.Noop,
+			World: &etcd.World{
-			NoWatch:  obj.NoWatch,
+				Hostname: hostname,
+				EmbdEtcd: EmbdEtcd,
+			},
+			Noop:    obj.Noop,
+			NoWatch: obj.NoWatch,
 		}
 		if err := obj.GAPI.Init(data); err != nil {
 			obj.Exit(fmt.Errorf("Main: GAPI: Init failed: %v", err))
 		} else if !obj.NoWatch {
-			gapiChan = obj.GAPI.SwitchStream() // stream of graph switch events!
+			gapiChan = obj.GAPI.Next() // stream of graph switch events!
 		}
 	}
 
 	exitchan := make(chan struct{}) // exit on close
 	go func() {
-		startchan := make(chan struct{}) // start signal
+		startChan := make(chan struct{}) // start signal
-		go func() { startchan <- struct{}{} }()
+		go func() { startChan <- struct{}{} }()
 
 		log.Println("Etcd: Starting...")
-		etcdchan := etcd.EtcdWatch(EmbdEtcd)
+		etcdChan := etcd.EtcdWatch(EmbdEtcd)
 		first := true // first loop or not
 		for {
 			log.Println("Main: Waiting...")
 			select {
-			case <-startchan: // kick the loop once at start
+			case <-startChan: // kick the loop once at start
 				// pass
 
-			case b := <-etcdchan:
+			case b := <-etcdChan:
 				if !b { // ignore the message
 					continue
 				}
@@ -302,6 +370,10 @@ func (obj *Main) Run() error {
 
 			case err, ok := <-gapiChan:
 				if !ok { // channel closed
+					if obj.Flags.Debug {
+						log.Printf("Main: GAPI exited")
+					}
+					gapiChan = nil // disable it
 					continue
 				}
 				if err != nil {
@@ -338,11 +410,18 @@ func (obj *Main) Run() error {
 				log.Printf("Config: Error creating new graph: %v", err)
 				// unpause!
 				if !first {
-					G.Start(&wg, first) // sync
+					G.Start(first)    // sync
-					converger.Start()   // after G.Start()
+					converger.Start() // after G.Start()
 				}
 				continue
 			}
+			newGraph.Flags = pgraph.Flags{Debug: obj.Flags.Debug}
+			// pass in the information we need
+			newGraph.AssociateData(&resources.Data{
+				Converger: converger,
+				Prefix:    pgraphPrefix,
+				Debug:     obj.Flags.Debug,
+			})
 
 			// apply the global noop parameter if requested
 			if obj.Noop {
@@ -359,8 +438,8 @@ func (obj *Main) Run() error {
 				log.Printf("Config: Error running graph sync: %v", err)
 				// unpause!
 				if !first {
-					G.Start(&wg, first) // sync
+					G.Start(first)    // sync
-					converger.Start()   // after G.Start()
+					converger.Start() // after G.Start()
 				}
 				continue
 			}
@@ -370,6 +449,7 @@ func (obj *Main) Run() error {
 			G.AutoEdges() // add autoedges; modifies the graph
 			G.AutoGroup() // run autogroup; modifies the graph
 			// TODO: do we want to do a transitive reduction?
+			// FIXME: run a type checker that verifies all the send->recv relationships
 
 			log.Printf("Graph: %v", G) // show graph
 			if obj.GraphvizFilter != "" {
@@ -379,19 +459,19 @@ func (obj *Main) Run() error {
 					log.Printf("Graphviz: Successfully generated graph!")
 				}
 			}
-			G.AssociateData(converger)
 			// G.Start(...) needs to be synchronous or wait,
 			// because if half of the nodes are started and
 			// some are not ready yet and the EtcdWatch
 			// loops, we'll cause G.Pause(...) before we
 			// even got going, thus causing nil pointer errors
-			G.Start(&wg, first) // sync
+			G.Start(first)    // sync
-			converger.Start()   // after G.Start()
+			converger.Start() // after G.Start()
 			first = false
 		}
 	}()
 
 	configWatcher := recwatch.NewConfigWatcher()
+	configWatcher.Flags = recwatch.Flags{Debug: obj.Flags.Debug}
 	events := configWatcher.Events()
 	if !obj.NoWatch {
 		configWatcher.Add(obj.Remotes...) // add all the files...
@@ -428,7 +508,10 @@ func (obj *Main) Run() error {
 		prefix,
 		converger,
 		convergerCb,
-		obj.Program,
+		remote.Flags{
+			Program: obj.Program,
+			Debug:   obj.Flags.Debug,
+		},
 	)
 
 	// TODO: is there any benefit to running the remotes above in the loop?
@@ -457,23 +540,21 @@ func (obj *Main) Run() error {
 		reterr = multierr.Append(reterr, err) // list of errors
 	}
 
-	G.Exit() // tell all the children to exit
-
 	// tell inner main loop to exit
 	close(exitchan)
 
+	G.Exit() // tell all the children to exit, and waits for them to do so
+
 	// cleanup etcd main loop last so it can process everything first
 	if err := EmbdEtcd.Destroy(); err != nil { // shutdown and cleanup etcd
 		err = errwrap.Wrapf(err, "Etcd exited poorly!")
 		reterr = multierr.Append(reterr, err) // list of errors
 	}
 
-	if obj.DEBUG {
+	if obj.Flags.Debug {
-		log.Printf("Graph: %v", G)
+		log.Printf("Main: Graph: %v", G)
 	}
 
-	wg.Wait() // wait for primary go routines to exit
-
 	// TODO: wait for each vertex to exit...
 	log.Println("Goodbye!")
 	return reterr

main.go

@@ -21,7 +21,14 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/purpleidea/mgmt/mgmtmain"
+	mgmt "github.com/purpleidea/mgmt/lib"
+)
+
+// These constants are some global variables that are used throughout the code.
+const (
+	DEBUG   = false // add additional log messages
+	TRACE   = false // add execution flow log messages
+	VERBOSE = false // add extra log message output
 )
 
 // set at compile time
@@ -31,7 +38,12 @@ var (
 )
 
 func main() {
-	if err := mgmtmain.CLI(program, version); err != nil {
+	flags := mgmt.Flags{
+		Debug:   DEBUG,
+		Trace:   TRACE,
+		Verbose: VERBOSE,
+	}
+	if err := mgmt.CLI(program, version, flags); err != nil {
 		fmt.Println(err)
 		os.Exit(1)
 		return

misc/make-deps.sh

@@ -12,7 +12,14 @@ fi
 sudo_command=$(which sudo)
 
 YUM=`which yum 2>/dev/null`
+DNF=`which dnf 2>/dev/null`
 APT=`which apt-get 2>/dev/null`
+
+# if DNF is available use it
+if [ -x "$DNF" ]; then
+	YUM=$DNF
+fi
+
 if [ -z "$YUM" -a -z "$APT" ]; then
 	echo "The package managers can't be found."
 	exit 1

pgp/pgp.go

@@ -0,0 +1,230 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgp
+
+import (
+	"bufio"
+	"bytes"
+	"crypto"
+	"encoding/base64"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+
+	errwrap "github.com/pkg/errors"
+	"golang.org/x/crypto/openpgp"
+	"golang.org/x/crypto/openpgp/packet"
+)
+
+// DefaultKeyringFile is the default file name for keyrings.
+const DefaultKeyringFile = "keyring.pgp"
+
+// CONFIG set default Hash.
+var CONFIG packet.Config
+
+func init() {
+	CONFIG.DefaultHash = crypto.SHA256
+}
+
+// PGP contains base entity.
+type PGP struct {
+	Entity *openpgp.Entity
+}
+
+// Import private key from defined path.
+func Import(privKeyPath string) (*PGP, error) {
+
+	privKeyFile, err := os.Open(privKeyPath)
+	if err != nil {
+		return nil, err
+	}
+	defer privKeyFile.Close()
+
+	file := packet.NewReader(bufio.NewReader(privKeyFile))
+	entity, err := openpgp.ReadEntity(file)
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "can't read entity from path")
+	}
+
+	obj := &PGP{
+		Entity: entity,
+	}
+
+	log.Printf("PGP: Imported key: %s", obj.Entity.PrivateKey.KeyIdShortString())
+	return obj, nil
+}
+
+// Generate creates new key pair. This key pair must be saved or it will be lost.
+func Generate(name, comment, email string, hash *crypto.Hash) (*PGP, error) {
+	if hash != nil {
+		CONFIG.DefaultHash = *hash
+	}
+	// generate a new public/private key pair
+	entity, err := openpgp.NewEntity(name, comment, email, &CONFIG)
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "can't generate entity")
+	}
+
+	obj := &PGP{
+		Entity: entity,
+	}
+
+	log.Printf("PGP: Created key: %s", obj.Entity.PrivateKey.KeyIdShortString())
+	return obj, nil
+}
+
+// SaveKey writes the whole entity (including private key!) to a .gpg file.
+func (obj *PGP) SaveKey(path string) error {
+	f, err := os.Create(path)
+	if err != nil {
+		return errwrap.Wrapf(err, "can't create file from given path")
+	}
+
+	w := bufio.NewWriter(f)
+	if err != nil {
+		return errwrap.Wrapf(err, "can't create writer")
+	}
+
+	if err := obj.Entity.SerializePrivate(w, &CONFIG); err != nil {
+		return errwrap.Wrapf(err, "can't serialize private key")
+	}
+
+	for _, ident := range obj.Entity.Identities {
+		for _, sig := range ident.Signatures {
+			if err := sig.Serialize(w); err != nil {
+				return errwrap.Wrapf(err, "can't serialize signature")
+			}
+		}
+	}
+
+	if err := w.Flush(); err != nil {
+		return errwrap.Wrapf(err, "enable to flush writer")
+	}
+
+	return nil
+}
+
+// WriteFile from given buffer in specified path.
+func (obj *PGP) WriteFile(path string, buff *bytes.Buffer) error {
+	w, err := createWriter(path)
+	if err != nil {
+		return errwrap.Wrapf(err, "can't create writer")
+	}
+	buff.WriteTo(w)
+
+	if err := w.Flush(); err != nil {
+		return errwrap.Wrapf(err, "can't flush buffered data")
+	}
+	return nil
+}
+
+// CreateWriter remove duplicate function.
+func createWriter(path string) (*bufio.Writer, error) {
+	f, err := os.Create(path)
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "can't create file from given path")
+	}
+	return bufio.NewWriter(f), nil
+}
+
+// Encrypt message for specified entity.
+func (obj *PGP) Encrypt(to *openpgp.Entity, msg string) (string, error) {
+	buf, err := obj.EncryptMsg(to, msg)
+	if err != nil {
+		return "", errwrap.Wrapf(err, "can't encrypt message")
+	}
+
+	// encode to base64
+	bytes, err := ioutil.ReadAll(buf)
+	if err != nil {
+		return "", errwrap.Wrapf(err, "can't read unverified body")
+	}
+	return base64.StdEncoding.EncodeToString(bytes), nil
+}
+
+// EncryptMsg encrypts the message.
+func (obj *PGP) EncryptMsg(to *openpgp.Entity, msg string) (*bytes.Buffer, error) {
+	ents := []*openpgp.Entity{to}
+
+	buf := new(bytes.Buffer)
+	w, err := openpgp.Encrypt(buf, ents, obj.Entity, nil, nil)
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "can't encrypt message")
+	}
+
+	_, err = w.Write([]byte(msg))
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "can't write to buffer")
+	}
+
+	if err = w.Close(); err != nil {
+		return nil, errwrap.Wrapf(err, "can't close writer")
+	}
+	return buf, nil
+}
+
+// Decrypt an encrypted msg.
+func (obj *PGP) Decrypt(encString string) (string, error) {
+	entityList := openpgp.EntityList{obj.Entity}
+
+	// decode the base64 string
+	dec, err := base64.StdEncoding.DecodeString(encString)
+	if err != nil {
+		return "", errwrap.Wrapf(err, "fail at decoding encrypted string")
+	}
+
+	// decrypt it with the contents of the private key
+	md, err := openpgp.ReadMessage(bytes.NewBuffer(dec), entityList, nil, nil)
+	if err != nil {
+		return "", errwrap.Wrapf(err, "can't read message")
+	}
+
+	bytes, err := ioutil.ReadAll(md.UnverifiedBody)
+	if err != nil {
+		return "", errwrap.Wrapf(err, "can't read unverified body")
+	}
+	return string(bytes), nil
+}
+
+// GetIdentities return the first identities from current object.
+func (obj *PGP) GetIdentities() (string, error) {
+	identities := []*openpgp.Identity{}
+
+	for _, v := range obj.Entity.Identities {
+		identities = append(identities, v)
+	}
+	return identities[0].Name, nil
+}
+
+// ParseIdentity parses an identity into name, comment and email components.
+func ParseIdentity(identity string) (name, comment, email string, err error) {
+	// get name
+	n := strings.Split(identity, " <")
+	if len(n) != 2 {
+		return "", "", "", errwrap.Wrap(err, "user string mal formated")
+	}
+
+	// get email and comment
+	ec := strings.Split(n[1], "> ")
+	if len(ec) != 2 {
+		return "", "", "", errwrap.Wrap(err, "user string mal formated")
+	}
+
+	return n[0], ec[1], ec[0], nil
+}

pgraph/actions.go

@@ -0,0 +1,535 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgraph
+
+import (
+	"fmt"
+	"log"
+	"math"
+	"sync"
+	"time"
+
+	"github.com/purpleidea/mgmt/event"
+	"github.com/purpleidea/mgmt/resources"
+
+	errwrap "github.com/pkg/errors"
+)
+
+// GetTimestamp returns the timestamp of a vertex
+func (v *Vertex) GetTimestamp() int64 {
+	return v.timestamp
+}
+
+// UpdateTimestamp updates the timestamp on a vertex and returns the new value
+func (v *Vertex) UpdateTimestamp() int64 {
+	v.timestamp = time.Now().UnixNano() // update
+	return v.timestamp
+}
+
+// OKTimestamp returns true if this element can run right now?
+func (g *Graph) OKTimestamp(v *Vertex) bool {
+	// these are all the vertices pointing TO v, eg: ??? -> v
+	for _, n := range g.IncomingGraphVertices(v) {
+		// if the vertex has a greater timestamp than any pre-req (n)
+		// then we can't run right now...
+		// if they're equal (eg: on init of 0) then we also can't run
+		// b/c we should let our pre-req's go first...
+		x, y := v.GetTimestamp(), n.GetTimestamp()
+		if g.Flags.Debug {
+			log.Printf("%s[%s]: OKTimestamp: (%v) >= %s[%s](%v): !%v", v.Kind(), v.GetName(), x, n.Kind(), n.GetName(), y, x >= y)
+		}
+		if x >= y {
+			return false
+		}
+	}
+	return true
+}
+
+// Poke notifies nodes after me in the dependency graph that they need refreshing...
+// NOTE: this assumes that this can never fail or need to be rescheduled
+func (g *Graph) Poke(v *Vertex, activity bool) error {
+	var wg sync.WaitGroup
+	// these are all the vertices pointing AWAY FROM v, eg: v -> ???
+	for _, n := range g.OutgoingGraphVertices(v) {
+		// XXX: if we're in state event and haven't been cancelled by
+		// apply, then we can cancel a poke to a child, right? XXX
+		// XXX: if n.Res.getState() != resources.ResStateEvent || activity { // is this correct?
+		if true || activity { // XXX: ???
+			if g.Flags.Debug {
+				log.Printf("%s[%s]: Poke: %s[%s]", v.Kind(), v.GetName(), n.Kind(), n.GetName())
+			}
+			wg.Add(1)
+			go func(nn *Vertex) error {
+				defer wg.Done()
+				edge := g.Adjacency[v][nn] // lookup
+				notify := edge.Notify && edge.Refresh()
+
+				// FIXME: is it okay that this is sync?
+				nn.SendEvent(event.EventPoke, true, notify)
+				// TODO: check return value?
+				return nil // never error for now...
+			}(n)
+
+		} else {
+			if g.Flags.Debug {
+				log.Printf("%s[%s]: Poke: %s[%s]: Skipped!", v.Kind(), v.GetName(), n.Kind(), n.GetName())
+			}
+		}
+	}
+	wg.Wait() // wait for all the pokes to complete
+	return nil
+}
+
+// BackPoke pokes the pre-requisites that are stale and need to run before I can run.
+func (g *Graph) BackPoke(v *Vertex) {
+	// these are all the vertices pointing TO v, eg: ??? -> v
+	for _, n := range g.IncomingGraphVertices(v) {
+		x, y, s := v.GetTimestamp(), n.GetTimestamp(), n.Res.GetState()
+		// if the parent timestamp needs poking AND it's not in state
+		// ResStateEvent, then poke it. If the parent is in ResStateEvent it
+		// means that an event is pending, so we'll be expecting a poke
+		// back soon, so we can safely discard the extra parent poke...
+		// TODO: implement a stateLT (less than) to tell if something
+		// happens earlier in the state cycle and that doesn't wrap nil
+		if x >= y && (s != resources.ResStateEvent && s != resources.ResStateCheckApply) {
+			if g.Flags.Debug {
+				log.Printf("%s[%s]: BackPoke: %s[%s]", v.Kind(), v.GetName(), n.Kind(), n.GetName())
+			}
+			// FIXME: is it okay that this is sync?
+			n.SendEvent(event.EventBackPoke, true, false)
+		} else {
+			if g.Flags.Debug {
+				log.Printf("%s[%s]: BackPoke: %s[%s]: Skipped!", v.Kind(), v.GetName(), n.Kind(), n.GetName())
+			}
+		}
+	}
+}
+
+// RefreshPending determines if any previous nodes have a refresh pending here.
+// If this is true, it means I am expected to apply a refresh when I next run.
+func (g *Graph) RefreshPending(v *Vertex) bool {
+	var refresh bool
+	for _, edge := range g.IncomingGraphEdges(v) {
+		// if we asked for a notify *and* if one is pending!
+		if edge.Notify && edge.Refresh() {
+			refresh = true
+			break
+		}
+	}
+	return refresh
+}
+
+// SetUpstreamRefresh sets the refresh value to any upstream vertices.
+func (g *Graph) SetUpstreamRefresh(v *Vertex, b bool) {
+	for _, edge := range g.IncomingGraphEdges(v) {
+		if edge.Notify {
+			edge.SetRefresh(b)
+		}
+	}
+}
+
+// SetDownstreamRefresh sets the refresh value to any downstream vertices.
+func (g *Graph) SetDownstreamRefresh(v *Vertex, b bool) {
+	for _, edge := range g.OutgoingGraphEdges(v) {
+		// if we asked for a notify *and* if one is pending!
+		if edge.Notify {
+			edge.SetRefresh(b)
+		}
+	}
+}
+
+// Process is the primary function to execute for a particular vertex in the graph.
+func (g *Graph) Process(v *Vertex) error {
+	obj := v.Res
+	if g.Flags.Debug {
+		log.Printf("%s[%s]: Process()", obj.Kind(), obj.GetName())
+	}
+	obj.SetState(resources.ResStateEvent)
+	var ok = true
+	var applied = false // did we run an apply?
+	// is it okay to run dependency wise right now?
+	// if not, that's okay because when the dependency runs, it will poke
+	// us back and we will run if needed then!
+	if g.OKTimestamp(v) {
+		if g.Flags.Debug {
+			log.Printf("%s[%s]: OKTimestamp(%v)", obj.Kind(), obj.GetName(), v.GetTimestamp())
+		}
+
+		obj.SetState(resources.ResStateCheckApply)
+
+		// connect any senders to receivers and detect if values changed
+		if updated, err := obj.SendRecv(obj); err != nil {
+			return errwrap.Wrapf(err, "could not SendRecv in Process")
+		} else if len(updated) > 0 {
+			for _, changed := range updated {
+				if changed { // at least one was updated
+					obj.StateOK(false) // invalidate cache, mark as dirty
+					break
+				}
+			}
+		}
+
+		var noop = obj.Meta().Noop // lookup the noop value
+		var refresh bool
+		var checkOK bool
+		var err error
+
+		if g.Flags.Debug {
+			log.Printf("%s[%s]: CheckApply(%t)", obj.Kind(), obj.GetName(), !noop)
+		}
+
+		// lookup the refresh (notification) variable
+		refresh = g.RefreshPending(v) // do i need to perform a refresh?
+		obj.SetRefresh(refresh)       // tell the resource
+
+		// check cached state, to skip CheckApply; can't skip if refreshing
+		if !refresh && obj.IsStateOK() {
+			checkOK, err = true, nil
+
+			// NOTE: technically this block is wrong because we don't know
+			// if the resource implements refresh! If it doesn't, we could
+			// skip this, but it doesn't make a big difference under noop!
+		} else if noop && refresh { // had a refresh to do w/ noop!
+			checkOK, err = false, nil // therefore the state is wrong
+
+			// run the CheckApply!
+		} else {
+			// if this fails, don't UpdateTimestamp()
+			checkOK, err = obj.CheckApply(!noop)
+		}
+
+		if checkOK && err != nil { // should never return this way
+			log.Fatalf("%s[%s]: CheckApply(): %t, %+v", obj.Kind(), obj.GetName(), checkOK, err)
+		}
+		if g.Flags.Debug {
+			log.Printf("%s[%s]: CheckApply(): %t, %v", obj.Kind(), obj.GetName(), checkOK, err)
+		}
+
+		// if CheckApply ran without noop and without error, state should be good
+		if !noop && err == nil { // aka !noop || checkOK
+			obj.StateOK(true) // reset
+			if refresh {
+				g.SetUpstreamRefresh(v, false) // refresh happened, clear the request
+				obj.SetRefresh(false)
+			}
+		}
+
+		if !checkOK { // if state *was* not ok, we had to have apply'ed
+			if err != nil { // error during check or apply
+				ok = false
+			} else {
+				applied = true
+			}
+		}
+
+		// when noop is true we always want to update timestamp
+		if noop && err == nil {
+			ok = true
+		}
+
+		if ok {
+			// did we actually do work?
+			activity := applied
+			if noop {
+				activity = false // no we didn't do work...
+			}
+
+			if activity { // add refresh flag to downstream edges...
+				g.SetDownstreamRefresh(v, true)
+			}
+
+			// update this timestamp *before* we poke or the poked
+			// nodes might fail due to having a too old timestamp!
+			v.UpdateTimestamp()                    // this was touched...
+			obj.SetState(resources.ResStatePoking) // can't cancel parent poke
+			if err := g.Poke(v, activity); err != nil {
+				return errwrap.Wrapf(err, "the Poke() failed")
+			}
+		}
+		// poke at our pre-req's instead since they need to refresh/run...
+		return errwrap.Wrapf(err, "could not Process() successfully")
+	}
+	// else... only poke at the pre-req's that need to run
+	go g.BackPoke(v)
+	return nil
+}
+
+// SentinelErr is a sentinal as an error type that wraps an arbitrary error.
+type SentinelErr struct {
+	err error
+}
+
+// Error is the required method to fulfill the error type.
+func (obj *SentinelErr) Error() string {
+	return obj.err.Error()
+}
+
+// Worker is the common run frontend of the vertex. It handles all of the retry
+// and retry delay common code, and ultimately returns the final status of this
+// vertex execution.
+func (g *Graph) Worker(v *Vertex) error {
+	// listen for chan events from Watch() and run
+	// the Process() function when they're received
+	// this avoids us having to pass the data into
+	// the Watch() function about which graph it is
+	// running on, which isolates things nicely...
+	obj := v.Res
+	// TODO: is there a better system for the `Watching` flag?
+	obj.SetWatching(true)
+	defer obj.SetWatching(false)
+	processChan := make(chan event.Event)
+	go func() {
+		running := false
+		var timer = time.NewTimer(time.Duration(math.MaxInt64)) // longest duration
+		if !timer.Stop() {
+			<-timer.C // unnecessary, shouldn't happen
+		}
+		var delay = time.Duration(v.Meta().Delay) * time.Millisecond
+		var retry = v.Meta().Retry // number of tries left, -1 for infinite
+		var saved event.Event
+	Loop:
+		for {
+			// this has to be synchronous, because otherwise the Res
+			// event loop will keep running and change state,
+			// causing the converged timeout to fire!
+			select {
+			case event, ok := <-processChan: // must use like this
+				if running && ok {
+					// we got an event that wasn't a close,
+					// while we were waiting for the timer!
+					// if this happens, it might be a bug:(
+					log.Fatalf("%s[%s]: Worker: Unexpected event: %+v", v.Kind(), v.GetName(), event)
+				}
+				if !ok { // processChan closed, let's exit
+					break Loop // no event, so no ack!
+				}
+
+				// the above mentioned synchronous part, is the
+				// running of this function, paired with an ack.
+				if e := g.Process(v); e != nil {
+					saved = event
+					log.Printf("%s[%s]: CheckApply errored: %v", v.Kind(), v.GetName(), e)
+					if retry == 0 {
+						// wrap the error in the sentinel
+						event.ACKNACK(&SentinelErr{e}) // fail the Watch()
+						break Loop
+					}
+					if retry > 0 { // don't decrement the -1
+						retry--
+					}
+					log.Printf("%s[%s]: CheckApply: Retrying after %.4f seconds (%d left)", v.Kind(), v.GetName(), delay.Seconds(), retry)
+					// start the timer...
+					timer.Reset(delay)
+					running = true
+					continue
+				}
+				retry = v.Meta().Retry // reset on success
+				event.ACK()            // sync
+
+			case <-timer.C:
+				if !timer.Stop() {
+					//<-timer.C // blocks, docs are wrong!
+				}
+				running = false
+				log.Printf("%s[%s]: CheckApply delay expired!", v.Kind(), v.GetName())
+				// re-send this failed event, to trigger a CheckApply()
+				go func() { processChan <- saved }()
+				// TODO: should we send a fake event instead?
+				//saved = nil
+			}
+		}
+	}()
+	var err error // propagate the error up (this is a permanent BAD error!)
+	// the watch delay runs inside of the Watch resource loop, so that it
+	// can still process signals and exit if needed. It shouldn't run any
+	// resource specific code since this is supposed to be a retry delay.
+	// NOTE: we're using the same retry and delay metaparams that CheckApply
+	// uses. This is for practicality. We can separate them later if needed!
+	var watchDelay time.Duration
+	var watchRetry = v.Meta().Retry // number of tries left, -1 for infinite
+	// watch blocks until it ends, & errors to retry
+	for {
+		// TODO: do we have to stop the converged-timeout when in this block (perhaps we're in the delay block!)
+		// TODO: should we setup/manage some of the converged timeout stuff in here anyways?
+
+		// if a retry-delay was requested, wait, but don't block our events!
+		if watchDelay > 0 {
+			//var pendingSendEvent bool
+			timer := time.NewTimer(watchDelay)
+		Loop:
+			for {
+				select {
+				case <-timer.C: // the wait is over
+					break Loop // critical
+
+				// TODO: resources could have a separate exit channel to avoid this complexity!?
+				case event := <-obj.Events():
+					// NOTE: this code should match the similar Res code!
+					//cuid.SetConverged(false) // TODO: ?
+					if exit, send := obj.ReadEvent(&event); exit {
+						return nil // exit
+					} else if send {
+						// if we dive down this rabbit hole, our
+						// timer.C won't get seen until we get out!
+						// in this situation, the Watch() is blocked
+						// from performing until CheckApply returns
+						// successfully, or errors out. This isn't
+						// so bad, but we should document it. Is it
+						// possible that some resource *needs* Watch
+						// to run to be able to execute a CheckApply?
+						// That situation shouldn't be common, and
+						// should probably not be allowed. Can we
+						// avoid it though?
+						//if exit, err := doSend(); exit || err != nil {
+						//	return err // we exit or bubble up a NACK...
+						//}
+						// Instead of doing the above, we can
+						// add events to a pending list, and
+						// when we finish the delay, we can run
+						// them.
+						//pendingSendEvent = true // all events are identical for now...
+					}
+				}
+			}
+			timer.Stop() // it's nice to cleanup
+			log.Printf("%s[%s]: Watch delay expired!", v.Kind(), v.GetName())
+			// NOTE: we can avoid the send if running Watch guarantees
+			// one CheckApply event on startup!
+			//if pendingSendEvent { // TODO: should this become a list in the future?
+			//	if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
+			//		return err // we exit or bubble up a NACK...
+			//	}
+			//}
+		}
+
+		// TODO: reset the watch retry count after some amount of success
+		v.Res.RegisterConverger()
+		e := v.Res.Watch(processChan)
+		v.Res.UnregisterConverger()
+		if e == nil { // exit signal
+			err = nil // clean exit
+			break
+		}
+		if sentinelErr, ok := e.(*SentinelErr); ok { // unwrap the sentinel
+			err = sentinelErr.err
+			break // sentinel means, perma-exit
+		}
+		log.Printf("%s[%s]: Watch errored: %v", v.Kind(), v.GetName(), e)
+		if watchRetry == 0 {
+			err = fmt.Errorf("Permanent watch error: %v", e)
+			break
+		}
+		if watchRetry > 0 { // don't decrement the -1
+			watchRetry--
+		}
+		watchDelay = time.Duration(v.Meta().Delay) * time.Millisecond
+		log.Printf("%s[%s]: Watch: Retrying after %.4f seconds (%d left)", v.Kind(), v.GetName(), watchDelay.Seconds(), watchRetry)
+		// We need to trigger a CheckApply after Watch restarts, so that
+		// we catch any lost events that happened while down. We do this
+		// by getting the Watch resource to send one event once it's up!
+		//v.SendEvent(eventPoke, false, false)
+	}
+	close(processChan)
+	return err
+}
+
+// Start is a main kick to start the graph. It goes through in reverse topological
+// sort order so that events can't hit un-started vertices.
+func (g *Graph) Start(first bool) { // start or continue
+	log.Printf("State: %v -> %v", g.setState(graphStateStarting), g.getState())
+	defer log.Printf("State: %v -> %v", g.setState(graphStateStarted), g.getState())
+	var wg sync.WaitGroup
+	t, _ := g.TopologicalSort()
+	// TODO: only calculate indegree if `first` is true to save resources
+	indegree := g.InDegree() // compute all of the indegree's
+	for _, v := range Reverse(t) {
+		// selective poke: here we reduce the number of initial pokes
+		// to the minimum required to activate every vertex in the
+		// graph, either by direct action, or by getting poked by a
+		// vertex that was previously activated. if we poke each vertex
+		// that has no incoming edges, then we can be sure to reach the
+		// whole graph. Please note: this may mask certain optimization
+		// failures, such as any poke limiting code in Poke() or
+		// BackPoke(). You might want to disable this selective start
+		// when experimenting with and testing those elements.
+		// if we are unpausing (since it's not the first run of this
+		// function) we need to poke to *unpause* every graph vertex,
+		// and not just selectively the subset with no indegree.
+		if (!first) || indegree[v] == 0 {
+			v.Res.Starter(true) // let the startup code know to poke
+		}
+
+		if !v.Res.IsWatching() { // if Watch() is not running...
+			g.wg.Add(1)
+			// must pass in value to avoid races...
+			// see: https://ttboj.wordpress.com/2015/07/27/golang-parallelism-issues-causing-too-many-open-files-error/
+			go func(vv *Vertex) {
+				defer g.wg.Done()
+				// TODO: if a sufficient number of workers error,
+				// should something be done? Will these restart
+				// after perma-failure if we have a graph change?
+				if err := g.Worker(vv); err != nil { // contains the Watch and CheckApply loops
+					log.Printf("%s[%s]: Exited with failure: %v", vv.Kind(), vv.GetName(), err)
+					return
+				}
+				log.Printf("%s[%s]: Exited", vv.Kind(), vv.GetName())
+			}(v)
+		}
+
+		// let the vertices run their startup code in parallel
+		wg.Add(1)
+		go func(vv *Vertex) {
+			defer wg.Done()
+			vv.Res.Started() // block until started
+		}(v)
+
+		if !first { // unpause!
+			v.Res.SendEvent(event.EventStart, true, false) // sync!
+		}
+	}
+
+	wg.Wait() // wait for everyone
+}
+
+// Pause sends pause events to the graph in a topological sort order.
+func (g *Graph) Pause() {
+	log.Printf("State: %v -> %v", g.setState(graphStatePausing), g.getState())
+	defer log.Printf("State: %v -> %v", g.setState(graphStatePaused), g.getState())
+	t, _ := g.TopologicalSort()
+	for _, v := range t { // squeeze out the events...
+		v.SendEvent(event.EventPause, true, false)
+	}
+}
+
+// Exit sends exit events to the graph in a topological sort order.
+func (g *Graph) Exit() {
+	if g == nil {
+		return
+	} // empty graph that wasn't populated yet
+	t, _ := g.TopologicalSort()
+	for _, v := range t { // squeeze out the events...
+		// turn off the taps...
+		// XXX: consider instead doing this by closing the Res.events channel instead?
+		// XXX: do this by sending an exit signal, and then returning
+		// when we hit the 'default' in the select statement!
+		// XXX: we can do this to quiesce, but it's not necessary now
+
+		v.SendEvent(event.EventExit, true, false)
+	}
+	g.wg.Wait() // for now, this doesn't need to be a separate Wait() method
+}

pgraph/autoedge.go

@@ -22,7 +22,6 @@ import (
 	"fmt"
 	"log"
 
-	"github.com/purpleidea/mgmt/global"
 	"github.com/purpleidea/mgmt/resources"
 )
 
@@ -39,7 +38,7 @@ func (g *Graph) addEdgesByMatchingUIDS(v *Vertex, uids []resources.ResUID) []boo
 			if v == vv { // skip self
 				continue
 			}
-			if global.DEBUG {
+			if g.Flags.Debug {
 				log.Printf("Compile: AutoEdge: Match: %v[%v] with UID: %v[%v]", vv.Kind(), vv.GetName(), uid.Kind(), uid.GetName())
 			}
 			// we must match to an effective UID for the resource,
@@ -85,7 +84,7 @@ func (g *Graph) AutoEdges() {
 				log.Printf("%v[%v]: Config: The auto edge list is empty!", v.Kind(), v.GetName())
 				break // inner loop
 			}
-			if global.DEBUG {
+			if g.Flags.Debug {
 				log.Println("Compile: AutoEdge: UIDS:")
 				for i, u := range uids {
 					log.Printf("Compile: AutoEdge: UID%d: %v", i, u)

pgraph/autogroup.go

@@ -21,8 +21,6 @@ import (
 	"fmt"
 	"log"
 
-	"github.com/purpleidea/mgmt/global"
-
 	errwrap "github.com/pkg/errors"
 )
 
@@ -221,7 +219,7 @@ func (g *Graph) VertexMerge(v1, v2 *Vertex, vertexMergeFn func(*Vertex, *Vertex)
 	}
 
 	// 2) edges that point towards v2 from X now point to v1 from X (no dupes)
-	for _, x := range g.IncomingGraphEdges(v2) { // all to vertex v (??? -> v)
+	for _, x := range g.IncomingGraphVertices(v2) { // all to vertex v (??? -> v)
 		e := g.Adjacency[x][v2] // previous edge
 		r := g.Reachability(x, v1)
 		// merge e with ex := g.Adjacency[x][v1] if it exists!
@@ -248,7 +246,7 @@ func (g *Graph) VertexMerge(v1, v2 *Vertex, vertexMergeFn func(*Vertex, *Vertex)
 	}
 
 	// 3) edges that point from v2 to X now point from v1 to X (no dupes)
-	for _, x := range g.OutgoingGraphEdges(v2) { // all from vertex v (v -> ???)
+	for _, x := range g.OutgoingGraphVertices(v2) { // all from vertex v (v -> ???)
 		e := g.Adjacency[v2][x] // previous edge
 		r := g.Reachability(v1, x)
 		// merge e with ex := g.Adjacency[v1][x] if it exists!
@@ -279,7 +277,7 @@ func (g *Graph) VertexMerge(v1, v2 *Vertex, vertexMergeFn func(*Vertex, *Vertex)
 		if v, err := vertexMergeFn(v1, v2); err != nil {
 			return err
 		} else if v != nil { // replace v1 with the "merged" version...
-			v1 = v // XXX: will this replace v1 the way we want?
+			*v1 = *v // TODO: is this safe? (replacing mutexes is undefined!)
 		}
 	}
 	g.DeleteVertex(v2) // remove grouped vertex
@@ -312,7 +310,7 @@ func (g *Graph) autoGroup(ag AutoGrouper) chan string {
 			wStr := fmt.Sprintf("%s", w)
 
 			if err := ag.vertexCmp(v, w); err != nil { // cmp ?
-				if global.DEBUG {
+				if g.Flags.Debug {
 					strch <- fmt.Sprintf("Compile: Grouping: !GroupCmp for: %s into %s", wStr, vStr)
 				}
 

pgraph/graphviz.go

@@ -0,0 +1,110 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgraph
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"strconv"
+	"syscall"
+)
+
+// Graphviz outputs the graph in graphviz format.
+// https://en.wikipedia.org/wiki/DOT_%28graph_description_language%29
+func (g *Graph) Graphviz() (out string) {
+	//digraph g {
+	//	label="hello world";
+	//	node [shape=box];
+	//	A [label="A"];
+	//	B [label="B"];
+	//	C [label="C"];
+	//	D [label="D"];
+	//	E [label="E"];
+	//	A -> B [label=f];
+	//	B -> C [label=g];
+	//	D -> E [label=h];
+	//}
+	out += fmt.Sprintf("digraph %s {\n", g.GetName())
+	out += fmt.Sprintf("\tlabel=\"%s\";\n", g.GetName())
+	//out += "\tnode [shape=box];\n"
+	str := ""
+	for i := range g.Adjacency { // reverse paths
+		out += fmt.Sprintf("\t%s [label=\"%s[%s]\"];\n", i.GetName(), i.Kind(), i.GetName())
+		for j := range g.Adjacency[i] {
+			k := g.Adjacency[i][j]
+			// use str for clearer output ordering
+			str += fmt.Sprintf("\t%s -> %s [label=%s];\n", i.GetName(), j.GetName(), k.Name)
+		}
+	}
+	out += str
+	out += "}\n"
+	return
+}
+
+// ExecGraphviz writes out the graphviz data and runs the correct graphviz
+// filter command.
+func (g *Graph) ExecGraphviz(program, filename string) error {
+
+	switch program {
+	case "dot", "neato", "twopi", "circo", "fdp":
+	default:
+		return fmt.Errorf("Invalid graphviz program selected!")
+	}
+
+	if filename == "" {
+		return fmt.Errorf("No filename given!")
+	}
+
+	// run as a normal user if possible when run with sudo
+	uid, err1 := strconv.Atoi(os.Getenv("SUDO_UID"))
+	gid, err2 := strconv.Atoi(os.Getenv("SUDO_GID"))
+
+	err := ioutil.WriteFile(filename, []byte(g.Graphviz()), 0644)
+	if err != nil {
+		return fmt.Errorf("Error writing to filename!")
+	}
+
+	if err1 == nil && err2 == nil {
+		if err := os.Chown(filename, uid, gid); err != nil {
+			return fmt.Errorf("Error changing file owner!")
+		}
+	}
+
+	path, err := exec.LookPath(program)
+	if err != nil {
+		return fmt.Errorf("Graphviz is missing!")
+	}
+
+	out := fmt.Sprintf("%s.png", filename)
+	cmd := exec.Command(path, "-Tpng", fmt.Sprintf("-o%s", out), filename)
+
+	if err1 == nil && err2 == nil {
+		cmd.SysProcAttr = &syscall.SysProcAttr{}
+		cmd.SysProcAttr.Credential = &syscall.Credential{
+			Uid: uint32(uid),
+			Gid: uint32(gid),
+		}
+	}
+	_, err = cmd.Output()
+	if err != nil {
+		return fmt.Errorf("Error writing to image!")
+	}
+	return nil
+}

pgraph/pgraph.go

@@ -20,20 +20,10 @@ package pgraph
 
 import (
 	"fmt"
-	"io/ioutil"
-	"log"
-	"math"
-	"os"
-	"os/exec"
 	"sort"
-	"strconv"
 	"sync"
-	"syscall"
-	"time"
 
-	"github.com/purpleidea/mgmt/converger"
 	"github.com/purpleidea/mgmt/event"
-	"github.com/purpleidea/mgmt/global"
 	"github.com/purpleidea/mgmt/resources"
 
 	errwrap "github.com/pkg/errors"
@@ -50,6 +40,11 @@ const (
 	graphStatePaused
 )
 
+// Flags contains specific constants used by the graph.
+type Flags struct {
+	Debug bool
+}
+
 // Graph is the graph structure in this library.
 // The graph abstract data type (ADT) is defined as follows:
 // * the directed graph arrows point from left to right ( -> )
@@ -59,8 +54,10 @@ const (
 type Graph struct {
 	Name      string
 	Adjacency map[*Vertex]map[*Vertex]*Edge // *Vertex -> *Vertex (edge)
+	Flags     Flags
 	state     graphState
-	mutex     sync.Mutex // used when modifying graph State variable
+	mutex     *sync.Mutex // used when modifying graph State variable
+	wg        *sync.WaitGroup
 }
 
 // Vertex is the primary vertex struct in this library.
@@ -71,7 +68,10 @@ type Vertex struct {
 
 // Edge is the primary edge struct in this library.
 type Edge struct {
-	Name string
+	Name   string
+	Notify bool // should we send a refresh notification along this edge?
+
+	refresh bool // is there a notify pending for the dest vertex ?
 }
 
 // NewGraph builds a new graph.
@@ -80,6 +80,9 @@ func NewGraph(name string) *Graph {
 		Name:      name,
 		Adjacency: make(map[*Vertex]map[*Vertex]*Edge),
 		state:     graphStateNil,
+		// ptr b/c: Mutex/WaitGroup must not be copied after first use
+		mutex: &sync.Mutex{},
+		wg:    &sync.WaitGroup{},
 	}
 }
 
@@ -97,12 +100,25 @@ func NewEdge(name string) *Edge {
 	}
 }
 
+// Refresh returns the pending refresh status of this edge.
+func (obj *Edge) Refresh() bool {
+	return obj.refresh
+}
+
+// SetRefresh sets the pending refresh status of this edge.
+func (obj *Edge) SetRefresh(b bool) {
+	obj.refresh = b
+}
+
 // Copy makes a copy of the graph struct
 func (g *Graph) Copy() *Graph {
 	newGraph := &Graph{
 		Name:      g.Name,
 		Adjacency: make(map[*Vertex]map[*Vertex]*Edge, len(g.Adjacency)),
+		Flags:     g.Flags,
 		state:     g.state,
+		mutex:     g.mutex,
+		wg:        g.wg,
 	}
 	for k, v := range g.Adjacency {
 		newGraph.Adjacency[k] = v // copy
@@ -259,92 +275,9 @@ func (v *Vertex) String() string {
 	return fmt.Sprintf("%s[%s]", v.Res.Kind(), v.Res.GetName())
 }
 
-// Graphviz outputs the graph in graphviz format.
+// IncomingGraphVertices returns an array (slice) of all directed vertices to
-// https://en.wikipedia.org/wiki/DOT_%28graph_description_language%29
-func (g *Graph) Graphviz() (out string) {
-	//digraph g {
-	//	label="hello world";
-	//	node [shape=box];
-	//	A [label="A"];
-	//	B [label="B"];
-	//	C [label="C"];
-	//	D [label="D"];
-	//	E [label="E"];
-	//	A -> B [label=f];
-	//	B -> C [label=g];
-	//	D -> E [label=h];
-	//}
-	out += fmt.Sprintf("digraph %v {\n", g.GetName())
-	out += fmt.Sprintf("\tlabel=\"%v\";\n", g.GetName())
-	//out += "\tnode [shape=box];\n"
-	str := ""
-	for i := range g.Adjacency { // reverse paths
-		out += fmt.Sprintf("\t%v [label=\"%v[%v]\"];\n", i.GetName(), i.Kind(), i.GetName())
-		for j := range g.Adjacency[i] {
-			k := g.Adjacency[i][j]
-			// use str for clearer output ordering
-			str += fmt.Sprintf("\t%v -> %v [label=%v];\n", i.GetName(), j.GetName(), k.Name)
-		}
-	}
-	out += str
-	out += "}\n"
-	return
-}
-
-// ExecGraphviz writes out the graphviz data and runs the correct graphviz
-// filter command.
-func (g *Graph) ExecGraphviz(program, filename string) error {
-
-	switch program {
-	case "dot", "neato", "twopi", "circo", "fdp":
-	default:
-		return fmt.Errorf("Invalid graphviz program selected!")
-	}
-
-	if filename == "" {
-		return fmt.Errorf("No filename given!")
-	}
-
-	// run as a normal user if possible when run with sudo
-	uid, err1 := strconv.Atoi(os.Getenv("SUDO_UID"))
-	gid, err2 := strconv.Atoi(os.Getenv("SUDO_GID"))
-
-	err := ioutil.WriteFile(filename, []byte(g.Graphviz()), 0644)
-	if err != nil {
-		return fmt.Errorf("Error writing to filename!")
-	}
-
-	if err1 == nil && err2 == nil {
-		if err := os.Chown(filename, uid, gid); err != nil {
-			return fmt.Errorf("Error changing file owner!")
-		}
-	}
-
-	path, err := exec.LookPath(program)
-	if err != nil {
-		return fmt.Errorf("Graphviz is missing!")
-	}
-
-	out := fmt.Sprintf("%v.png", filename)
-	cmd := exec.Command(path, "-Tpng", fmt.Sprintf("-o%v", out), filename)
-
-	if err1 == nil && err2 == nil {
-		cmd.SysProcAttr = &syscall.SysProcAttr{}
-		cmd.SysProcAttr.Credential = &syscall.Credential{
-			Uid: uint32(uid),
-			Gid: uint32(gid),
-		}
-	}
-	_, err = cmd.Output()
-	if err != nil {
-		return fmt.Errorf("Error writing to image!")
-	}
-	return nil
-}
-
-// IncomingGraphEdges returns an array (slice) of all directed vertices to
 // vertex v (??? -> v). OKTimestamp should probably use this.
-func (g *Graph) IncomingGraphEdges(v *Vertex) []*Vertex {
+func (g *Graph) IncomingGraphVertices(v *Vertex) []*Vertex {
 	// TODO: we might be able to implement this differently by reversing
 	// the Adjacency graph and then looping through it again...
 	var s []*Vertex
@@ -358,9 +291,9 @@ func (g *Graph) IncomingGraphEdges(v *Vertex) []*Vertex {
 	return s
 }
 
-// OutgoingGraphEdges returns an array (slice) of all vertices that vertex v
+// OutgoingGraphVertices returns an array (slice) of all vertices that vertex v
 // points to (v -> ???). Poke should probably use this.
-func (g *Graph) OutgoingGraphEdges(v *Vertex) []*Vertex {
+func (g *Graph) OutgoingGraphVertices(v *Vertex) []*Vertex {
 	var s []*Vertex
 	for k := range g.Adjacency[v] { // forward paths
 		s = append(s, k)
@@ -368,15 +301,46 @@ func (g *Graph) OutgoingGraphEdges(v *Vertex) []*Vertex {
 	return s
 }
 
-// GraphEdges returns an array (slice) of all vertices that connect to vertex v.
+// GraphVertices returns an array (slice) of all vertices that connect to vertex v.
-// This is the union of IncomingGraphEdges and OutgoingGraphEdges.
+// This is the union of IncomingGraphVertices and OutgoingGraphVertices.
-func (g *Graph) GraphEdges(v *Vertex) []*Vertex {
+func (g *Graph) GraphVertices(v *Vertex) []*Vertex {
 	var s []*Vertex
-	s = append(s, g.IncomingGraphEdges(v)...)
+	s = append(s, g.IncomingGraphVertices(v)...)
-	s = append(s, g.OutgoingGraphEdges(v)...)
+	s = append(s, g.OutgoingGraphVertices(v)...)
 	return s
 }
 
+// IncomingGraphEdges returns all of the edges that point to vertex v (??? -> v).
+func (g *Graph) IncomingGraphEdges(v *Vertex) []*Edge {
+	var edges []*Edge
+	for v1 := range g.Adjacency { // reverse paths
+		for v2, e := range g.Adjacency[v1] {
+			if v2 == v {
+				edges = append(edges, e)
+			}
+		}
+	}
+	return edges
+}
+
+// OutgoingGraphEdges returns all of the edges that point from vertex v (v -> ???).
+func (g *Graph) OutgoingGraphEdges(v *Vertex) []*Edge {
+	var edges []*Edge
+	for _, e := range g.Adjacency[v] { // forward paths
+		edges = append(edges, e)
+	}
+	return edges
+}
+
+// GraphEdges returns an array (slice) of all edges that connect to vertex v.
+// This is the union of IncomingGraphEdges and OutgoingGraphEdges.
+func (g *Graph) GraphEdges(v *Vertex) []*Edge {
+	var edges []*Edge
+	edges = append(edges, g.IncomingGraphEdges(v)...)
+	edges = append(edges, g.OutgoingGraphEdges(v)...)
+	return edges
+}
+
 // DFS returns a depth first search for the graph, starting at the input vertex.
 func (g *Graph) DFS(start *Vertex) []*Vertex {
 	var d []*Vertex // discovered
@@ -392,7 +356,7 @@ func (g *Graph) DFS(start *Vertex) []*Vertex {
 		if !VertexContains(v, d) { // if not discovered
 			d = append(d, v) // label as discovered
 
-			for _, w := range g.GraphEdges(v) {
+			for _, w := range g.GraphVertices(v) {
 				s = append(s, w)
 			}
 		}
@@ -405,7 +369,7 @@ func (g *Graph) FilterGraph(name string, vertices []*Vertex) *Graph {
 	newgraph := NewGraph(name)
 	for k1, x := range g.Adjacency {
 		for k2, e := range x {
-			//log.Printf("Filter: %v -> %v # %v", k1.Name, k2.Name, e.Name)
+			//log.Printf("Filter: %s -> %s # %s", k1.Name, k2.Name, e.Name)
 			if VertexContains(k1, vertices) || VertexContains(k2, vertices) {
 				newgraph.AddEdge(k1, k2, e)
 			}
@@ -539,7 +503,7 @@ func (g *Graph) Reachability(a, b *Vertex) []*Vertex {
 	if a == nil || b == nil {
 		return nil
 	}
-	vertices := g.OutgoingGraphEdges(a) // what points away from a ?
+	vertices := g.OutgoingGraphVertices(a) // what points away from a ?
 	if len(vertices) == 0 {
 		return []*Vertex{} // nope
 	}
@@ -567,391 +531,6 @@ func (g *Graph) Reachability(a, b *Vertex) []*Vertex {
 	return result
 }
 
-// GetTimestamp returns the timestamp of a vertex
-func (v *Vertex) GetTimestamp() int64 {
-	return v.timestamp
-}
-
-// UpdateTimestamp updates the timestamp on a vertex and returns the new value
-func (v *Vertex) UpdateTimestamp() int64 {
-	v.timestamp = time.Now().UnixNano() // update
-	return v.timestamp
-}
-
-// OKTimestamp returns true if this element can run right now?
-func (g *Graph) OKTimestamp(v *Vertex) bool {
-	// these are all the vertices pointing TO v, eg: ??? -> v
-	for _, n := range g.IncomingGraphEdges(v) {
-		// if the vertex has a greater timestamp than any pre-req (n)
-		// then we can't run right now...
-		// if they're equal (eg: on init of 0) then we also can't run
-		// b/c we should let our pre-req's go first...
-		x, y := v.GetTimestamp(), n.GetTimestamp()
-		if global.DEBUG {
-			log.Printf("%v[%v]: OKTimestamp: (%v) >= %v[%v](%v): !%v", v.Kind(), v.GetName(), x, n.Kind(), n.GetName(), y, x >= y)
-		}
-		if x >= y {
-			return false
-		}
-	}
-	return true
-}
-
-// Poke notifies nodes after me in the dependency graph that they need refreshing...
-// NOTE: this assumes that this can never fail or need to be rescheduled
-func (g *Graph) Poke(v *Vertex, activity bool) {
-	// these are all the vertices pointing AWAY FROM v, eg: v -> ???
-	for _, n := range g.OutgoingGraphEdges(v) {
-		// XXX: if we're in state event and haven't been cancelled by
-		// apply, then we can cancel a poke to a child, right? XXX
-		// XXX: if n.Res.getState() != resources.ResStateEvent { // is this correct?
-		if true { // XXX
-			if global.DEBUG {
-				log.Printf("%v[%v]: Poke: %v[%v]", v.Kind(), v.GetName(), n.Kind(), n.GetName())
-			}
-			n.SendEvent(event.EventPoke, false, activity) // XXX: can this be switched to sync?
-		} else {
-			if global.DEBUG {
-				log.Printf("%v[%v]: Poke: %v[%v]: Skipped!", v.Kind(), v.GetName(), n.Kind(), n.GetName())
-			}
-		}
-	}
-}
-
-// BackPoke pokes the pre-requisites that are stale and need to run before I can run.
-func (g *Graph) BackPoke(v *Vertex) {
-	// these are all the vertices pointing TO v, eg: ??? -> v
-	for _, n := range g.IncomingGraphEdges(v) {
-		x, y, s := v.GetTimestamp(), n.GetTimestamp(), n.Res.GetState()
-		// if the parent timestamp needs poking AND it's not in state
-		// ResStateEvent, then poke it. If the parent is in ResStateEvent it
-		// means that an event is pending, so we'll be expecting a poke
-		// back soon, so we can safely discard the extra parent poke...
-		// TODO: implement a stateLT (less than) to tell if something
-		// happens earlier in the state cycle and that doesn't wrap nil
-		if x >= y && (s != resources.ResStateEvent && s != resources.ResStateCheckApply) {
-			if global.DEBUG {
-				log.Printf("%v[%v]: BackPoke: %v[%v]", v.Kind(), v.GetName(), n.Kind(), n.GetName())
-			}
-			n.SendEvent(event.EventBackPoke, false, false) // XXX: can this be switched to sync?
-		} else {
-			if global.DEBUG {
-				log.Printf("%v[%v]: BackPoke: %v[%v]: Skipped!", v.Kind(), v.GetName(), n.Kind(), n.GetName())
-			}
-		}
-	}
-}
-
-// Process is the primary function to execute for a particular vertex in the graph.
-func (g *Graph) Process(v *Vertex) error {
-	obj := v.Res
-	if global.DEBUG {
-		log.Printf("%v[%v]: Process()", obj.Kind(), obj.GetName())
-	}
-	obj.SetState(resources.ResStateEvent)
-	var ok = true
-	var apply = false // did we run an apply?
-	// is it okay to run dependency wise right now?
-	// if not, that's okay because when the dependency runs, it will poke
-	// us back and we will run if needed then!
-	if g.OKTimestamp(v) {
-		if global.DEBUG {
-			log.Printf("%v[%v]: OKTimestamp(%v)", obj.Kind(), obj.GetName(), v.GetTimestamp())
-		}
-
-		obj.SetState(resources.ResStateCheckApply)
-		// if this fails, don't UpdateTimestamp()
-		checkok, err := obj.CheckApply(!obj.Meta().Noop)
-		if checkok && err != nil { // should never return this way
-			log.Fatalf("%v[%v]: CheckApply(): %t, %+v", obj.Kind(), obj.GetName(), checkok, err)
-		}
-		if global.DEBUG {
-			log.Printf("%v[%v]: CheckApply(): %t, %v", obj.Kind(), obj.GetName(), checkok, err)
-		}
-
-		if !checkok { // if state *was* not ok, we had to have apply'ed
-			if err != nil { // error during check or apply
-				ok = false
-			} else {
-				apply = true
-			}
-		}
-
-		// when noop is true we always want to update timestamp
-		if obj.Meta().Noop && err == nil {
-			ok = true
-		}
-
-		if ok {
-			// update this timestamp *before* we poke or the poked
-			// nodes might fail due to having a too old timestamp!
-			v.UpdateTimestamp()                    // this was touched...
-			obj.SetState(resources.ResStatePoking) // can't cancel parent poke
-			g.Poke(v, apply)
-		}
-		// poke at our pre-req's instead since they need to refresh/run...
-		return err
-	}
-	// else... only poke at the pre-req's that need to run
-	go g.BackPoke(v)
-	return nil
-}
-
-// SentinelErr is a sentinal as an error type that wraps an arbitrary error.
-type SentinelErr struct {
-	err error
-}
-
-// Error is the required method to fulfill the error type.
-func (obj *SentinelErr) Error() string {
-	return obj.err.Error()
-}
-
-// Worker is the common run frontend of the vertex. It handles all of the retry
-// and retry delay common code, and ultimately returns the final status of this
-// vertex execution.
-func (g *Graph) Worker(v *Vertex) error {
-	// listen for chan events from Watch() and run
-	// the Process() function when they're received
-	// this avoids us having to pass the data into
-	// the Watch() function about which graph it is
-	// running on, which isolates things nicely...
-	obj := v.Res
-	chanProcess := make(chan event.Event)
-	go func() {
-		running := false
-		var timer = time.NewTimer(time.Duration(math.MaxInt64)) // longest duration
-		if !timer.Stop() {
-			<-timer.C // unnecessary, shouldn't happen
-		}
-		var delay = time.Duration(v.Meta().Delay) * time.Millisecond
-		var retry = v.Meta().Retry // number of tries left, -1 for infinite
-		var saved event.Event
-	Loop:
-		for {
-			// this has to be synchronous, because otherwise the Res
-			// event loop will keep running and change state,
-			// causing the converged timeout to fire!
-			select {
-			case event, ok := <-chanProcess: // must use like this
-				if running && ok {
-					// we got an event that wasn't a close,
-					// while we were waiting for the timer!
-					// if this happens, it might be a bug:(
-					log.Fatalf("%v[%v]: Worker: Unexpected event: %+v", v.Kind(), v.GetName(), event)
-				}
-				if !ok { // chanProcess closed, let's exit
-					break Loop // no event, so no ack!
-				}
-
-				// the above mentioned synchronous part, is the
-				// running of this function, paired with an ack.
-				if e := g.Process(v); e != nil {
-					saved = event
-					log.Printf("%v[%v]: CheckApply errored: %v", v.Kind(), v.GetName(), e)
-					if retry == 0 {
-						// wrap the error in the sentinel
-						event.ACKNACK(&SentinelErr{e}) // fail the Watch()
-						break Loop
-					}
-					if retry > 0 { // don't decrement the -1
-						retry--
-					}
-					log.Printf("%v[%v]: CheckApply: Retrying after %.4f seconds (%d left)", v.Kind(), v.GetName(), delay.Seconds(), retry)
-					// start the timer...
-					timer.Reset(delay)
-					running = true
-					continue
-				}
-				retry = v.Meta().Retry // reset on success
-				event.ACK()            // sync
-
-			case <-timer.C:
-				if !timer.Stop() {
-					//<-timer.C // blocks, docs are wrong!
-				}
-				running = false
-				log.Printf("%s[%s]: CheckApply delay expired!", v.Kind(), v.GetName())
-				// re-send this failed event, to trigger a CheckApply()
-				go func() { chanProcess <- saved }()
-				// TODO: should we send a fake event instead?
-				//saved = nil
-			}
-		}
-	}()
-	var err error // propagate the error up (this is a permanent BAD error!)
-	// the watch delay runs inside of the Watch resource loop, so that it
-	// can still process signals and exit if needed. It shouldn't run any
-	// resource specific code since this is supposed to be a retry delay.
-	// NOTE: we're using the same retry and delay metaparams that CheckApply
-	// uses. This is for practicality. We can separate them later if needed!
-	var watchDelay time.Duration
-	var watchRetry = v.Meta().Retry // number of tries left, -1 for infinite
-	// watch blocks until it ends, & errors to retry
-	for {
-		// TODO: do we have to stop the converged-timeout when in this block (perhaps we're in the delay block!)
-		// TODO: should we setup/manage some of the converged timeout stuff in here anyways?
-
-		// if a retry-delay was requested, wait, but don't block our events!
-		if watchDelay > 0 {
-			//var pendingSendEvent bool
-			timer := time.NewTimer(watchDelay)
-		Loop:
-			for {
-				select {
-				case <-timer.C: // the wait is over
-					break Loop // critical
-
-				// TODO: resources could have a separate exit channel to avoid this complexity!?
-				case event := <-obj.Events():
-					// NOTE: this code should match the similar Res code!
-					//cuid.SetConverged(false) // TODO: ?
-					if exit, send := obj.ReadEvent(&event); exit {
-						return nil // exit
-					} else if send {
-						// if we dive down this rabbit hole, our
-						// timer.C won't get seen until we get out!
-						// in this situation, the Watch() is blocked
-						// from performing until CheckApply returns
-						// successfully, or errors out. This isn't
-						// so bad, but we should document it. Is it
-						// possible that some resource *needs* Watch
-						// to run to be able to execute a CheckApply?
-						// That situation shouldn't be common, and
-						// should probably not be allowed. Can we
-						// avoid it though?
-						//if exit, err := doSend(); exit || err != nil {
-						//	return err // we exit or bubble up a NACK...
-						//}
-						// Instead of doing the above, we can
-						// add events to a pending list, and
-						// when we finish the delay, we can run
-						// them.
-						//pendingSendEvent = true // all events are identical for now...
-					}
-				}
-			}
-			timer.Stop() // it's nice to cleanup
-			log.Printf("%s[%s]: Watch delay expired!", v.Kind(), v.GetName())
-			// NOTE: we can avoid the send if running Watch guarantees
-			// one CheckApply event on startup!
-			//if pendingSendEvent { // TODO: should this become a list in the future?
-			//	if exit, err := obj.DoSend(chanProcess, ""); exit || err != nil {
-			//		return err // we exit or bubble up a NACK...
-			//	}
-			//}
-		}
-
-		// TODO: reset the watch retry count after some amount of success
-		e := v.Res.Watch(chanProcess)
-		if e == nil { // exit signal
-			err = nil // clean exit
-			break
-		}
-		if sentinelErr, ok := e.(*SentinelErr); ok { // unwrap the sentinel
-			err = sentinelErr.err
-			break // sentinel means, perma-exit
-		}
-		log.Printf("%v[%v]: Watch errored: %v", v.Kind(), v.GetName(), e)
-		if watchRetry == 0 {
-			err = fmt.Errorf("Permanent watch error: %v", e)
-			break
-		}
-		if watchRetry > 0 { // don't decrement the -1
-			watchRetry--
-		}
-		watchDelay = time.Duration(v.Meta().Delay) * time.Millisecond
-		log.Printf("%v[%v]: Watch: Retrying after %.4f seconds (%d left)", v.Kind(), v.GetName(), watchDelay.Seconds(), watchRetry)
-		// We need to trigger a CheckApply after Watch restarts, so that
-		// we catch any lost events that happened while down. We do this
-		// by getting the Watch resource to send one event once it's up!
-		//v.SendEvent(eventPoke, false, false)
-	}
-	close(chanProcess)
-	return err
-}
-
-// Start is a main kick to start the graph. It goes through in reverse topological
-// sort order so that events can't hit un-started vertices.
-func (g *Graph) Start(wg *sync.WaitGroup, first bool) { // start or continue
-	log.Printf("State: %v -> %v", g.setState(graphStateStarting), g.getState())
-	defer log.Printf("State: %v -> %v", g.setState(graphStateStarted), g.getState())
-	t, _ := g.TopologicalSort()
-	// TODO: only calculate indegree if `first` is true to save resources
-	indegree := g.InDegree() // compute all of the indegree's
-	for _, v := range Reverse(t) {
-
-		if !v.Res.IsWatching() { // if Watch() is not running...
-			wg.Add(1)
-			// must pass in value to avoid races...
-			// see: https://ttboj.wordpress.com/2015/07/27/golang-parallelism-issues-causing-too-many-open-files-error/
-			go func(vv *Vertex) {
-				defer wg.Done()
-				// TODO: if a sufficient number of workers error,
-				// should something be done? Will these restart
-				// after perma-failure if we have a graph change?
-				if err := g.Worker(vv); err != nil { // contains the Watch and CheckApply loops
-					log.Printf("%s[%s]: Exited with failure: %v", vv.Kind(), vv.GetName(), err)
-					return
-				}
-				log.Printf("%v[%v]: Exited", vv.Kind(), vv.GetName())
-			}(v)
-		}
-
-		// selective poke: here we reduce the number of initial pokes
-		// to the minimum required to activate every vertex in the
-		// graph, either by direct action, or by getting poked by a
-		// vertex that was previously activated. if we poke each vertex
-		// that has no incoming edges, then we can be sure to reach the
-		// whole graph. Please note: this may mask certain optimization
-		// failures, such as any poke limiting code in Poke() or
-		// BackPoke(). You might want to disable this selective start
-		// when experimenting with and testing those elements.
-		// if we are unpausing (since it's not the first run of this
-		// function) we need to poke to *unpause* every graph vertex,
-		// and not just selectively the subset with no indegree.
-		if (!first) || indegree[v] == 0 {
-			// ensure state is started before continuing on to next vertex
-			for !v.SendEvent(event.EventStart, true, false) {
-				if global.DEBUG {
-					// if SendEvent fails, we aren't up yet
-					log.Printf("%v[%v]: Retrying SendEvent(Start)", v.Kind(), v.GetName())
-					// sleep here briefly or otherwise cause
-					// a different goroutine to be scheduled
-					time.Sleep(1 * time.Millisecond)
-				}
-			}
-		}
-	}
-}
-
-// Pause sends pause events to the graph in a topological sort order.
-func (g *Graph) Pause() {
-	log.Printf("State: %v -> %v", g.setState(graphStatePausing), g.getState())
-	defer log.Printf("State: %v -> %v", g.setState(graphStatePaused), g.getState())
-	t, _ := g.TopologicalSort()
-	for _, v := range t { // squeeze out the events...
-		v.SendEvent(event.EventPause, true, false)
-	}
-}
-
-// Exit sends exit events to the graph in a topological sort order.
-func (g *Graph) Exit() {
-	if g == nil {
-		return
-	} // empty graph that wasn't populated yet
-	t, _ := g.TopologicalSort()
-	for _, v := range t { // squeeze out the events...
-		// turn off the taps...
-		// XXX: consider instead doing this by closing the Res.events channel instead?
-		// XXX: do this by sending an exit signal, and then returning
-		// when we hit the 'default' in the select statement!
-		// XXX: we can do this to quiesce, but it's not necessary now
-
-		v.SendEvent(event.EventExit, true, false)
-	}
-}
-
 // GraphSync updates the oldGraph so that it matches the newGraph receiver. It
 // leaves identical elements alone so that they don't need to be refreshed.
 // FIXME: add test cases
@@ -1040,10 +619,10 @@ func (g *Graph) GraphMetas() []*resources.MetaParams {
 	return metas
 }
 
-// AssociateData associates some data with the object in the graph in question
+// AssociateData associates some data with the object in the graph in question.
-func (g *Graph) AssociateData(converger converger.Converger) {
+func (g *Graph) AssociateData(data *resources.Data) {
-	for v := range g.GetVerticesChan() {
+	for k := range g.Adjacency {
-		v.Res.AssociateData(converger)
+		k.Res.AssociateData(data)
 	}
 }
 

puppet/gapi.go

@@ -70,12 +70,12 @@ func (obj *GAPI) Graph() (*pgraph.Graph, error) {
 	if config == nil {
 		return nil, fmt.Errorf("Puppet: ParseConfigFromPuppet returned nil!")
 	}
-	g, err := config.NewGraphFromConfig(obj.data.Hostname, obj.data.EmbdEtcd, obj.data.Noop)
+	g, err := config.NewGraphFromConfig(obj.data.Hostname, obj.data.World, obj.data.Noop)
 	return g, err
 }
 
-// SwitchStream returns nil errors every time there could be a new graph.
+// Next returns nil errors every time there could be a new graph.
-func (obj *GAPI) SwitchStream() chan error {
+func (obj *GAPI) Next() chan error {
 	if obj.data.NoWatch {
 		return nil
 	}

puppet/puppet.go

@@ -26,17 +26,18 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/purpleidea/mgmt/global"
 	"github.com/purpleidea/mgmt/yamlgraph"
 )
 
 const (
 	// PuppetYAMLBufferSize is the maximum buffer size for the yaml input data
 	PuppetYAMLBufferSize = 65535
+	// Debug is a local debug constant used in this module
+	Debug = false // FIXME: integrate with global debug flag
 )
 
 func runPuppetCommand(cmd *exec.Cmd) ([]byte, error) {
-	if global.DEBUG {
+	if Debug {
 		log.Printf("Puppet: running command: %v", cmd)
 	}
 
@@ -71,7 +72,7 @@ func runPuppetCommand(cmd *exec.Cmd) ([]byte, error) {
 		// will choke on an oversized slice. http://stackoverflow.com/a/33726617/3356612
 		result = append(result, data[0:count]...)
 	}
-	if global.DEBUG {
+	if Debug {
 		log.Printf("Puppet: read %v bytes of data from puppet", len(result))
 	}
 	for scanner := bufio.NewScanner(stderr); scanner.Scan(); {
@@ -117,7 +118,7 @@ func ParseConfigFromPuppet(puppetParam, puppetConf string) *yamlgraph.GraphConfi
 
 // PuppetInterval returns the graph refresh interval from the puppet configuration.
 func PuppetInterval(puppetConf string) int {
-	if global.DEBUG {
+	if Debug {
 		log.Printf("Puppet: determining graph refresh interval")
 	}
 	var cmd *exec.Cmd

recwatch/configwatch.go

@@ -20,12 +20,12 @@ package recwatch
 import (
 	"log"
 	"sync"
-
-	"github.com/purpleidea/mgmt/global"
 )
 
 // ConfigWatcher returns events on a channel anytime one of its files events.
 type ConfigWatcher struct {
+	Flags Flags
+
 	ch        chan string
 	wg        sync.WaitGroup
 	closechan chan struct{}
@@ -56,7 +56,7 @@ func (obj *ConfigWatcher) Add(file ...string) {
 	obj.wg.Add(1)
 	go func() {
 		defer obj.wg.Done()
-		ch := ConfigWatch(file[0])
+		ch := obj.ConfigWatch(file[0])
 		for {
 			select {
 			case e := <-ch:
@@ -100,7 +100,7 @@ func (obj *ConfigWatcher) Close() {
 }
 
 // ConfigWatch writes on the channel every time an event is seen for the path.
-func ConfigWatch(file string) chan error {
+func (obj *ConfigWatcher) ConfigWatch(file string) chan error {
 	ch := make(chan error)
 	go func() {
 		recWatcher, err := NewRecWatcher(file, false)
@@ -109,9 +109,10 @@ func ConfigWatch(file string) chan error {
 			close(ch)
 			return
 		}
+		recWatcher.Flags = obj.Flags
 		defer recWatcher.Close()
 		for {
-			if global.DEBUG {
+			if obj.Flags.Debug {
 				log.Printf("Watching: %v", file)
 			}
 			select {

recwatch/flags.go

@@ -15,12 +15,9 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-// Package global holds some global variables that are used throughout the code.
+package recwatch
-package global
 
-// These constants are used throughout the program.
+// Flags contains all the constant flags that recwatch needs.
-const (
+type Flags struct {
-	DEBUG   = false // add additional log messages
+	Debug bool
-	TRACE   = false // add execution flow log messages
+}
-	VERBOSE = false // add extra log message output
-)

recwatch/recwatch.go

@@ -29,7 +29,6 @@ import (
 	"sync"
 	"syscall"
 
-	"github.com/purpleidea/mgmt/global" // XXX: package mgmtmain instead?
 	"github.com/purpleidea/mgmt/util"
 
 	"gopkg.in/fsnotify.v1"
@@ -46,6 +45,7 @@ type Event struct {
 type RecWatcher struct {
 	Path     string // computed path
 	Recurse  bool   // should we watch recursively?
+	Flags    Flags
 	isDir    bool   // computed isDir
 	safename string // safe path
 	watcher  *fsnotify.Watcher
@@ -150,12 +150,12 @@ func (obj *RecWatcher) Watch() error {
 		if current == "" { // the empty string top is the root dir ("/")
 			current = "/"
 		}
-		if global.DEBUG {
+		if obj.Flags.Debug {
 			log.Printf("Watching: %s", current) // attempting to watch...
 		}
 		// initialize in the loop so that we can reset on rm-ed handles
 		if err := obj.watcher.Add(current); err != nil {
-			if global.DEBUG {
+			if obj.Flags.Debug {
 				log.Printf("watcher.Add(%s): Error: %v", current, err)
 			}
 
@@ -178,7 +178,7 @@ func (obj *RecWatcher) Watch() error {
 
 		select {
 		case event := <-obj.watcher.Events:
-			if global.DEBUG {
+			if obj.Flags.Debug {
 				log.Printf("Watch(%s), Event(%s): %v", current, event.Name, event.Op)
 			}
 			// the deeper you go, the bigger the deltaDepth is...
@@ -291,7 +291,7 @@ func (obj *RecWatcher) addSubFolders(p string) error {
 	}
 	// look at all subfolders...
 	walkFn := func(path string, info os.FileInfo, err error) error {
-		if global.DEBUG {
+		if obj.Flags.Debug {
 			log.Printf("Walk: %s (%v): %v", path, info, err)
 		}
 		if err != nil {

remote/remote.go

@@ -62,7 +62,6 @@ import (
 	"time"
 
 	cv "github.com/purpleidea/mgmt/converger"
-	"github.com/purpleidea/mgmt/global"
 	"github.com/purpleidea/mgmt/util"
 	"github.com/purpleidea/mgmt/yamlgraph"
 
@@ -85,6 +84,12 @@ const (
 	nonInteractivePasswordTimeout        = 5 * 2                                  // five minutes
 )
 
+// Flags are constants required by the remote lib.
+type Flags struct {
+	Program string
+	Debug   bool
+}
+
 // The SSH struct is the unit building block for a single remote SSH connection.
 type SSH struct {
 	hostname string // uuid of the host, as used by the --hostname argument
@@ -116,7 +121,7 @@ type SSH struct {
 	lock    sync.Mutex     // mutex to avoid exit races
 	exiting bool           // flag to let us know if we're exiting
 
-	program  string // name of the binary
+	flags    Flags  // constant runtime values
 	remotewd string // path to remote working directory
 	execpath string // path to remote mgmt binary
 	filepath string // path to remote file config
@@ -151,11 +156,11 @@ func (obj *SSH) Sftp() error {
 	var err error
 
 	if obj.client == nil {
-		return fmt.Errorf("Not dialed!")
+		return fmt.Errorf("not dialed")
 	}
 	// this check is needed because the golang path.Base function is weird!
 	if strings.HasSuffix(obj.file, "/") {
-		return fmt.Errorf("File must not be a directory.")
+		return fmt.Errorf("file must not be a directory")
 	}
 
 	// we run local operations first so that remote clean up is easier...
@@ -224,7 +229,7 @@ func (obj *SSH) Sftp() error {
 		break
 	}
 
-	obj.execpath = path.Join(obj.remotewd, obj.program) // program is a compile time string
+	obj.execpath = path.Join(obj.remotewd, obj.flags.Program) // program is a compile time string
 	log.Printf("Remote: Remote path is: %s", obj.execpath)
 
 	var same bool
@@ -249,7 +254,7 @@ func (obj *SSH) Sftp() error {
 	// make file executable; don't cache this in case it didn't ever happen
 	// TODO: do we want the group or other bits set?
 	if err := obj.sftp.Chmod(obj.execpath, 0770); err != nil {
-		return fmt.Errorf("Can't set file mode bits!")
+		return fmt.Errorf("can't set file mode bits")
 	}
 
 	// copy graph file
@@ -268,7 +273,7 @@ func (obj *SSH) Sftp() error {
 // SftpGraphCopy is a helper function used for re-copying the graph definition.
 func (obj *SSH) SftpGraphCopy() (int64, error) {
 	if obj.filepath == "" {
-		return -1, fmt.Errorf("Sftp session isn't ready yet!")
+		return -1, fmt.Errorf("sftp session isn't ready yet")
 	}
 	return obj.SftpCopy(obj.file, obj.filepath)
 }
@@ -276,7 +281,7 @@ func (obj *SSH) SftpGraphCopy() (int64, error) {
 // SftpCopy is a simple helper function that runs a local -> remote sftp copy.
 func (obj *SSH) SftpCopy(src, dst string) (int64, error) {
 	if obj.sftp == nil {
-		return -1, fmt.Errorf("Sftp session is not active!")
+		return -1, fmt.Errorf("sftp session is not active")
 	}
 	var err error
 	// TODO: add a check to make sure we don't run two copies of this
@@ -308,7 +313,7 @@ func (obj *SSH) SftpCopy(src, dst string) (int64, error) {
 		return n, fmt.Errorf("Can't copy to remote path: %v", err)
 	}
 	if n <= 0 {
-		return n, fmt.Errorf("Zero bytes copied!")
+		return n, fmt.Errorf("zero bytes copied")
 	}
 	return n, nil
 }
@@ -386,10 +391,10 @@ func (obj *SSH) Tunnel() error {
 	var err error
 
 	if len(obj.clientURLs) < 1 {
-		return fmt.Errorf("Need at least one client URL to tunnel!")
+		return fmt.Errorf("need at least one client URL to tunnel")
 	}
 	if len(obj.remoteURLs) < 1 {
-		return fmt.Errorf("Need at least one remote URL to tunnel!")
+		return fmt.Errorf("need at least one remote URL to tunnel")
 	}
 
 	// TODO: do something less arbitrary about which one we pick?
@@ -448,7 +453,7 @@ func (obj *SSH) forward(remoteConn net.Conn) net.Conn {
 			log.Printf("Remote: io.Copy error: %s", err)
 			// FIXME: what should we do here???
 		}
-		if global.DEBUG {
+		if obj.flags.Debug {
 			log.Printf("Remote: io.Copy finished: %d", n)
 		}
 	}
@@ -472,10 +477,10 @@ func (obj *SSH) TunnelClose() error {
 // Exec runs the binary on the remote server.
 func (obj *SSH) Exec() error {
 	if obj.execpath == "" {
-		return fmt.Errorf("Must have a binary path to execute!")
+		return fmt.Errorf("must have a binary path to execute")
 	}
 	if obj.filepath == "" {
-		return fmt.Errorf("Must have a graph definition to run!")
+		return fmt.Errorf("must have a graph definition to run")
 	}
 
 	var err error
@@ -563,7 +568,7 @@ func (obj *SSH) ExecExit() error {
 	}
 
 	// FIXME: workaround: force a signal!
-	if _, err := obj.simpleRun(fmt.Sprintf("killall -SIGINT %s", obj.program)); err != nil { // FIXME: low specificity
+	if _, err := obj.simpleRun(fmt.Sprintf("killall -SIGINT %s", obj.flags.Program)); err != nil { // FIXME: low specificity
 		log.Printf("Remote: Failed to send SIGINT: %s", err.Error())
 	}
 
@@ -572,12 +577,12 @@ func (obj *SSH) ExecExit() error {
 		// try killing the process more violently
 		time.Sleep(10 * time.Second)
 		//obj.session.Signal(ssh.SIGKILL)
-		cmd := fmt.Sprintf("killall -SIGKILL %s", obj.program) // FIXME: low specificity
+		cmd := fmt.Sprintf("killall -SIGKILL %s", obj.flags.Program) // FIXME: low specificity
 		obj.simpleRun(cmd)
 	}()
 
 	// FIXME: workaround: wait (spin lock) until process quits cleanly...
-	cmd := fmt.Sprintf("while killall -0 %s 2> /dev/null; do sleep 1s; done", obj.program) // FIXME: low specificity
+	cmd := fmt.Sprintf("while killall -0 %s 2> /dev/null; do sleep 1s; done", obj.flags.Program) // FIXME: low specificity
 	if _, err := obj.simpleRun(cmd); err != nil {
 		return fmt.Errorf("Error waiting: %s", err)
 	}
@@ -704,11 +709,11 @@ type Remotes struct {
 	cuids              map[string]cv.ConvergerUID // map to each SSH struct with the remote as the key
 	callbackCancelFunc func()                     // stored callback function cancel function
 
-	program string // name of the program
+	flags Flags // constant runtime values
 }
 
 // NewRemotes builds a Remotes struct.
-func NewRemotes(clientURLs, remoteURLs []string, noop bool, remotes []string, fileWatch chan string, cConns uint16, interactive bool, sshPrivIdRsa string, caching bool, depth uint16, prefix string, converger cv.Converger, convergerCb func(func(map[string]bool) error) (func(), error), program string) *Remotes {
+func NewRemotes(clientURLs, remoteURLs []string, noop bool, remotes []string, fileWatch chan string, cConns uint16, interactive bool, sshPrivIdRsa string, caching bool, depth uint16, prefix string, converger cv.Converger, convergerCb func(func(map[string]bool) error) (func(), error), flags Flags) *Remotes {
 	return &Remotes{
 		clientURLs:   clientURLs,
 		remoteURLs:   remoteURLs,
@@ -728,7 +733,7 @@ func NewRemotes(clientURLs, remoteURLs []string, noop bool, remotes []string, fi
 		semaphore:    NewSemaphore(int(cConns)),
 		hostnames:    make([]string, len(remotes)),
 		cuids:        make(map[string]cv.ConvergerUID),
-		program:      program,
+		flags:        flags,
 	}
 }
 
@@ -767,7 +772,7 @@ func (obj *Remotes) NewSSH(file string) (*SSH, error) {
 	}
 	host = x[0]
 	if host == "" {
-		return nil, fmt.Errorf("Empty hostname!")
+		return nil, fmt.Errorf("empty hostname")
 	}
 
 	user := defaultUser // default
@@ -790,7 +795,7 @@ func (obj *Remotes) NewSSH(file string) (*SSH, error) {
 	}
 
 	if len(auth) == 0 {
-		return nil, fmt.Errorf("No authentication methods available!")
+		return nil, fmt.Errorf("no authentication methods available")
 	}
 
 	//hostname := config.Hostname // TODO: optionally specify local hostname somehow
@@ -799,7 +804,7 @@ func (obj *Remotes) NewSSH(file string) (*SSH, error) {
 		hostname = host // default to above
 	}
 	if util.StrInList(hostname, obj.hostnames) {
-		return nil, fmt.Errorf("Remote: Hostname `%s` already exists!", hostname)
+		return nil, fmt.Errorf("Remote: Hostname `%s` already exists", hostname)
 	}
 	obj.hostnames = append(obj.hostnames, hostname)
 
@@ -818,14 +823,14 @@ func (obj *Remotes) NewSSH(file string) (*SSH, error) {
 		caching:    obj.caching,
 		converger:  obj.converger,
 		prefix:     obj.prefix,
-		program:    obj.program,
+		flags:      obj.flags,
 	}, nil
 }
 
 // sshKeyAuth is a helper function to get the ssh key auth struct needed
 func (obj *Remotes) sshKeyAuth() (ssh.AuthMethod, error) {
 	if obj.sshPrivIdRsa == "" {
-		return nil, fmt.Errorf("Empty path specified!")
+		return nil, fmt.Errorf("empty path specified")
 	}
 	p := ""
 	// TODO: this doesn't match strings of the form: ~james/.ssh/id_rsa
@@ -838,7 +843,7 @@ func (obj *Remotes) sshKeyAuth() (ssh.AuthMethod, error) {
 		p = path.Join(usr.HomeDir, obj.sshPrivIdRsa[len("~/"):])
 	}
 	if p == "" {
-		return nil, fmt.Errorf("Empty path specified!")
+		return nil, fmt.Errorf("empty path specified")
 	}
 	// A public key may be used to authenticate against the server by using
 	// an unencrypted PEM-encoded private key file. If you have an encrypted
@@ -887,7 +892,7 @@ func (obj *Remotes) passwordCallback(user, host string) func() (string, error) {
 		case e := <-failchan:
 			return "", e
 		case <-util.TimeAfterOrBlock(timeout):
-			return "", fmt.Errorf("Interactive timeout reached!")
+			return "", fmt.Errorf("interactive timeout reached")
 		}
 	}
 	return cb
@@ -924,7 +929,7 @@ func (obj *Remotes) Run() {
 			if !ok { // no status on hostname means unconverged!
 				continue
 			}
-			if global.DEBUG {
+			if obj.flags.Debug {
 				log.Printf("Remote: Converged: Status: %+v", obj.converger.Status())
 			}
 			// if exiting, don't update, it will be unregistered...

resources/exec.go

@@ -21,15 +21,15 @@ import (
 	"bufio"
 	"bytes"
 	"encoding/gob"
-	"errors"
 	"fmt"
 	"log"
 	"os/exec"
 	"strings"
-	"time"
 
 	"github.com/purpleidea/mgmt/event"
 	"github.com/purpleidea/mgmt/util"
+
+	errwrap "github.com/pkg/errors"
 )
 
 func init() {
@@ -110,22 +110,7 @@ func (obj *ExecRes) BufioChanScanner(scanner *bufio.Scanner) (chan string, chan
 
 // Watch is the primary listener for this resource and it outputs events.
 func (obj *ExecRes) Watch(processChan chan event.Event) error {
-	if obj.IsWatching() {
+	cuid := obj.Converger() // get the converger uid used to report status
-		return nil
-	}
-	obj.SetWatching(true)
-	defer obj.SetWatching(false)
-	cuid := obj.converger.Register()
-	defer cuid.Unregister()
-
-	var startup bool
-	Startup := func(block bool) <-chan time.Time {
-		if block {
-			return nil // blocks forever
-			//return make(chan time.Time) // blocks forever
-		}
-		return time.After(time.Duration(500) * time.Millisecond) // 1/2 the resolution of converged timeout
-	}
 
 	var send = false // send event?
 	var exit = false
@@ -151,7 +136,7 @@ func (obj *ExecRes) Watch(processChan chan event.Event) error {
 
 		cmdReader, err := cmd.StdoutPipe()
 		if err != nil {
-			return fmt.Errorf("%s[%s]: Error creating StdoutPipe for Cmd: %v", obj.Kind(), obj.GetName(), err)
+			return errwrap.Wrapf(err, "Error creating StdoutPipe for Cmd")
 		}
 		scanner := bufio.NewScanner(cmdReader)
 
@@ -162,19 +147,24 @@ func (obj *ExecRes) Watch(processChan chan event.Event) error {
 			cmd.Process.Kill() // TODO: is this necessary?
 		}()
 		if err := cmd.Start(); err != nil {
-			return fmt.Errorf("%s[%s]: Error starting Cmd: %v", obj.Kind(), obj.GetName(), err)
+			return errwrap.Wrapf(err, "Error starting Cmd")
 		}
 
 		bufioch, errch = obj.BufioChanScanner(scanner)
 	}
 
+	// notify engine that we're running
+	if err := obj.Running(processChan); err != nil {
+		return err // bubble up a NACK...
+	}
+
 	for {
 		obj.SetState(ResStateWatching) // reset
 		select {
 		case text := <-bufioch:
 			cuid.SetConverged(false)
 			// each time we get a line of output, we loop!
-			log.Printf("%v[%v]: Watch output: %s", obj.Kind(), obj.GetName(), text)
+			log.Printf("%s[%s]: Watch output: %s", obj.Kind(), obj.GetName(), text)
 			if text != "" {
 				send = true
 			}
@@ -184,10 +174,10 @@ func (obj *ExecRes) Watch(processChan chan event.Event) error {
 			if err == nil { // EOF
 				// FIXME: add an "if watch command ends/crashes"
 				// restart or generate error option
-				return fmt.Errorf("%s[%s]: Reached EOF", obj.Kind(), obj.GetName())
+				return fmt.Errorf("Reached EOF")
 			}
 			// error reading input?
-			return fmt.Errorf("Unknown %s[%s] error: %v", obj.Kind(), obj.GetName(), err)
+			return errwrap.Wrapf(err, "Unknown error")
 
 		case event := <-obj.Events():
 			cuid.SetConverged(false)
@@ -198,18 +188,13 @@ func (obj *ExecRes) Watch(processChan chan event.Event) error {
 		case <-cuid.ConvergedTimer():
 			cuid.SetConverged(true) // converged!
 			continue
-
-		case <-Startup(startup):
-			cuid.SetConverged(false)
-			send = true
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
-			startup = true // startup finished
 			send = false
 			// it is okay to invalidate the clean state on poke too
-			obj.isStateOK = false // something made state dirty
+			obj.StateOK(false) // something made state dirty
 			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
 				return err // we exit or bubble up a NACK...
 			}
@@ -220,12 +205,11 @@ func (obj *ExecRes) Watch(processChan chan event.Event) error {
 // CheckApply checks the resource state and applies the resource if the bool
 // input is true. It returns error info and if the state check passed or not.
 // TODO: expand the IfCmd to be a list of commands
-func (obj *ExecRes) CheckApply(apply bool) (checkok bool, err error) {
+func (obj *ExecRes) CheckApply(apply bool) (checkOK bool, err error) {
-	log.Printf("%v[%v]: CheckApply(%t)", obj.Kind(), obj.GetName(), apply)
 
 	// if there is a watch command, but no if command, run based on state
 	if obj.WatchCmd != "" && obj.IfCmd == "" {
-		if obj.isStateOK {
+		if obj.IsStateOK() { // FIXME: this is done by engine now...
 			return true, nil
 		}
 
@@ -263,7 +247,7 @@ func (obj *ExecRes) CheckApply(apply bool) (checkok bool, err error) {
 		// if there is no watcher and no onlyif check, assume we should run
 	} else { // if obj.WatchCmd == "" && obj.IfCmd == "" {
 		// just run if state is dirty
-		if obj.isStateOK {
+		if obj.IsStateOK() { // FIXME: this is done by engine now...
 			return true, nil
 		}
 	}
@@ -274,7 +258,7 @@ func (obj *ExecRes) CheckApply(apply bool) (checkok bool, err error) {
 	}
 
 	// apply portion
-	log.Printf("%v[%v]: Apply", obj.Kind(), obj.GetName())
+	log.Printf("%s[%s]: Apply", obj.Kind(), obj.GetName())
 	var cmdName string
 	var cmdArgs []string
 	if obj.Shell == "" {
@@ -295,9 +279,8 @@ func (obj *ExecRes) CheckApply(apply bool) (checkok bool, err error) {
 	var out bytes.Buffer
 	cmd.Stdout = &out
 
-	if err = cmd.Start(); err != nil {
+	if err := cmd.Start(); err != nil {
-		log.Printf("%v[%v]: Error starting Cmd: %v", obj.Kind(), obj.GetName(), err)
+		return false, errwrap.Wrapf(err, "Error starting Cmd")
-		return false, err
 	}
 
 	timeout := obj.Timeout
@@ -308,35 +291,35 @@ func (obj *ExecRes) CheckApply(apply bool) (checkok bool, err error) {
 	go func() { done <- cmd.Wait() }()
 
 	select {
-	case err = <-done:
+	case err := <-done:
 		if err != nil {
-			log.Printf("%v[%v]: Error waiting for Cmd: %v", obj.Kind(), obj.GetName(), err)
+			e := errwrap.Wrapf(err, "Error waiting for Cmd")
-			return false, err
+			return false, e
 		}
 
 	case <-util.TimeAfterOrBlock(timeout):
-		log.Printf("%v[%v]: Timeout waiting for Cmd", obj.Kind(), obj.GetName())
 		//cmd.Process.Kill() // TODO: is this necessary?
-		return false, errors.New("Timeout waiting for Cmd!")
+		return false, fmt.Errorf("Timeout waiting for Cmd!")
 	}
 
 	// TODO: if we printed the stdout while the command is running, this
 	// would be nice, but it would require terminal log output that doesn't
 	// interleave all the parallel parts which would mix it all up...
 	if s := out.String(); s == "" {
-		log.Printf("Exec[%v]: Command output is empty!", obj.Name)
+		log.Printf("%s[%s]: Command output is empty!", obj.Kind(), obj.GetName())
+
 	} else {
-		log.Printf("Exec[%v]: Command output is:", obj.Name)
+		log.Printf("%s[%s]: Command output is:", obj.Kind(), obj.GetName())
 		log.Printf(out.String())
 	}
 	// XXX: return based on exit value!!
 
-	// the state tracking is for exec resources that can't "detect" their
+	// The state tracking is for exec resources that can't "detect" their
 	// state, and assume it's invalid when the Watch() function triggers.
-	// if we apply state successfully, we should reset it here so that we
+	// If we apply state successfully, we should reset it here so that we
 	// know that we have applied since the state was set not ok by event!
-	obj.isStateOK = true // reset
+	// This now happens automatically after the engine runs CheckApply().
-	return false, nil    // success
+	return false, nil // success
 }
 
 // ExecUID is the UID struct for ExecRes.

resources/file.go

@@ -30,12 +30,12 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/purpleidea/mgmt/event"
-	"github.com/purpleidea/mgmt/global" // XXX: package mgmtmain instead?
 	"github.com/purpleidea/mgmt/recwatch"
 	"github.com/purpleidea/mgmt/util"
+
+	errwrap "github.com/pkg/errors"
 )
 
 func init() {
@@ -45,22 +45,22 @@ func init() {
 // FileRes is a file and directory resource.
 type FileRes struct {
 	BaseRes    `yaml:",inline"`
-	Path       string `yaml:"path"` // path variable (should default to name)
+	Path       string  `yaml:"path"` // path variable (should default to name)
-	Dirname    string `yaml:"dirname"`
+	Dirname    string  `yaml:"dirname"`
-	Basename   string `yaml:"basename"`
+	Basename   string  `yaml:"basename"`
-	Content    string `yaml:"content"` // FIXME: how do you describe: "leave content alone" - state = "create" ?
+	Content    *string `yaml:"content"` // nil to mark as undefined
-	Source     string `yaml:"source"`  // file path for source content
+	Source     string  `yaml:"source"`  // file path for source content
-	State      string `yaml:"state"`   // state: exists/present?, absent, (undefined?)
+	State      string  `yaml:"state"`   // state: exists/present?, absent, (undefined?)
-	Recurse    bool   `yaml:"recurse"`
+	Recurse    bool    `yaml:"recurse"`
-	Force      bool   `yaml:"force"`
+	Force      bool    `yaml:"force"`
-	path       string // computed path
+	path       string  // computed path
-	isDir      bool   // computed isDir
+	isDir      bool    // computed isDir
 	sha256sum  string
 	recWatcher *recwatch.RecWatcher
 }
 
 // NewFileRes is a constructor for this resource. It also calls Init() for you.
-func NewFileRes(name, path, dirname, basename, content, source, state string, recurse, force bool) (*FileRes, error) {
+func NewFileRes(name, path, dirname, basename string, content *string, source, state string, recurse, force bool) (*FileRes, error) {
 	obj := &FileRes{
 		BaseRes: BaseRes{
 			Name: name,
@@ -118,11 +118,11 @@ func (obj *FileRes) Validate() error {
 		return fmt.Errorf("Basename must not start with a slash.")
 	}
 
-	if obj.Content != "" && obj.Source != "" {
+	if obj.Content != nil && obj.Source != "" {
 		return fmt.Errorf("Can't specify both Content and Source.")
 	}
 
-	if obj.isDir && obj.Content != "" { // makes no sense
+	if obj.isDir && obj.Content != nil { // makes no sense
 		return fmt.Errorf("Can't specify Content when creating a Dir.")
 	}
 
@@ -141,22 +141,7 @@ func (obj *FileRes) Validate() error {
 // must be restarted. On a clean exit it returns nil.
 // FIXME: Also watch the source directory when using obj.Source !!!
 func (obj *FileRes) Watch(processChan chan event.Event) error {
-	if obj.IsWatching() {
+	cuid := obj.Converger() // get the converger uid used to report status
-		return nil // TODO: should this be an error?
-	}
-	obj.SetWatching(true)
-	defer obj.SetWatching(false)
-	cuid := obj.converger.Register()
-	defer cuid.Unregister()
-
-	var startup bool
-	Startup := func(block bool) <-chan time.Time {
-		if block {
-			return nil // blocks forever
-			//return make(chan time.Time) // blocks forever
-		}
-		return time.After(time.Duration(500) * time.Millisecond) // 1/2 the resolution of converged timeout
-	}
 
 	var err error
 	obj.recWatcher, err = recwatch.NewRecWatcher(obj.Path, obj.Recurse)
@@ -165,12 +150,16 @@ func (obj *FileRes) Watch(processChan chan event.Event) error {
 	}
 	defer obj.recWatcher.Close()
 
+	// notify engine that we're running
+	if err := obj.Running(processChan); err != nil {
+		return err // bubble up a NACK...
+	}
+
 	var send = false // send event?
 	var exit = false
-	var dirty = false
 
 	for {
-		if global.DEBUG {
+		if obj.debug {
 			log.Printf("%s[%s]: Watching: %s", obj.Kind(), obj.GetName(), obj.Path) // attempting to watch...
 		}
 
@@ -182,40 +171,29 @@ func (obj *FileRes) Watch(processChan chan event.Event) error {
 			}
 			cuid.SetConverged(false)
 			if err := event.Error; err != nil {
-				return fmt.Errorf("Unknown %s[%s] watcher error: %v", obj.Kind(), obj.GetName(), err)
+				return errwrap.Wrapf(err, "Unknown %s[%s] watcher error", obj.Kind(), obj.GetName())
 			}
-			if global.DEBUG { // don't access event.Body if event.Error isn't nil
+			if obj.debug { // don't access event.Body if event.Error isn't nil
 				log.Printf("%s[%s]: Event(%s): %v", obj.Kind(), obj.GetName(), event.Body.Name, event.Body.Op)
 			}
 			send = true
-			dirty = true
+			obj.StateOK(false) // dirty
 
 		case event := <-obj.Events():
 			cuid.SetConverged(false)
 			if exit, send = obj.ReadEvent(&event); exit {
 				return nil // exit
 			}
-			//dirty = false // these events don't invalidate state
+			//obj.StateOK(false) // dirty // these events don't invalidate state
 
 		case <-cuid.ConvergedTimer():
 			cuid.SetConverged(true) // converged!
 			continue
-
-		case <-Startup(startup):
-			cuid.SetConverged(false)
-			send = true
-			dirty = true
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
-			startup = true // startup finished
 			send = false
-			// only invalid state on certain types of events
-			if dirty {
-				dirty = false
-				obj.isStateOK = false // something made state dirty
-			}
 			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
 				return err // we exit or bubble up a NACK...
 			}
@@ -258,7 +236,7 @@ func ReadDir(path string) ([]FileInfo, error) {
 		abs := path + smartPath(fi)
 		rel, err := filepath.Rel(path, abs) // NOTE: calls Clean()
 		if err != nil {                     // shouldn't happen
-			return nil, fmt.Errorf("ReadDir: Unhandled error: %v", err)
+			return nil, errwrap.Wrapf(err, "ReadDir: Unhandled error")
 		}
 		if fi.IsDir() {
 			rel += "/" // add a trailing slash for dirs
@@ -293,7 +271,7 @@ func mapPaths(fileInfos []FileInfo) map[string]FileInfo {
 func (obj *FileRes) fileCheckApply(apply bool, src io.ReadSeeker, dst string, sha256sum string) (string, bool, error) {
 	// TODO: does it make sense to switch dst to an io.Writer ?
 	// TODO: use obj.Force when dealing with symlinks and other file types!
-	if global.DEBUG {
+	if obj.debug {
 		log.Printf("fileCheckApply: %s -> %s", src, dst)
 	}
 
@@ -390,7 +368,7 @@ func (obj *FileRes) fileCheckApply(apply bool, src io.ReadSeeker, dst string, sh
 	if !apply {
 		return sha256sum, false, nil
 	}
-	if global.DEBUG {
+	if obj.debug {
 		log.Printf("fileCheckApply: Apply: %s -> %s", src, dst)
 	}
 
@@ -411,12 +389,12 @@ func (obj *FileRes) fileCheckApply(apply bool, src io.ReadSeeker, dst string, sh
 	// syscall.Splice(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int64, err error)
 
 	// TODO: should we offer a way to cancel the copy on ^C ?
-	if global.DEBUG {
+	if obj.debug {
 		log.Printf("fileCheckApply: Copy: %s -> %s", src, dst)
 	}
 	if n, err := io.Copy(dstFile, src); err != nil {
 		return sha256sum, false, err
-	} else if global.DEBUG {
+	} else if obj.debug {
 		log.Printf("fileCheckApply: Copied: %v", n)
 	}
 	return sha256sum, false, dstFile.Sync()
@@ -426,7 +404,7 @@ func (obj *FileRes) fileCheckApply(apply bool, src io.ReadSeeker, dst string, sh
 // It is recursive and can create directories directly, and files via the usual
 // fileCheckApply method. It returns checkOK and error as is normally expected.
 func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
-	if global.DEBUG {
+	if obj.debug {
 		log.Printf("syncCheckApply: %s -> %s", src, dst)
 	}
 	if src == "" || dst == "" {
@@ -444,12 +422,12 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 	}
 
 	if !srcIsDir && !dstIsDir {
-		if global.DEBUG {
+		if obj.debug {
 			log.Printf("syncCheckApply: %s -> %s", src, dst)
 		}
 		fin, err := os.Open(src)
 		if err != nil {
-			if global.DEBUG && os.IsNotExist(err) { // if we get passed an empty src
+			if obj.debug && os.IsNotExist(err) { // if we get passed an empty src
 				log.Printf("syncCheckApply: Missing src: %s", src)
 			}
 			return false, err
@@ -505,7 +483,7 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 					delete(smartDst, relPathFile) // rm from purge list
 				}
 
-				if global.DEBUG {
+				if obj.debug {
 					log.Printf("syncCheckApply: mkdir -m %s '%s'", fileInfo.Mode(), absDst)
 				}
 				if err := os.Mkdir(absDst, fileInfo.Mode()); err != nil {
@@ -516,12 +494,12 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 			// if we're a regular file, the recurse will create it
 		}
 
-		if global.DEBUG {
+		if obj.debug {
 			log.Printf("syncCheckApply: Recurse: %s -> %s", absSrc, absDst)
 		}
 		if obj.Recurse {
 			if c, err := obj.syncCheckApply(apply, absSrc, absDst); err != nil { // recurse
-				return false, fmt.Errorf("syncCheckApply: Recurse failed: %v", err)
+				return false, errwrap.Wrapf(err, "syncCheckApply: Recurse failed")
 			} else if !c { // don't let subsequent passes make this true
 				checkOK = false
 			}
@@ -562,7 +540,7 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 		_ = absSrc
 		//log.Printf("syncCheckApply: Recurse rm: %s -> %s", absSrc, absDst)
 		//if c, err := obj.syncCheckApply(apply, absSrc, absDst); err != nil {
-		//	return false, fmt.Errorf("syncCheckApply: Recurse rm failed: %v", err)
+		//	return false, errwrap.Wrapf(err, "syncCheckApply: Recurse rm failed")
 		//} else if !c { // don't let subsequent passes make this true
 		//	checkOK = false
 		//}
@@ -580,7 +558,7 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 
 // contentCheckApply performs a CheckApply for the file existence and content.
 func (obj *FileRes) contentCheckApply(apply bool) (checkOK bool, _ error) {
-	log.Printf("%v[%v]: contentCheckApply(%t)", obj.Kind(), obj.GetName(), apply)
+	log.Printf("%s[%s]: contentCheckApply(%t)", obj.Kind(), obj.GetName(), apply)
 
 	if obj.State == "absent" {
 		if _, err := os.Stat(obj.path); os.IsNotExist(err) {
@@ -608,12 +586,17 @@ func (obj *FileRes) contentCheckApply(apply bool) (checkOK bool, _ error) {
 		return false, err             // either nil or not
 	}
 
+	// content is not defined, leave it alone...
+	if obj.Content == nil {
+		return true, nil
+	}
+
 	if obj.Source == "" { // do the obj.Content checks first...
 		if obj.isDir { // TODO: should we create an empty dir this way?
 			log.Fatal("XXX: Not implemented!") // XXX
 		}
 
-		bufferSrc := bytes.NewReader([]byte(obj.Content))
+		bufferSrc := bytes.NewReader([]byte(*obj.Content))
 		sha256sum, checkOK, err := obj.fileCheckApply(apply, bufferSrc, obj.path, obj.sha256sum)
 		if sha256sum != "" { // empty values mean errored or didn't hash
 			// this can be valid even when the whole function errors
@@ -638,10 +621,17 @@ func (obj *FileRes) contentCheckApply(apply bool) (checkOK bool, _ error) {
 // CheckApply checks the resource state and applies the resource if the bool
 // input is true. It returns error info and if the state check passed or not.
 func (obj *FileRes) CheckApply(apply bool) (checkOK bool, _ error) {
-	log.Printf("%v[%v]: CheckApply(%t)", obj.Kind(), obj.GetName(), apply)
 
-	if obj.isStateOK { // cache the state
+	// NOTE: all send/recv change notifications *must* be processed before
-		return true, nil
+	// there is a possibility of failure in CheckApply. This is because if
+	// we fail (and possibly run again) the subsequent send->recv transfer
+	// might not have a new value to copy, and therefore we won't see this
+	// notification of change. Therefore, it is important to process these
+	// promptly, if they must not be lost, such as for cache invalidation.
+	if val, exists := obj.Recv["Content"]; exists && val.Changed {
+		// if we received on Content, and it changed, invalidate the cache!
+		log.Printf("contentCheckApply: Invalidating sha256sum of `Content`")
+		obj.sha256sum = "" // invalidate!!
 	}
 
 	checkOK = true
@@ -666,10 +656,6 @@ func (obj *FileRes) CheckApply(apply bool) (checkOK bool, _ error) {
 	//	checkOK = false
 	//}
 
-	// if we did work successfully, or are in a good state, then state is ok
-	if apply || checkOK {
-		obj.isStateOK = true
-	}
 	return checkOK, nil // w00t
 }
 
@@ -784,12 +770,17 @@ func (obj *FileRes) Compare(res Res) bool {
 		if obj.Name != res.Name {
 			return false
 		}
-		if obj.path != res.Path {
+		if obj.path != res.path {
 			return false
 		}
-		if obj.Content != res.Content {
+		if (obj.Content == nil) != (res.Content == nil) { // xor
 			return false
 		}
+		if obj.Content != nil && res.Content != nil {
+			if *obj.Content != *res.Content { // compare the strings
+				return false
+			}
+		}
 		if obj.Source != res.Source {
 			return false
 		}

resources/hostname.go

@@ -0,0 +1,295 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"encoding/gob"
+	"errors"
+	"fmt"
+	"log"
+
+	"github.com/purpleidea/mgmt/event"
+	"github.com/purpleidea/mgmt/util"
+
+	"github.com/godbus/dbus"
+	errwrap "github.com/pkg/errors"
+)
+
+// ErrResourceInsufficientParameters is returned when the configuration of the resource
+// is insufficient for the resource to do any useful work.
+var ErrResourceInsufficientParameters = errors.New(
+	"Insufficient parameters for this resource")
+
+func init() {
+	gob.Register(&HostnameRes{})
+}
+
+const (
+	hostname1Path  = "/org/freedesktop/hostname1"
+	hostname1Iface = "org.freedesktop.hostname1"
+	dbusAddMatch   = "org.freedesktop.DBus.AddMatch"
+)
+
+// HostnameRes is a resource that allows setting and watching the hostname.
+//
+// StaticHostname is the one configured in /etc/hostname or a similar file.
+// It is chosen by the local user. It is not always in sync with the current
+// host name as returned by the gethostname() system call.
+//
+// TransientHostname is the one configured via the kernel's sethostbyname().
+// It can be different from the static hostname in case DHCP or mDNS have been
+// configured to change the name based on network information.
+//
+// PrettyHostname is a free-form UTF8 host name for presentation to the user.
+//
+// Hostname is the fallback value for all 3 fields above, if only Hostname is
+// specified, it will set all 3 fields to this value.
+type HostnameRes struct {
+	BaseRes           `yaml:",inline"`
+	Hostname          string `yaml:"hostname"`
+	PrettyHostname    string `yaml:"pretty_hostname"`
+	StaticHostname    string `yaml:"static_hostname"`
+	TransientHostname string `yaml:"transient_hostname"`
+
+	conn *dbus.Conn
+}
+
+// NewHostnameRes is a constructor for this resource. It also calls Init() for you.
+func NewHostnameRes(name, staticHostname, transientHostname, prettyHostname string) (*HostnameRes, error) {
+	obj := &HostnameRes{
+		BaseRes: BaseRes{
+			Name: name,
+		},
+		PrettyHostname:    prettyHostname,
+		StaticHostname:    staticHostname,
+		TransientHostname: transientHostname,
+	}
+	return obj, obj.Init()
+}
+
+// Init runs some startup code for this resource.
+func (obj *HostnameRes) Init() error {
+	obj.BaseRes.kind = "Hostname"
+	if obj.PrettyHostname == "" {
+		obj.PrettyHostname = obj.Hostname
+	}
+	if obj.StaticHostname == "" {
+		obj.StaticHostname = obj.Hostname
+	}
+	if obj.TransientHostname == "" {
+		obj.TransientHostname = obj.Hostname
+	}
+	return obj.BaseRes.Init() // call base init, b/c we're overriding
+}
+
+// Validate if the params passed in are valid data.
+// FIXME: where should this get called ?
+func (obj *HostnameRes) Validate() error {
+	if obj.PrettyHostname == "" && obj.StaticHostname == "" && obj.TransientHostname == "" {
+		return ErrResourceInsufficientParameters
+	}
+	return nil
+}
+
+// Watch is the primary listener for this resource and it outputs events.
+func (obj *HostnameRes) Watch(processChan chan event.Event) error {
+	cuid := obj.Converger() // get the converger uid used to report status
+
+	// if we share the bus with others, we will get each others messages!!
+	bus, err := util.SystemBusPrivateUsable() // don't share the bus connection!
+	if err != nil {
+		return errwrap.Wrap(err, "Failed to connect to bus")
+	}
+	defer bus.Close()
+	callResult := bus.BusObject().Call(
+		"org.freedesktop.DBus.AddMatch", 0,
+		fmt.Sprintf("type='signal',path='%s',interface='org.freedesktop.DBus.Properties',member='PropertiesChanged'", hostname1Path))
+	if callResult.Err != nil {
+		return errwrap.Wrap(callResult.Err, "Failed to subscribe to DBus events for hostname1")
+	}
+
+	signals := make(chan *dbus.Signal, 10) // closed by dbus package
+	bus.Signal(signals)
+
+	// notify engine that we're running
+	if err := obj.Running(processChan); err != nil {
+		return err // bubble up a NACK...
+	}
+
+	var send = false // send event?
+
+	for {
+		obj.SetState(ResStateWatching) // reset
+		select {
+		case <-signals:
+			cuid.SetConverged(false)
+			send = true
+			obj.StateOK(false) // dirty
+
+		case event := <-obj.Events():
+			cuid.SetConverged(false)
+			// we avoid sending events on unpause
+			if exit, _ := obj.ReadEvent(&event); exit {
+				return nil // exit
+			}
+			send = true
+			obj.StateOK(false) // dirty
+
+		case <-cuid.ConvergedTimer():
+			cuid.SetConverged(true) // converged!
+			continue
+		}
+
+		// do all our event sending all together to avoid duplicate msgs
+		if send {
+			send = false
+
+			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
+				return err // we exit or bubble up a NACK...
+			}
+		}
+	}
+}
+
+func updateHostnameProperty(object dbus.BusObject, expectedValue, property, setterName string, apply bool) (checkOK bool, err error) {
+	propertyObject, err := object.GetProperty("org.freedesktop.hostname1." + property)
+	if err != nil {
+		return false, errwrap.Wrapf(err, "failed to get org.freedesktop.hostname1.%s", property)
+	}
+	if propertyObject.Value() == nil {
+		return false, errwrap.Errorf("Unexpected nil value received when reading property %s", property)
+	}
+
+	propertyValue, ok := propertyObject.Value().(string)
+	if !ok {
+		return false, fmt.Errorf("Received unexpected type as %s value, expected string got '%T'", property, propertyValue)
+	}
+
+	// expected value and actual value match => checkOk
+	if propertyValue == expectedValue {
+		return true, nil
+	}
+
+	// nothing to do anymore
+	if !apply {
+		return false, nil
+	}
+
+	// attempting to apply the changes
+	log.Printf("Changing %s: %s => %s", property, propertyValue, expectedValue)
+	if err := object.Call("org.freedesktop.hostname1."+setterName, 0, expectedValue, false).Err; err != nil {
+		return false, errwrap.Wrapf(err, "failed to call org.freedesktop.hostname1.%s", setterName)
+	}
+
+	// all good changes should now be applied again
+	return false, nil
+}
+
+// CheckApply method for Hostname resource.
+func (obj *HostnameRes) CheckApply(apply bool) (checkOK bool, err error) {
+	conn, err := util.SystemBusPrivateUsable()
+	if err != nil {
+		return false, errwrap.Wrap(err, "Failed to connect to the private system bus")
+	}
+	defer conn.Close()
+
+	hostnameObject := conn.Object(hostname1Iface, hostname1Path)
+
+	checkOK = true
+	if obj.PrettyHostname != "" {
+		propertyCheckOK, err := updateHostnameProperty(hostnameObject, obj.PrettyHostname, "PrettyHostname", "SetPrettyHostname", apply)
+		if err != nil {
+			return false, err
+		}
+		checkOK = checkOK && propertyCheckOK
+	}
+	if obj.StaticHostname != "" {
+		propertyCheckOK, err := updateHostnameProperty(hostnameObject, obj.StaticHostname, "StaticHostname", "SetStaticHostname", apply)
+		if err != nil {
+			return false, err
+		}
+		checkOK = checkOK && propertyCheckOK
+	}
+	if obj.TransientHostname != "" {
+		propertyCheckOK, err := updateHostnameProperty(hostnameObject, obj.TransientHostname, "Hostname", "SetHostname", apply)
+		if err != nil {
+			return false, err
+		}
+		checkOK = checkOK && propertyCheckOK
+	}
+
+	return checkOK, nil
+}
+
+// HostnameUID is the UID struct for HostnameRes.
+type HostnameUID struct {
+	BaseUID
+	name              string
+	prettyHostname    string
+	staticHostname    string
+	transientHostname string
+}
+
+// AutoEdges returns the AutoEdge interface. In this case no autoedges are used.
+func (obj *HostnameRes) AutoEdges() AutoEdge {
+	return nil
+}
+
+// GetUIDs includes all params to make a unique identification of this object.
+// Most resources only return one, although some resources can return multiple.
+func (obj *HostnameRes) GetUIDs() []ResUID {
+	x := &HostnameUID{
+		BaseUID:           BaseUID{name: obj.GetName(), kind: obj.Kind()},
+		name:              obj.Name,
+		prettyHostname:    obj.PrettyHostname,
+		staticHostname:    obj.StaticHostname,
+		transientHostname: obj.TransientHostname,
+	}
+	return []ResUID{x}
+}
+
+// GroupCmp returns whether two resources can be grouped together or not.
+func (obj *HostnameRes) GroupCmp(r Res) bool {
+	return false
+}
+
+// Compare two resources and return if they are equivalent.
+func (obj *HostnameRes) Compare(res Res) bool {
+	switch res := res.(type) {
+	// we can only compare HostnameRes to others of the same resource
+	case *HostnameRes:
+		if !obj.BaseRes.Compare(res) { // call base Compare
+			return false
+		}
+		if obj.Name != res.Name {
+			return false
+		}
+		if obj.PrettyHostname != res.PrettyHostname {
+			return false
+		}
+		if obj.StaticHostname != res.StaticHostname {
+			return false
+		}
+		if obj.TransientHostname != res.TransientHostname {
+			return false
+		}
+	default:
+		return false
+	}
+	return true
+}

resources/msg.go

@@ -23,7 +23,6 @@ import (
 	"log"
 	"regexp"
 	"strings"
-	"time"
 
 	"github.com/purpleidea/mgmt/event"
 
@@ -94,23 +93,53 @@ func (obj *MsgRes) Validate() error {
 	return nil
 }
 
+// isAllStateOK derives a compound state from all internal cache flags that apply to this resource.
+func (obj *MsgRes) isAllStateOK() bool {
+	if obj.Journal && !obj.journalStateOK {
+		return false
+	}
+	if obj.Syslog && !obj.syslogStateOK {
+		return false
+	}
+	return obj.logStateOK
+}
+
+// updateStateOK sets the global state so it can be read by the engine.
+func (obj *MsgRes) updateStateOK() {
+	obj.StateOK(obj.isAllStateOK())
+}
+
+// JournalPriority converts a string description to a numeric priority.
+// XXX: Have Validate() make sure it actually is one of these.
+func (obj *MsgRes) journalPriority() journal.Priority {
+	switch obj.Priority {
+	case "Emerg":
+		return journal.PriEmerg
+	case "Alert":
+		return journal.PriAlert
+	case "Crit":
+		return journal.PriCrit
+	case "Err":
+		return journal.PriErr
+	case "Warning":
+		return journal.PriWarning
+	case "Notice":
+		return journal.PriNotice
+	case "Info":
+		return journal.PriInfo
+	case "Debug":
+		return journal.PriDebug
+	}
+	return journal.PriNotice
+}
+
 // Watch is the primary listener for this resource and it outputs events.
 func (obj *MsgRes) Watch(processChan chan event.Event) error {
-	if obj.IsWatching() {
+	cuid := obj.Converger() // get the converger uid used to report status
-		return nil
-	}
-	obj.SetWatching(true)
-	defer obj.SetWatching(false)
-	cuid := obj.converger.Register()
-	defer cuid.Unregister()
 
-	var startup bool
+	// notify engine that we're running
-	Startup := func(block bool) <-chan time.Time {
+	if err := obj.Running(processChan); err != nil {
-		if block {
+		return err // bubble up a NACK...
-			return nil // blocks forever
-			//return make(chan time.Time) // blocks forever
-		}
-		return time.After(time.Duration(500) * time.Millisecond) // 1/2 the resolution of converged timeout
 	}
 
 	var send = false // send event?
@@ -125,33 +154,14 @@ func (obj *MsgRes) Watch(processChan chan event.Event) error {
 				return nil // exit
 			}
 
-			/*
-				// TODO: invalidate cached state on poke events
-				obj.logStateOK = false
-				if obj.Journal {
-					obj.journalStateOK = false
-				}
-				if obj.Syslog {
-					obj.syslogStateOK = false
-				}
-			*/
-			send = true
-
 		case <-cuid.ConvergedTimer():
 			cuid.SetConverged(true) // converged!
 			continue
-
-		case <-Startup(startup):
-			cuid.SetConverged(false)
-			send = true
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
-			startup = true // startup finished
 			send = false
-			// only do this on certain types of events
-			//obj.isStateOK = false // something made state dirty
 			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
 				return err // we exit or bubble up a NACK...
 			}
@@ -159,6 +169,51 @@ func (obj *MsgRes) Watch(processChan chan event.Event) error {
 	}
 }
 
+// CheckApply method for Msg resource.
+// Every check leads to an apply, meaning that the message is flushed to the journal.
+func (obj *MsgRes) CheckApply(apply bool) (bool, error) {
+
+	// isStateOK() done by engine, so we updateStateOK() to pass in value
+	//if obj.isAllStateOK() {
+	//	return true, nil
+	//}
+
+	if obj.Refresh() { // if we were notified...
+		// invalidate cached state...
+		obj.logStateOK = false
+		if obj.Journal {
+			obj.journalStateOK = false
+		}
+		if obj.Syslog {
+			obj.syslogStateOK = false
+		}
+		obj.updateStateOK()
+	}
+
+	if !obj.logStateOK {
+		log.Printf("%s[%s]: Body: %s", obj.Kind(), obj.GetName(), obj.Body)
+		obj.logStateOK = true
+		obj.updateStateOK()
+	}
+
+	if !apply {
+		return false, nil
+	}
+	if obj.Journal && !obj.journalStateOK {
+		if err := journal.Send(obj.Body, obj.journalPriority(), obj.Fields); err != nil {
+			return false, err
+		}
+		obj.journalStateOK = true
+		obj.updateStateOK()
+	}
+	if obj.Syslog && !obj.syslogStateOK {
+		// TODO: implement syslog client
+		obj.syslogStateOK = true
+		obj.updateStateOK()
+	}
+	return false, nil
+}
+
 // GetUIDs includes all params to make a unique identification of this object.
 // Most resources only return one, although some resources can return multiple.
 func (obj *MsgRes) GetUIDs() []ResUID {
@@ -204,68 +259,3 @@ func (obj *MsgRes) Compare(res Res) bool {
 	}
 	return true
 }
-
-// IsAllStateOK derives a compound state from all internal cache flags that apply to this resource.
-func (obj *MsgRes) isAllStateOK() bool {
-	if obj.Journal && !obj.journalStateOK {
-		return false
-	}
-	if obj.Syslog && !obj.syslogStateOK {
-		return false
-	}
-	return obj.logStateOK
-}
-
-// JournalPriority converts a string description to a numeric priority.
-// XXX: Have Validate() make sure it actually is one of these.
-func (obj *MsgRes) journalPriority() journal.Priority {
-	switch obj.Priority {
-	case "Emerg":
-		return journal.PriEmerg
-	case "Alert":
-		return journal.PriAlert
-	case "Crit":
-		return journal.PriCrit
-	case "Err":
-		return journal.PriErr
-	case "Warning":
-		return journal.PriWarning
-	case "Notice":
-		return journal.PriNotice
-	case "Info":
-		return journal.PriInfo
-	case "Debug":
-		return journal.PriDebug
-	}
-	return journal.PriNotice
-}
-
-// CheckApply method for Msg resource.
-// Every check leads to an apply, meaning that the message is flushed to the journal.
-func (obj *MsgRes) CheckApply(apply bool) (bool, error) {
-	log.Printf("%s[%s]: CheckApply(%t)", obj.Kind(), obj.GetName(), apply)
-
-	if obj.isAllStateOK() {
-		return true, nil
-	}
-
-	if !obj.logStateOK {
-		log.Printf("%s[%s]: Body: %s", obj.Kind(), obj.GetName(), obj.Body)
-		obj.logStateOK = true
-	}
-
-	if !apply {
-		return false, nil
-	}
-	if obj.Journal && !obj.journalStateOK {
-		if err := journal.Send(obj.Body, obj.journalPriority(), obj.Fields); err != nil {
-			return false, err
-		}
-		obj.journalStateOK = true
-	}
-	if obj.Syslog && !obj.syslogStateOK {
-		// TODO: implement syslog client
-		obj.syslogStateOK = true
-	}
-	return false, nil
-}

resources/noop.go

@@ -20,7 +20,6 @@ package resources
 import (
 	"encoding/gob"
 	"log"
-	"time"
 
 	"github.com/purpleidea/mgmt/event"
 )
@@ -60,21 +59,11 @@ func (obj *NoopRes) Validate() error {
 
 // Watch is the primary listener for this resource and it outputs events.
 func (obj *NoopRes) Watch(processChan chan event.Event) error {
-	if obj.IsWatching() {
+	cuid := obj.Converger() // get the converger uid used to report status
-		return nil // TODO: should this be an error?
-	}
-	obj.SetWatching(true)
-	defer obj.SetWatching(false)
-	cuid := obj.converger.Register()
-	defer cuid.Unregister()
 
-	var startup bool
+	// notify engine that we're running
-	Startup := func(block bool) <-chan time.Time {
+	if err := obj.Running(processChan); err != nil {
-		if block {
+		return err // bubble up a NACK...
-			return nil // blocks forever
-			//return make(chan time.Time) // blocks forever
-		}
-		return time.After(time.Duration(500) * time.Millisecond) // 1/2 the resolution of converged timeout
 	}
 
 	var send = false // send event?
@@ -92,18 +81,11 @@ func (obj *NoopRes) Watch(processChan chan event.Event) error {
 		case <-cuid.ConvergedTimer():
 			cuid.SetConverged(true) // converged!
 			continue
-
-		case <-Startup(startup):
-			cuid.SetConverged(false)
-			send = true
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
-			startup = true // startup finished
 			send = false
-			// only do this on certain types of events
-			//obj.isStateOK = false // something made state dirty
 			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
 				return err // we exit or bubble up a NACK...
 			}
@@ -112,8 +94,10 @@ func (obj *NoopRes) Watch(processChan chan event.Event) error {
 }
 
 // CheckApply method for Noop resource. Does nothing, returns happy!
-func (obj *NoopRes) CheckApply(apply bool) (checkok bool, err error) {
+func (obj *NoopRes) CheckApply(apply bool) (checkOK bool, err error) {
-	log.Printf("%v[%v]: CheckApply(%t)", obj.Kind(), obj.GetName(), apply)
+	if obj.Refresh() {
+		log.Printf("%s[%s]: Received a notification!", obj.Kind(), obj.GetName())
+	}
 	return true, nil // state is always okay
 }
 

resources/nspawn.go

@@ -0,0 +1,306 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"encoding/gob"
+	"errors"
+	"fmt"
+	"log"
+
+	"github.com/purpleidea/mgmt/event"
+	"github.com/purpleidea/mgmt/util"
+
+	systemdUtil "github.com/coreos/go-systemd/util"
+	"github.com/godbus/dbus"
+	errwrap "github.com/pkg/errors"
+	machined "github.com/purpleidea/go-systemd/machine1"
+)
+
+const (
+	running           = "running"
+	stopped           = "stopped"
+	dbusInterface     = "org.freedesktop.machine1.Manager"
+	machineNew        = "org.freedesktop.machine1.Manager.MachineNew"
+	machineRemoved    = "org.freedesktop.machine1.Manager.MachineRemoved"
+	nspawnServiceTmpl = "systemd-nspawn@%s"
+)
+
+func init() {
+	gob.Register(&NspawnRes{})
+}
+
+// NspawnRes is an nspawn container resource
+type NspawnRes struct {
+	BaseRes `yaml:",inline"`
+	State   string `yaml:"state"`
+	// we're using the svc resource to start the machine because that's
+	// what machinectl does. We're not using svc.Watch because then we
+	// would have two watches potentially racing each other and producing
+	// potentially unexpected results. We get everything we need to
+	// monitor the machine state changes from the org.freedesktop.machine1 object.
+	svc *SvcRes
+}
+
+// Init runs some startup code for this resource
+func (obj *NspawnRes) Init() error {
+	var serviceName = fmt.Sprintf(nspawnServiceTmpl, obj.GetName())
+	obj.svc = &SvcRes{}
+	obj.svc.Name = serviceName
+	obj.svc.State = obj.State
+	if err := obj.svc.Init(); err != nil {
+		return err
+	}
+	obj.BaseRes.kind = "Nspawn"
+	return obj.BaseRes.Init()
+}
+
+// NewNspawnRes is the constructor for this resource
+func NewNspawnRes(name string, state string) (*NspawnRes, error) {
+	obj := &NspawnRes{
+		BaseRes: BaseRes{
+			Name: name,
+		},
+		State: state,
+	}
+	return obj, obj.Init()
+}
+
+// Validate params
+func (obj *NspawnRes) Validate() error {
+	validStates := map[string]struct{}{
+		stopped: {},
+		running: {},
+	}
+	if _, exists := validStates[obj.State]; !exists {
+		return fmt.Errorf("Invalid State: %s", obj.State)
+	}
+	return obj.svc.Validate()
+}
+
+// Watch for state changes and sends a message to the bus if there is a change
+func (obj *NspawnRes) Watch(processChan chan event.Event) error {
+	cuid := obj.Converger() // get the converger uid used to report status
+
+	// this resource depends on systemd ensure that it's running
+	if !systemdUtil.IsRunningSystemd() {
+		return fmt.Errorf("Systemd is not running.")
+	}
+
+	// create a private message bus
+	bus, err := util.SystemBusPrivateUsable()
+	if err != nil {
+		return errwrap.Wrapf(err, "Failed to connect to bus")
+	}
+
+	// add a match rule to match messages going through the message bus
+	call := bus.BusObject().Call("org.freedesktop.DBus.AddMatch", 0,
+		fmt.Sprintf("type='signal',interface='%s',eavesdrop='true'",
+			dbusInterface))
+	// <-call.Done
+	if err := call.Err; err != nil {
+		return err
+	}
+	buschan := make(chan *dbus.Signal, 10)
+	bus.Signal(buschan)
+
+	// notify engine that we're running
+	if err := obj.Running(processChan); err != nil {
+		return err // bubble up a NACK...
+	}
+
+	var send = false
+	var exit = false
+
+	for {
+		obj.SetState(ResStateWatching)
+		select {
+		case event := <-buschan:
+			// process org.freedesktop.machine1 events for this resource's name
+			if event.Body[0] == obj.GetName() {
+				log.Printf("%s[%s]: Event received: %v", obj.Kind(), obj.GetName(), event.Name)
+				if event.Name == machineNew {
+					log.Printf("%s[%s]: Machine started", obj.Kind(), obj.GetName())
+				} else if event.Name == machineRemoved {
+					log.Printf("%s[%s]: Machine stopped", obj.Kind(), obj.GetName())
+				} else {
+					return fmt.Errorf("Unknown event: %s", event.Name)
+				}
+				send = true
+				obj.StateOK(false) // dirty
+			}
+
+		case event := <-obj.Events():
+			cuid.SetConverged(false)
+			if exit, send = obj.ReadEvent(&event); exit {
+				return nil // exit
+			}
+
+		case <-cuid.ConvergedTimer():
+			cuid.SetConverged(true) // converged!
+			continue
+		}
+
+		// do all our event sending all together to avoid duplicate msgs
+		if send {
+			send = false
+			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
+				return err // we exit or bubble up a NACK...
+			}
+		}
+	}
+}
+
+// CheckApply is run to check the state and, if apply is true, to apply the
+// necessary changes to reach the desired state. this is run before Watch and
+// again if watch finds a change occurring to the state
+func (obj *NspawnRes) CheckApply(apply bool) (checkOK bool, err error) {
+	// this resource depends on systemd ensure that it's running
+	if !systemdUtil.IsRunningSystemd() {
+		return false, errors.New("Systemd is not running.")
+	}
+
+	// connect to org.freedesktop.machine1.Manager
+	conn, err := machined.New()
+	if err != nil {
+		return false, errwrap.Wrapf(err, "Failed to connect to dbus")
+	}
+
+	// compare the current state with the desired state and perform the
+	// appropriate action
+	var exists = true
+	properties, err := conn.GetProperties(obj.GetName())
+	if err != nil {
+		if err, ok := err.(dbus.Error); ok && err.Name !=
+			"org.freedesktop.machine1.NoSuchMachine" {
+			return false, err
+		}
+		exists = false
+		// if we could not successfully get the properties because
+		// there's no such machine the machine is stopped
+		// error if we need the image ignore if we don't
+		if _, err = conn.GetImage(obj.GetName()); err != nil && obj.State != stopped {
+			return false, fmt.Errorf(
+				"No machine nor image named '%s'",
+				obj.GetName())
+		}
+	}
+	if obj.debug {
+		log.Printf("%s[%s]: properties: %v", obj.Kind(), obj.GetName(), properties)
+	}
+	// if the machine doesn't exist and is supposed to
+	// be stopped or the state matches we're done
+	if !exists && obj.State == stopped || properties["State"] == obj.State {
+		if obj.debug {
+			log.Printf("%s[%s]: CheckApply() in valid state", obj.Kind(), obj.GetName())
+		}
+		return true, nil
+	}
+
+	// end of state checking. if we're here, checkOK is false
+	if !apply {
+		return false, nil
+	}
+
+	if obj.debug {
+		log.Printf("%s[%s]: CheckApply() applying '%s' state", obj.Kind(), obj.GetName(), obj.State)
+	}
+
+	if obj.State == running {
+		// start the machine using svc resource
+		log.Printf("%s[%s]: Starting machine", obj.Kind(), obj.GetName())
+		// assume state had to be changed at this point, ignore checkOK
+		if _, err := obj.svc.CheckApply(apply); err != nil {
+			return false, errwrap.Wrapf(err, "Nested svc failed")
+		}
+	}
+	if obj.State == stopped {
+		// terminate the machine with
+		// org.freedesktop.machine1.Manager.KillMachine
+		log.Printf("%s[%s]: Stopping machine", obj.Kind(), obj.GetName())
+		if err := conn.TerminateMachine(obj.GetName()); err != nil {
+			return false, errwrap.Wrapf(err, "Failed to stop machine")
+		}
+	}
+
+	return false, nil
+}
+
+// NspawnUID is a unique resource identifier
+type NspawnUID struct {
+	// NOTE: there is also a name variable in the BaseUID struct, this is
+	// information about where this UID came from, and is unrelated to the
+	// information about the resource we're matching. That data which is
+	// used in the IFF function, is what you see in the struct fields here
+	BaseUID
+	name string // the machine name
+}
+
+// IFF aka if and only if they are equivalent, return true. If not, false
+func (obj *NspawnUID) IFF(uid ResUID) bool {
+	res, ok := uid.(*NspawnUID)
+	if !ok {
+		return false
+	}
+	return obj.name == res.name
+}
+
+// GetUIDs includes all params to make a unique identification of this object
+// most resources only return one although some resources can return multiple
+func (obj *NspawnRes) GetUIDs() []ResUID {
+	x := &NspawnUID{
+		BaseUID: BaseUID{name: obj.GetName(), kind: obj.Kind()},
+		name:    obj.Name, // svc name
+	}
+	return append([]ResUID{x}, obj.svc.GetUIDs()...)
+}
+
+// GroupCmp returns whether two resources can be grouped together or not
+func (obj *NspawnRes) GroupCmp(r Res) bool {
+	_, ok := r.(*NspawnRes)
+	if !ok {
+		return false
+	}
+	// TODO: this would be quite useful for this resource!
+	return false
+}
+
+// Compare two resources and return if they are equivalent
+func (obj *NspawnRes) Compare(res Res) bool {
+	switch res.(type) {
+	case *NspawnRes:
+		res := res.(*NspawnRes)
+		if !obj.BaseRes.Compare(res) {
+			return false
+		}
+		if obj.Name != res.Name {
+			return false
+		}
+		if !obj.svc.Compare(res.svc) {
+			return false
+		}
+	default:
+		return false
+	}
+	return true
+}
+
+// AutoEdges returns the AutoEdge interface in this case no autoedges are used
+func (obj *NspawnRes) AutoEdges() AutoEdge {
+	return nil
+}

resources/password.go

@@ -0,0 +1,365 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"crypto/rand"
+	"encoding/gob"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"math/big"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/purpleidea/mgmt/event"
+	"github.com/purpleidea/mgmt/recwatch"
+
+	errwrap "github.com/pkg/errors"
+)
+
+func init() {
+	gob.Register(&PasswordRes{})
+}
+
+const (
+	alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+	newline  = "\n" // something not in alphabet that TrimSpace can trim
+)
+
+// PasswordRes is a no-op resource that returns a random password string.
+type PasswordRes struct {
+	BaseRes `yaml:",inline"`
+	// FIXME: is uint16 too big?
+	Length        uint16  `yaml:"length"` // number of characters to return
+	Saved         bool    // this caches the password in the clear locally
+	CheckRecovery bool    // recovery from integrity checks by re-generating
+	Password      *string // the generated password, read only, do not set!
+
+	path       string // the path to local storage
+	recWatcher *recwatch.RecWatcher
+}
+
+// NewPasswordRes is a constructor for this resource. It also calls Init() for you.
+func NewPasswordRes(name string, length uint16) (*PasswordRes, error) {
+	obj := &PasswordRes{
+		BaseRes: BaseRes{
+			Name: name,
+		},
+		Length: length,
+	}
+	return obj, obj.Init()
+}
+
+// Init generates a new password for this resource if one was not provided. It
+// will save this into a local file. It will load it back in from previous runs.
+func (obj *PasswordRes) Init() error {
+	obj.BaseRes.kind = "Password" // must be set before using VarDir
+
+	dir, err := obj.VarDir("")
+	if err != nil {
+		return errwrap.Wrapf(err, "could not get VarDir in Init()")
+	}
+	obj.path = path.Join(dir, "password") // return a unique file
+
+	return obj.BaseRes.Init() // call base init, b/c we're overriding
+}
+
+// Validate if the params passed in are valid data.
+// FIXME: where should this get called ?
+func (obj *PasswordRes) Validate() error {
+	return nil
+}
+
+func (obj *PasswordRes) read() (string, error) {
+	file, err := os.Open(obj.path) // open a handle to read the file
+	if err != nil {
+		return "", err
+	}
+	defer file.Close()
+	data, err := ioutil.ReadAll(file)
+	if err != nil {
+		return "", errwrap.Wrapf(err, "could not read from file")
+	}
+	return strings.TrimSpace(string(data)), nil
+}
+
+func (obj *PasswordRes) write(password string) (int, error) {
+	file, err := os.Create(obj.path) // open a handle to create the file
+	if err != nil {
+		return -1, errwrap.Wrapf(err, "can't create file")
+	}
+	defer file.Close()
+	var c int
+	if c, err = file.Write([]byte(password + newline)); err != nil {
+		return c, errwrap.Wrapf(err, "can't write file")
+	}
+	return c, file.Sync()
+}
+
+// generate generates a new password.
+func (obj *PasswordRes) generate() (string, error) {
+	max := len(alphabet) - 1 // last index
+	output := ""
+
+	// FIXME: have someone verify this is cryptographically secure & correct
+	for i := uint16(0); i < obj.Length; i++ {
+		big, err := rand.Int(rand.Reader, big.NewInt(int64(max)))
+		if err != nil {
+			return "", errwrap.Wrapf(err, "could not generate password")
+		}
+		ix := big.Int64()
+		output += string(alphabet[ix])
+	}
+
+	if output == "" { // safety against empty passwords
+		return "", fmt.Errorf("password is empty")
+	}
+
+	if uint16(len(output)) != obj.Length { // safety against weird bugs
+		return "", fmt.Errorf("password length is too short") // bug!
+	}
+
+	return output, nil
+}
+
+// check validates a stored password string
+func (obj *PasswordRes) check(value string) error {
+	length := uint16(len(value))
+
+	if !obj.Saved && length == 0 { // expecting an empty string
+		return nil
+	}
+	if !obj.Saved && length != 0 { // should have no stored password
+		return fmt.Errorf("Expected empty token only!")
+	}
+
+	if length != obj.Length {
+		return fmt.Errorf("String length is not %d", obj.Length)
+	}
+Loop:
+	for i := uint16(0); i < length; i++ {
+		for j := 0; j < len(alphabet); j++ {
+			if value[i] == alphabet[j] {
+				continue Loop
+			}
+		}
+		// we couldn't find that character, so error!
+		return fmt.Errorf("Invalid character `%s`", string(value[i]))
+	}
+	return nil
+}
+
+// Watch is the primary listener for this resource and it outputs events.
+func (obj *PasswordRes) Watch(processChan chan event.Event) error {
+	cuid := obj.Converger() // get the converger uid used to report status
+
+	var err error
+	obj.recWatcher, err = recwatch.NewRecWatcher(obj.path, false)
+	if err != nil {
+		return err
+	}
+	defer obj.recWatcher.Close()
+
+	// notify engine that we're running
+	if err := obj.Running(processChan); err != nil {
+		return err // bubble up a NACK...
+	}
+
+	var send = false // send event?
+	var exit = false
+	for {
+		obj.SetState(ResStateWatching) // reset
+		select {
+		// NOTE: this part is very similar to the file resource code
+		case event, ok := <-obj.recWatcher.Events():
+			if !ok { // channel shutdown
+				return nil
+			}
+			cuid.SetConverged(false)
+			if err := event.Error; err != nil {
+				return errwrap.Wrapf(err, "Unknown %s[%s] watcher error", obj.Kind(), obj.GetName())
+			}
+			send = true
+			obj.StateOK(false) // dirty
+
+		case event := <-obj.Events():
+			cuid.SetConverged(false)
+			// we avoid sending events on unpause
+			if exit, send = obj.ReadEvent(&event); exit {
+				return nil // exit
+			}
+
+		case <-cuid.ConvergedTimer():
+			cuid.SetConverged(true) // converged!
+			continue
+		}
+
+		// do all our event sending all together to avoid duplicate msgs
+		if send {
+			send = false
+			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
+				return err // we exit or bubble up a NACK...
+			}
+		}
+	}
+}
+
+// CheckApply method for Password resource. Does nothing, returns happy!
+func (obj *PasswordRes) CheckApply(apply bool) (checkOK bool, err error) {
+
+	var refresh = obj.Refresh() // do we have a pending reload to apply?
+	var exists = true           // does the file (aka the token) exist?
+	var generate bool           // do we need to generate a new password?
+	var write bool              // do we need to write out to disk?
+
+	password, err := obj.read() // password might be empty if just a token
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return false, errwrap.Wrapf(err, "unknown read error")
+		}
+		exists = false
+	}
+
+	if exists {
+		if err := obj.check(password); err != nil {
+			if !obj.CheckRecovery {
+				return false, errwrap.Wrapf(err, "check failed")
+			}
+			log.Printf("%s[%s]: Integrity check failed", obj.Kind(), obj.GetName())
+			generate = true // okay to build a new one
+			write = true    // make sure to write over the old one
+		}
+	} else { // doesn't exist, write one
+		write = true
+	}
+
+	// if we previously had !obj.Saved, and now we want it, we re-generate!
+	if refresh || !exists || (obj.Saved && password == "") {
+		generate = true
+	}
+
+	// stored password isn't consistent with memory
+	if p := obj.Password; obj.Saved && (p != nil && *p != password) {
+		write = true
+	}
+
+	if !refresh && exists && !generate && !write { // nothing to do, done!
+		return true, nil
+	}
+	// a refresh was requested, the token doesn't exist, or the check failed
+
+	if !apply {
+		return false, nil
+	}
+
+	if generate {
+		// we'll need to write this out...
+		if obj.Saved || (!obj.Saved && password != "") {
+			write = true
+		}
+		// generate the actual password
+		var err error
+		log.Printf("%s[%s]: Generating new password...", obj.Kind(), obj.GetName())
+		if password, err = obj.generate(); err != nil { // generate one!
+			return false, errwrap.Wrapf(err, "could not generate password")
+		}
+	}
+
+	obj.Password = &password // save in memory
+
+	var output string // the string to write out
+
+	// if memory value != value on disk, save it
+	if write {
+		if obj.Saved { // save password as clear text
+			// TODO: would it make sense to encrypt this password?
+			output = password
+		}
+		// write either an empty token, or the password
+		log.Printf("%s[%s]: Writing password token...", obj.Kind(), obj.GetName())
+		if _, err := obj.write(output); err != nil {
+			return false, errwrap.Wrapf(err, "can't write to file")
+		}
+	}
+
+	return false, nil
+}
+
+// PasswordUID is the UID struct for PasswordRes.
+type PasswordUID struct {
+	BaseUID
+	name string
+}
+
+// AutoEdges returns the AutoEdge interface. In this case no autoedges are used.
+func (obj *PasswordRes) AutoEdges() AutoEdge {
+	return nil
+}
+
+// GetUIDs includes all params to make a unique identification of this object.
+// Most resources only return one, although some resources can return multiple.
+func (obj *PasswordRes) GetUIDs() []ResUID {
+	x := &PasswordUID{
+		BaseUID: BaseUID{name: obj.GetName(), kind: obj.Kind()},
+		name:    obj.Name,
+	}
+	return []ResUID{x}
+}
+
+// GroupCmp returns whether two resources can be grouped together or not.
+func (obj *PasswordRes) GroupCmp(r Res) bool {
+	_, ok := r.(*PasswordRes)
+	if !ok {
+		return false
+	}
+	return false // TODO: this is doable, but probably not very useful
+	// TODO: it could be useful to group our tokens into a single write, and
+	// as a result, we save inotify watches too!
+}
+
+// Compare two resources and return if they are equivalent.
+func (obj *PasswordRes) Compare(res Res) bool {
+	switch res.(type) {
+	// we can only compare PasswordRes to others of the same resource
+	case *PasswordRes:
+		res := res.(*PasswordRes)
+		if !obj.BaseRes.Compare(res) { // call base Compare
+			return false
+		}
+
+		if obj.Name != res.Name {
+			return false
+		}
+		if obj.Length != res.Length {
+			return false
+		}
+		// TODO: we *could* optimize by allowing CheckApply to move from
+		// saved->!saved, by removing the file, but not likely worth it!
+		if obj.Saved != res.Saved {
+			return false
+		}
+		if obj.CheckRecovery != res.CheckRecovery {
+			return false
+		}
+	default:
+		return false
+	}
+	return true
+}

resources/pkg.go

@@ -19,17 +19,16 @@ package resources
 
 import (
 	"encoding/gob"
-	"errors"
 	"fmt"
 	"log"
 	"path"
 	"strings"
-	"time"
 
 	"github.com/purpleidea/mgmt/event"
-	"github.com/purpleidea/mgmt/global" // XXX: package mgmtmain instead?
 	"github.com/purpleidea/mgmt/resources/packagekit"
 	"github.com/purpleidea/mgmt/util"
+
+	errwrap "github.com/pkg/errors"
 )
 
 func init() {
@@ -76,7 +75,7 @@ func (obj *PkgRes) Init() error {
 
 	result, err := obj.pkgMappingHelper(bus)
 	if err != nil {
-		return fmt.Errorf("The pkgMappingHelper failed with: %v.", err)
+		return errwrap.Wrapf(err, "The pkgMappingHelper failed")
 	}
 
 	data, ok := result[obj.Name] // lookup single package (init does just one)
@@ -88,7 +87,7 @@ func (obj *PkgRes) Init() error {
 	packageIDs := []string{data.PackageID} // just one for now
 	filesMap, err := bus.GetFilesByPackageID(packageIDs)
 	if err != nil {
-		return fmt.Errorf("Can't run GetFilesByPackageID: %v", err)
+		return errwrap.Wrapf(err, "Can't run GetFilesByPackageID")
 	}
 	if files, ok := filesMap[data.PackageID]; ok {
 		obj.fileList = util.DirifyFileList(files, false)
@@ -110,41 +109,30 @@ func (obj *PkgRes) Validate() error {
 // TODO: https://github.com/hughsie/PackageKit/issues/109
 // TODO: https://github.com/hughsie/PackageKit/issues/110
 func (obj *PkgRes) Watch(processChan chan event.Event) error {
-	if obj.IsWatching() {
+	cuid := obj.Converger() // get the converger uid used to report status
-		return nil
-	}
-	obj.SetWatching(true)
-	defer obj.SetWatching(false)
-	cuid := obj.converger.Register()
-	defer cuid.Unregister()
-
-	var startup bool
-	Startup := func(block bool) <-chan time.Time {
-		if block {
-			return nil // blocks forever
-			//return make(chan time.Time) // blocks forever
-		}
-		return time.After(time.Duration(500) * time.Millisecond) // 1/2 the resolution of converged timeout
-	}
 
 	bus := packagekit.NewBus()
 	if bus == nil {
-		log.Fatal("Can't connect to PackageKit bus.")
+		return fmt.Errorf("Can't connect to PackageKit bus.")
 	}
 	defer bus.Close()
 
 	ch, err := bus.WatchChanges()
 	if err != nil {
-		log.Fatalf("Error adding signal match: %v", err)
+		return errwrap.Wrapf(err, "Error adding signal match")
+	}
+
+	// notify engine that we're running
+	if err := obj.Running(processChan); err != nil {
+		return err // bubble up a NACK...
 	}
 
 	var send = false // send event?
 	var exit = false
-	var dirty = false
 
 	for {
-		if global.DEBUG {
+		if obj.debug {
-			log.Printf("%v: Watching...", obj.fmtNames(obj.getNames()))
+			log.Printf("%s: Watching...", obj.fmtNames(obj.getNames()))
 		}
 
 		obj.SetState(ResStateWatching) // reset
@@ -153,8 +141,8 @@ func (obj *PkgRes) Watch(processChan chan event.Event) error {
 			cuid.SetConverged(false)
 
 			// FIXME: ask packagekit for info on what packages changed
-			if global.DEBUG {
+			if obj.debug {
-				log.Printf("%v: Event: %v", obj.fmtNames(obj.getNames()), event.Name)
+				log.Printf("%s: Event: %v", obj.fmtNames(obj.getNames()), event.Name)
 			}
 
 			// since the chan is buffered, remove any supplemental
@@ -164,34 +152,23 @@ func (obj *PkgRes) Watch(processChan chan event.Event) error {
 			}
 
 			send = true
-			dirty = true
+			obj.StateOK(false) // dirty
 
 		case event := <-obj.Events():
 			cuid.SetConverged(false)
 			if exit, send = obj.ReadEvent(&event); exit {
 				return nil // exit
 			}
-			dirty = false // these events don't invalidate state
+			//obj.StateOK(false) // these events don't invalidate state
 
 		case <-cuid.ConvergedTimer():
 			cuid.SetConverged(true) // converged!
 			continue
-
-		case <-Startup(startup):
-			cuid.SetConverged(false)
-			send = true
-			dirty = true
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
-			startup = true // startup finished
 			send = false
-			// only invalid state on certain types of events
-			if dirty {
-				dirty = false
-				obj.isStateOK = false // something made state dirty
-			}
 			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
 				return err // we exit or bubble up a NACK...
 			}
@@ -217,9 +194,9 @@ func (obj *PkgRes) getNames() []string {
 // pretty print for header values
 func (obj *PkgRes) fmtNames(names []string) string {
 	if len(obj.GetGroup()) > 0 { // grouped elements
-		return fmt.Sprintf("%v[autogroup:(%v)]", obj.Kind(), strings.Join(names, ","))
+		return fmt.Sprintf("%s[autogroup:(%v)]", obj.Kind(), strings.Join(names, ","))
 	}
-	return fmt.Sprintf("%v[%v]", obj.Kind(), obj.GetName())
+	return fmt.Sprintf("%s[%s]", obj.Kind(), obj.GetName())
 }
 
 func (obj *PkgRes) groupMappingHelper() map[string]string {
@@ -228,7 +205,7 @@ func (obj *PkgRes) groupMappingHelper() map[string]string {
 		for _, x := range g {
 			pkg, ok := x.(*PkgRes) // convert from Res
 			if !ok {
-				log.Fatalf("Grouped member %v is not a %v", x, obj.Kind())
+				log.Fatalf("Grouped member %v is not a %s", x, obj.Kind())
 			}
 			result[pkg.Name] = pkg.State
 		}
@@ -254,35 +231,27 @@ func (obj *PkgRes) pkgMappingHelper(bus *packagekit.Conn) (map[string]*packageki
 	if !obj.AllowUnsupported {
 		filter += packagekit.PK_FILTER_ENUM_SUPPORTED
 	}
-	result, e := bus.PackagesToPackageIDs(packageMap, filter)
+	result, err := bus.PackagesToPackageIDs(packageMap, filter)
-	if e != nil {
+	if err != nil {
-		return nil, fmt.Errorf("Can't run PackagesToPackageIDs: %v", e)
+		return nil, errwrap.Wrapf(err, "Can't run PackagesToPackageIDs")
 	}
 	return result, nil
 }
 
 // CheckApply checks the resource state and applies the resource if the bool
 // input is true. It returns error info and if the state check passed or not.
-func (obj *PkgRes) CheckApply(apply bool) (checkok bool, err error) {
+func (obj *PkgRes) CheckApply(apply bool) (checkOK bool, err error) {
-	log.Printf("%v: CheckApply(%t)", obj.fmtNames(obj.getNames()), apply)
+	log.Printf("%s: Check", obj.fmtNames(obj.getNames()))
-
-	if obj.State == "" { // TODO: Validate() should replace this check!
-		log.Fatalf("%v: Package state is undefined!", obj.fmtNames(obj.getNames()))
-	}
-
-	if obj.isStateOK { // cache the state
-		return true, nil
-	}
 
 	bus := packagekit.NewBus()
 	if bus == nil {
-		return false, errors.New("Can't connect to PackageKit bus.")
+		return false, fmt.Errorf("Can't connect to PackageKit bus.")
 	}
 	defer bus.Close()
 
 	result, err := obj.pkgMappingHelper(bus)
 	if err != nil {
-		return false, fmt.Errorf("The pkgMappingHelper failed with: %v.", err)
+		return false, errwrap.Wrapf(err, "The pkgMappingHelper failed")
 	}
 
 	packageMap := obj.groupMappingHelper() // map[string]string
@@ -295,7 +264,7 @@ func (obj *PkgRes) CheckApply(apply bool) (checkok bool, err error) {
 	// eventually we might be able to drop this constraint!
 	states, err := packagekit.FilterState(result, packageList, obj.State)
 	if err != nil {
-		return false, fmt.Errorf("The FilterState method failed with: %v.", err)
+		return false, errwrap.Wrapf(err, "The FilterState method failed")
 	}
 	data, _ := result[obj.Name] // if above didn't error, we won't either!
 	validState := util.BoolMapTrue(util.BoolMapValues(states))
@@ -308,12 +277,10 @@ func (obj *PkgRes) CheckApply(apply bool) (checkok bool, err error) {
 		fallthrough
 	case "newest":
 		if validState {
-			obj.isStateOK = true // reset
+			return true, nil // state is correct, exit!
-			return true, nil     // state is correct, exit!
 		}
 	default: // version string
 		if obj.State == data.Version && data.Version != "" {
-			obj.isStateOK = true // reset
 			return true, nil
 		}
 	}
@@ -324,7 +291,7 @@ func (obj *PkgRes) CheckApply(apply bool) (checkok bool, err error) {
 	}
 
 	// apply portion
-	log.Printf("%v: Apply", obj.fmtNames(obj.getNames()))
+	log.Printf("%s: Apply", obj.fmtNames(obj.getNames()))
 	readyPackages, err := packagekit.FilterPackageState(result, packageList, obj.State)
 	if err != nil {
 		return false, err // fail
@@ -338,7 +305,7 @@ func (obj *PkgRes) CheckApply(apply bool) (checkok bool, err error) {
 		transactionFlags += packagekit.PK_TRANSACTION_FLAG_ENUM_ONLY_TRUSTED
 	}
 	// apply correct state!
-	log.Printf("%v: Set: %v...", obj.fmtNames(util.StrListIntersection(applyPackages, obj.getNames())), obj.State)
+	log.Printf("%s: Set: %v...", obj.fmtNames(util.StrListIntersection(applyPackages, obj.getNames())), obj.State)
 	switch obj.State {
 	case "uninstalled": // run remove
 		// NOTE: packageID is different than when installed, because now
@@ -356,9 +323,8 @@ func (obj *PkgRes) CheckApply(apply bool) (checkok bool, err error) {
 	if err != nil {
 		return false, err // fail
 	}
-	log.Printf("%v: Set: %v success!", obj.fmtNames(util.StrListIntersection(applyPackages, obj.getNames())), obj.State)
+	log.Printf("%s: Set: %v success!", obj.fmtNames(util.StrListIntersection(applyPackages, obj.getNames())), obj.State)
-	obj.isStateOK = true // reset
+	return false, nil // success
-	return false, nil    // success
 }
 
 // PkgUID is the UID struct for PkgRes.

resources/refresh.go

@@ -0,0 +1,104 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strings"
+
+	errwrap "github.com/pkg/errors"
+)
+
+// Refresh returns the pending state of a notification. It should only be called
+// in the CheckApply portion of a resource where a refresh should be acted upon.
+func (obj *BaseRes) Refresh() bool {
+	return obj.refresh
+}
+
+// SetRefresh sets the pending state of a notification. It should only be called
+// by the mgmt engine.
+func (obj *BaseRes) SetRefresh(b bool) {
+	obj.refresh = b
+}
+
+// StatefulBool is an interface for storing a boolean flag in a permanent spot.
+type StatefulBool interface {
+	Get() (bool, error) // get value of token
+	Set() error         // set token to true
+	Del() error         // rm token if it exists
+}
+
+// DiskBool stores a boolean variable on disk for stateful access across runs.
+// The absence of the path is treated as false. If the path contains a special
+// value, then it is treated as true. All the other non-error cases are false.
+type DiskBool struct {
+	Path string // path to token
+}
+
+// str returns the string data which represents true (aka set).
+func (obj *DiskBool) str() string {
+	const TrueToken = "true"
+	const newline = "\n"
+	return TrueToken + newline
+}
+
+// Get returns if the boolean setting, if no error reading the value occurs.
+func (obj *DiskBool) Get() (bool, error) {
+	file, err := os.Open(obj.Path) // open a handle to read the file
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil // no token means value is false
+		}
+		return false, errwrap.Wrapf(err, "could not read token")
+	}
+	defer file.Close()
+	data, err := ioutil.ReadAll(file)
+	if err != nil {
+		return false, errwrap.Wrapf(err, "could not read from file")
+	}
+	return strings.TrimSpace(string(data)) == strings.TrimSpace(obj.str()), nil
+}
+
+// Set stores the true boolean value, if no error setting the value occurs.
+func (obj *DiskBool) Set() error {
+	file, err := os.Create(obj.Path) // open a handle to create the file
+	if err != nil {
+		return errwrap.Wrapf(err, "can't create file")
+	}
+	defer file.Close()
+	str := obj.str()
+	if c, err := file.Write([]byte(str)); err != nil {
+		return errwrap.Wrapf(err, "error writing to file")
+	} else if l := len(str); c != l {
+		return fmt.Errorf("wrote %d bytes instead of %d", c, l)
+	}
+	return file.Sync() // guarantee it!
+}
+
+// Del stores the false boolean value, if no error clearing the value occurs.
+func (obj *DiskBool) Del() error {
+	if err := os.Remove(obj.Path); err != nil { // remove the file
+		if os.IsNotExist(err) {
+			return nil // no file means this is already fine
+		}
+		return errwrap.Wrapf(err, "could not delete token")
+	}
+	return nil
+}

resources/resources.go

@@ -24,11 +24,14 @@ import (
 	"encoding/gob"
 	"fmt"
 	"log"
+	"os"
+	"path"
 
 	// TODO: should each resource be a sub-package?
 	"github.com/purpleidea/mgmt/converger"
 	"github.com/purpleidea/mgmt/event"
-	"github.com/purpleidea/mgmt/global"
+
+	errwrap "github.com/pkg/errors"
 )
 
 //go:generate stringer -type=ResState -output=resstate_stringer.go
@@ -45,6 +48,18 @@ const (
 	ResStatePoking
 )
 
+const refreshPathToken = "refresh"
+
+// Data is the set of input values passed into the pgraph for the resources.
+type Data struct {
+	//Hostname string         // uuid for the host
+	//Noop     bool
+	Converger converger.Converger
+	Prefix    string // the prefix to be used for the pgraph namespace
+	Debug     bool
+	// NOTE: we can add more fields here if needed for the resources.
+}
+
 // ResUID is a unique identifier for a resource, namely it's name, and the kind ("type").
 type ResUID interface {
 	GetName() string
@@ -112,20 +127,32 @@ type Base interface {
 	Kind() string
 	Meta() *MetaParams
 	Events() chan event.Event
-	AssociateData(converger.Converger)
+	AssociateData(*Data)
 	IsWatching() bool
 	SetWatching(bool)
+	RegisterConverger()
+	UnregisterConverger()
+	Converger() converger.ConvergerUID
 	GetState() ResState
 	SetState(ResState)
 	DoSend(chan event.Event, string) (bool, error)
 	SendEvent(event.EventName, bool, bool) bool
-	ReadEvent(*event.Event) (bool, bool) // TODO: optional here?
+	ReadEvent(*event.Event) (bool, bool)   // TODO: optional here?
-	GroupCmp(Res) bool                   // TODO: is there a better name for this?
+	Refresh() bool                         // is there a pending refresh to run?
-	GroupRes(Res) error                  // group resource (arg) into self
+	SetRefresh(bool)                       // set the refresh state of this resource
-	IsGrouped() bool                     // am I grouped?
+	SendRecv(Res) (map[string]bool, error) // send->recv data passing function
-	SetGrouped(bool)                     // set grouped bool
+	IsStateOK() bool
-	GetGroup() []Res                     // return everyone grouped inside me
+	StateOK(b bool)
+	GroupCmp(Res) bool  // TODO: is there a better name for this?
+	GroupRes(Res) error // group resource (arg) into self
+	IsGrouped() bool    // am I grouped?
+	SetGrouped(bool)    // set grouped bool
+	GetGroup() []Res    // return everyone grouped inside me
 	SetGroup([]Res)
+	VarDir(string) (string, error)
+	Running(chan event.Event) error // notify the engine that Watch started
+	Started() <-chan struct{}       // returns when the resource has started
+	Starter(bool)
 }
 
 // Res is the minimum interface you need to implement to define a new resource.
@@ -143,16 +170,25 @@ type Res interface {
 
 // BaseRes is the base struct that gets used in every resource.
 type BaseRes struct {
-	Name       string     `yaml:"name"`
+	Name       string           `yaml:"name"`
-	MetaParams MetaParams `yaml:"meta"` // struct of all the metaparams
+	MetaParams MetaParams       `yaml:"meta"` // struct of all the metaparams
-	kind       string
+	Recv       map[string]*Send // mapping of key to receive on from value
-	events     chan event.Event
+
-	converger  converger.Converger // converged tracking
+	kind      string
-	state      ResState
+	events    chan event.Event
-	watching   bool  // is Watch() loop running ?
+	converger converger.Converger // converged tracking
-	isStateOK  bool  // whether the state is okay based on events or not
+	cuid      converger.ConvergerUID
-	isGrouped  bool  // am i contained within a group?
+	prefix    string // base prefix for this resource
-	grouped    []Res // list of any grouped resources
+	debug     bool
+	state     ResState
+	watching  bool          // is Watch() loop running ?
+	started   chan struct{} // closed when worker is started/running
+	starter   bool          // does this have indegree == 0 ? XXX: usually?
+	isStateOK bool          // whether the state is okay based on events or not
+	isGrouped bool          // am i contained within a group?
+	grouped   []Res         // list of any grouped resources
+	refresh   bool          // does this resource have a refresh to run?
+	//refreshState StatefulBool // TODO: future stateful bool
 }
 
 // UIDExistsInUIDs wraps the IFF method when used with a list of UID's.
@@ -201,7 +237,14 @@ func (obj *BaseRes) Init() error {
 	if obj.kind == "" {
 		return fmt.Errorf("Resource did not set kind!")
 	}
-	obj.events = make(chan event.Event) // unbuffered chan size to avoid stale events
+	obj.events = make(chan event.Event) // unbuffered chan to avoid stale events
+	obj.started = make(chan struct{})   // closes when started
+	//dir, err := obj.VarDir("")
+	//if err != nil {
+	//	return errwrap.Wrapf(err, "VarDir failed in Init()")
+	//}
+	// TODO: this StatefulBool implementation could be eventually swappable
+	//obj.refreshState = &DiskBool{Path: path.Join(dir, refreshPathToken)}
 	return nil
 }
 
@@ -236,20 +279,40 @@ func (obj *BaseRes) Events() chan event.Event {
 }
 
 // AssociateData associates some data with the object in question.
-func (obj *BaseRes) AssociateData(converger converger.Converger) {
+func (obj *BaseRes) AssociateData(data *Data) {
-	obj.converger = converger
+	obj.converger = data.Converger
+	obj.prefix = data.Prefix
+	obj.debug = data.Debug
 }
 
-// IsWatching tells us if the Watch() function is running.
+// IsWatching tells us if the Worker() function is running.
 func (obj *BaseRes) IsWatching() bool {
 	return obj.watching
 }
 
-// SetWatching stores the status of if the Watch() function is running.
+// SetWatching stores the status of if the Worker() function is running.
 func (obj *BaseRes) SetWatching(b bool) {
 	obj.watching = b
 }
 
+// RegisterConverger sets up the cuid for the resource. This is a helper
+// function for the engine, and shouldn't be called by the resources directly.
+func (obj *BaseRes) RegisterConverger() {
+	obj.cuid = obj.converger.Register()
+}
+
+// UnregisterConverger tears down the cuid for the resource. This is a helper
+// function for the engine, and shouldn't be called by the resources directly.
+func (obj *BaseRes) UnregisterConverger() {
+	obj.cuid.Unregister()
+}
+
+// Converger returns the ConvergerUID for the resource. This should be called
+// by the Watch method of the resource to set the converged state.
+func (obj *BaseRes) Converger() converger.ConvergerUID {
+	return obj.cuid
+}
+
 // GetState returns the state of the resource.
 func (obj *BaseRes) GetState() ResState {
 	return obj.state
@@ -257,91 +320,20 @@ func (obj *BaseRes) GetState() ResState {
 
 // SetState sets the state of the resource.
 func (obj *BaseRes) SetState(state ResState) {
-	if global.DEBUG {
+	if obj.debug {
-		log.Printf("%v[%v]: State: %v -> %v", obj.Kind(), obj.GetName(), obj.GetState(), state)
+		log.Printf("%s[%s]: State: %v -> %v", obj.Kind(), obj.GetName(), obj.GetState(), state)
 	}
 	obj.state = state
 }
 
-// DoSend sends off an event, but doesn't block the incoming event queue. It can
+// IsStateOK returns the cached state value.
-// also recursively call itself when events need processing during the wait.
+func (obj *BaseRes) IsStateOK() bool {
-// I'm not completely comfortable with this fn, but it will have to do for now.
+	return obj.isStateOK
-func (obj *BaseRes) DoSend(processChan chan event.Event, comment string) (bool, error) {
-	resp := event.NewResp()
-	processChan <- event.Event{Name: event.EventNil, Resp: resp, Msg: comment, Activity: true} // trigger process
-	e := resp.Wait()
-	return false, e // XXX: at the moment, we don't use the exit bool.
-	// XXX: this can cause a deadlock. do we need to recursively send? fix event stuff!
-	//select {
-	//case e := <-resp: // wait for the ACK()
-	//	if e != nil { // we got a NACK
-	//		return true, e // exit with error
-	//	}
-	//case event := <-obj.events:
-	//	// NOTE: this code should match the similar code below!
-	//	//cuid.SetConverged(false) // TODO: ?
-	//	if exit, send := obj.ReadEvent(&event); exit {
-	//		return true, nil // exit, without error
-	//	} else if send {
-	//		return obj.DoSend(processChan, comment) // recurse
-	//	}
-	//}
-	//return false, nil // return, no error or exit signal
 }
 
-// SendEvent pushes an event into the message queue for a particular vertex
+// StateOK sets the cached state value.
-func (obj *BaseRes) SendEvent(ev event.EventName, sync bool, activity bool) bool {
+func (obj *BaseRes) StateOK(b bool) {
-	// TODO: isn't this race-y ?
+	obj.isStateOK = b
-	if !obj.IsWatching() { // element has already exited
-		return false // if we don't return, we'll block on the send
-	}
-	if !sync {
-		obj.events <- event.Event{Name: ev, Resp: nil, Msg: "", Activity: activity}
-		return true
-	}
-
-	resp := event.NewResp()
-	obj.events <- event.Event{Name: ev, Resp: resp, Msg: "", Activity: activity}
-	resp.ACKWait() // waits until true (nil) value
-	return true
-}
-
-// ReadEvent processes events when a select gets one, and handles the pause
-// code too! The return values specify if we should exit and poke respectively.
-func (obj *BaseRes) ReadEvent(ev *event.Event) (exit, poke bool) {
-	ev.ACK()
-	switch ev.Name {
-	case event.EventStart:
-		return false, true
-
-	case event.EventPoke:
-		return false, true
-
-	case event.EventBackPoke:
-		return false, true // forward poking in response to a back poke!
-
-	case event.EventExit:
-		return true, false
-
-	case event.EventPause:
-		// wait for next event to continue
-		select {
-		case e := <-obj.Events():
-			e.ACK()
-			if e.Name == event.EventExit {
-				return true, false
-			} else if e.Name == event.EventStart { // eventContinue
-				return false, false // don't poke on unpause!
-			} else {
-				// if we get a poke event here, it's a bug!
-				log.Fatalf("%v[%v]: Unknown event: %v, while paused!", obj.Kind(), obj.GetName(), e)
-			}
-		}
-
-	default:
-		log.Fatal("Unknown event: ", ev)
-	}
-	return true, false // required to keep the stupid go compiler happy
 }
 
 // GroupCmp compares two resources and decides if they're suitable for grouping
@@ -384,7 +376,7 @@ func (obj *BaseRes) SetGroup(g []Res) {
 	obj.grouped = g
 }
 
-// Compare is the base compare method, which also handles the metaparams cmp
+// Compare is the base compare method, which also handles the metaparams cmp.
 func (obj *BaseRes) Compare(res Res) bool {
 	// TODO: should the AutoEdge values be compared?
 	if obj.Meta().AutoEdge != res.Meta().AutoEdge {
@@ -415,6 +407,37 @@ func (obj *BaseRes) CollectPattern(pattern string) {
 	// XXX: default method is empty
 }
 
+// VarDir returns the path to a working directory for the resource. It will try
+// and create the directory first, and return an error if this failed.
+func (obj *BaseRes) VarDir(extra string) (string, error) {
+	// Using extra adds additional dirs onto our namespace. An empty extra
+	// adds no additional directories.
+	if obj.prefix == "" {
+		return "", fmt.Errorf("VarDir prefix is empty!")
+	}
+	if obj.Kind() == "" {
+		return "", fmt.Errorf("VarDir kind is empty!")
+	}
+	if obj.GetName() == "" {
+		return "", fmt.Errorf("VarDir name is empty!")
+	}
+
+	// FIXME: is obj.GetName() sufficiently unique to use as a UID here?
+	uid := obj.GetName()
+	p := fmt.Sprintf("%s/", path.Join(obj.prefix, obj.Kind(), uid, extra))
+	if err := os.MkdirAll(p, 0770); err != nil {
+		return "", errwrap.Wrapf(err, "Can't create prefix for %s[%s]", obj.Kind(), obj.GetName())
+	}
+	return p, nil
+}
+
+// Started returns a channel that closes when the resource has started up.
+func (obj *BaseRes) Started() <-chan struct{} { return obj.started }
+
+// Starter sets the starter bool. This defines if a vertex has an indegree of 0.
+// If we have an indegree of 0, we'll need to be a poke initiator in the graph.
+func (obj *BaseRes) Starter(b bool) { obj.starter = b }
+
 // ResToB64 encodes a resource to a base64 encoded string (after serialization)
 func ResToB64(res Res) (string, error) {
 	b := bytes.Buffer{}

resources/sendrecv.go

@@ -0,0 +1,216 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"fmt"
+	"log"
+	"reflect"
+
+	"github.com/purpleidea/mgmt/event"
+
+	multierr "github.com/hashicorp/go-multierror"
+	errwrap "github.com/pkg/errors"
+)
+
+// SendEvent pushes an event into the message queue for a particular vertex
+func (obj *BaseRes) SendEvent(ev event.EventName, sync bool, activity bool) bool {
+	// TODO: isn't this race-y ?
+	if !obj.IsWatching() { // element has already exited
+		return false // if we don't return, we'll block on the send
+	}
+	if !sync {
+		obj.events <- event.Event{Name: ev, Resp: nil, Msg: "", Activity: activity}
+		return true
+	}
+
+	resp := event.NewResp()
+	obj.events <- event.Event{Name: ev, Resp: resp, Msg: "", Activity: activity}
+	resp.ACKWait() // waits until true (nil) value
+	return true
+}
+
+// DoSend sends off an event, but doesn't block the incoming event queue.
+func (obj *BaseRes) DoSend(processChan chan event.Event, comment string) (exit bool, err error) {
+	resp := event.NewResp()
+	processChan <- event.Event{Name: event.EventNil, Resp: resp, Activity: false, Msg: comment} // trigger process
+	e := resp.Wait()
+	return false, e // XXX: at the moment, we don't use the exit bool.
+}
+
+// ReadEvent processes events when a select gets one, and handles the pause
+// code too! The return values specify if we should exit and poke respectively.
+func (obj *BaseRes) ReadEvent(ev *event.Event) (exit, send bool) {
+	ev.ACK()
+	var poke bool
+	// ensure that a CheckApply runs by sending with a dirty state...
+	if ev.GetActivity() { // if previous node did work, and we were notified...
+		//obj.StateOK(false) // not necessarily
+		poke = true // poke!
+		// XXX: this should be elsewhere in case Watch isn't used (eg: Polling instead...)
+		// XXX: unless this is used in our "fallback" polling implementation???
+		//obj.SetRefresh(true) // TODO: is this redundant?
+	}
+
+	switch ev.Name {
+	case event.EventStart:
+		send = true || poke
+		return
+
+	case event.EventPoke:
+		send = true || poke
+		return
+
+	case event.EventBackPoke:
+		send = true || poke
+		return // forward poking in response to a back poke!
+
+	case event.EventExit:
+		// FIXME: what do we do if we have a pending refresh (poke) and an exit?
+		return true, false
+
+	case event.EventPause:
+		// wait for next event to continue
+		select {
+		case e, ok := <-obj.Events():
+			if !ok { // shutdown
+				return true, false
+			}
+			e.ACK()
+			if e.Name == event.EventExit {
+				return true, false
+			} else if e.Name == event.EventStart { // eventContinue
+				return false, false // don't poke on unpause!
+			} else {
+				// if we get a poke event here, it's a bug!
+				log.Fatalf("%s[%s]: Unknown event: %v, while paused!", obj.Kind(), obj.GetName(), e)
+			}
+		}
+
+	default:
+		log.Fatal("Unknown event: ", ev)
+	}
+	return true, false // required to keep the stupid go compiler happy
+}
+
+// Running is called by the Watch method of the resource once it has started up.
+// This signals to the engine to kick off the initial CheckApply resource check.
+func (obj *BaseRes) Running(processChan chan event.Event) error {
+	obj.StateOK(false)       // assume we're initially dirty
+	cuid := obj.Converger()  // get the converger uid used to report status
+	cuid.SetConverged(false) // a reasonable initial assumption
+	close(obj.started)       // send started signal
+
+	// FIXME: exit return value is unused atm, so ignore it for now...
+	//if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
+	var err error
+	if obj.starter { // vertices of indegree == 0 should send initial pokes
+		_, err = obj.DoSend(processChan, "") // trigger a CheckApply
+	}
+	return err // bubble up any possible error (or nil)
+}
+
+// Send points to a value that a resource will send.
+type Send struct {
+	Res Res    // a handle to the resource which is sending a value
+	Key string // the key in the resource that we're sending
+
+	Changed bool // set to true if this key was updated, read only!
+}
+
+// SendRecv pulls in the sent values into the receive slots. It is called by the
+// receiver and must be given as input the full resource struct to receive on.
+func (obj *BaseRes) SendRecv(res Res) (map[string]bool, error) {
+	if obj.debug {
+		// NOTE: this could expose private resource data like passwords
+		log.Printf("%s[%s]: SendRecv: %+v", obj.Kind(), obj.GetName(), obj.Recv)
+	}
+	var updated = make(map[string]bool) // list of updated keys
+	var err error
+	for k, v := range obj.Recv {
+		updated[k] = false // default
+		v.Changed = false  // reset to the default
+		// send
+		obj1 := reflect.Indirect(reflect.ValueOf(v.Res))
+		type1 := obj1.Type()
+		value1 := obj1.FieldByName(v.Key)
+		kind1 := value1.Kind()
+
+		// recv
+		obj2 := reflect.Indirect(reflect.ValueOf(res)) // pass in full struct
+		type2 := obj2.Type()
+		value2 := obj2.FieldByName(k)
+		kind2 := value2.Kind()
+
+		if obj.debug {
+			log.Printf("Send(%s) has %v: %v", type1, kind1, value1)
+			log.Printf("Recv(%s) has %v: %v", type2, kind2, value2)
+		}
+
+		// i think we probably want the same kind, at least for now...
+		if kind1 != kind2 {
+			e := fmt.Errorf("Kind mismatch between %s[%s]: %s and %s[%s]: %s", v.Res.Kind(), v.Res.GetName(), kind1, obj.Kind(), obj.GetName(), kind2)
+			err = multierr.Append(err, e) // list of errors
+			continue
+		}
+
+		// if the types don't match, we can't use send->recv
+		// TODO: do we want to relax this for string -> *string ?
+		if e := TypeCmp(value1, value2); e != nil {
+			e := errwrap.Wrapf(e, "Type mismatch between %s[%s] and %s[%s]", v.Res.Kind(), v.Res.GetName(), obj.Kind(), obj.GetName())
+			err = multierr.Append(err, e) // list of errors
+			continue
+		}
+
+		// if we can't set, then well this is pointless!
+		if !value2.CanSet() {
+			e := fmt.Errorf("Can't set %s[%s].%s", obj.Kind(), obj.GetName(), k)
+			err = multierr.Append(err, e) // list of errors
+			continue
+		}
+
+		// if we can't interface, we can't compare...
+		if !value1.CanInterface() || !value2.CanInterface() {
+			e := fmt.Errorf("Can't interface %s[%s].%s", obj.Kind(), obj.GetName(), k)
+			err = multierr.Append(err, e) // list of errors
+			continue
+		}
+
+		// if the values aren't equal, we're changing the receiver
+		if !reflect.DeepEqual(value1.Interface(), value2.Interface()) {
+			// TODO: can we catch the panics here in case they happen?
+			value2.Set(value1) // do it for all types that match
+			updated[k] = true  // we updated this key!
+			v.Changed = true   // tag this key as updated!
+			log.Printf("SendRecv: %s[%s].%s -> %s[%s].%s", v.Res.Kind(), v.Res.GetName(), v.Key, obj.Kind(), obj.GetName(), k)
+		}
+	}
+	return updated, err
+}
+
+// TypeCmp compares two reflect values to see if they are the same Kind. It can
+// look into a ptr Kind to see if the underlying pair of ptr's can TypeCmp too!
+func TypeCmp(a, b reflect.Value) error {
+	ta, tb := a.Type(), b.Type()
+	if ta != tb {
+		return fmt.Errorf("Type mismatch: %s != %s", ta, tb)
+	}
+	// NOTE: it seems we don't need to recurse into pointers to sub check!
+
+	return nil // identical Type()'s
+}

resources/svc.go

@@ -21,10 +21,8 @@ package resources
 
 import (
 	"encoding/gob"
-	"errors"
 	"fmt"
 	"log"
-	"time"
 
 	"github.com/purpleidea/mgmt/event"
 	"github.com/purpleidea/mgmt/util"
@@ -32,6 +30,7 @@ import (
 	systemd "github.com/coreos/go-systemd/dbus" // change namespace
 	systemdUtil "github.com/coreos/go-systemd/util"
 	"github.com/godbus/dbus" // namespace collides with systemd wrapper
+	errwrap "github.com/pkg/errors"
 )
 
 func init() {
@@ -76,22 +75,7 @@ func (obj *SvcRes) Validate() error {
 
 // Watch is the primary listener for this resource and it outputs events.
 func (obj *SvcRes) Watch(processChan chan event.Event) error {
-	if obj.IsWatching() {
+	cuid := obj.Converger() // get the converger uid used to report status
-		return nil
-	}
-	obj.SetWatching(true)
-	defer obj.SetWatching(false)
-	cuid := obj.converger.Register()
-	defer cuid.Unregister()
-
-	var startup bool
-	Startup := func(block bool) <-chan time.Time {
-		if block {
-			return nil // blocks forever
-			//return make(chan time.Time) // blocks forever
-		}
-		return time.After(time.Duration(500) * time.Millisecond) // 1/2 the resolution of converged timeout
-	}
 
 	// obj.Name: svc name
 	if !systemdUtil.IsRunningSystemd() {
@@ -100,14 +84,14 @@ func (obj *SvcRes) Watch(processChan chan event.Event) error {
 
 	conn, err := systemd.NewSystemdConnection() // needs root access
 	if err != nil {
-		return fmt.Errorf("Failed to connect to systemd: %s", err)
+		return errwrap.Wrapf(err, "Failed to connect to systemd")
 	}
 	defer conn.Close()
 
 	// if we share the bus with others, we will get each others messages!!
 	bus, err := util.SystemBusPrivateUsable() // don't share the bus connection!
 	if err != nil {
-		return fmt.Errorf("Failed to connect to bus: %s", err)
+		return errwrap.Wrapf(err, "Failed to connect to bus")
 	}
 
 	// XXX: will this detect new units?
@@ -116,10 +100,14 @@ func (obj *SvcRes) Watch(processChan chan event.Event) error {
 	buschan := make(chan *dbus.Signal, 10)
 	bus.Signal(buschan)
 
-	var svc = fmt.Sprintf("%v.service", obj.Name) // systemd name
+	// notify engine that we're running
+	if err := obj.Running(processChan); err != nil {
+		return err // bubble up a NACK...
+	}
+
+	var svc = fmt.Sprintf("%s.service", obj.Name) // systemd name
 	var send = false                              // send event?
 	var exit = false
-	var dirty = false
 	var invalid = false              // does the svc exist or not?
 	var previous bool                // previous invalid value
 	set := conn.NewSubscriptionSet() // no error should be returned
@@ -143,18 +131,18 @@ func (obj *SvcRes) Watch(processChan chan event.Event) error {
 		if !invalid {
 			var notFound = (loadstate.Value == dbus.MakeVariant("not-found"))
 			if notFound { // XXX: in the loop we'll handle changes better...
-				log.Printf("Failed to find svc: %v", svc)
+				log.Printf("Failed to find svc: %s", svc)
 				invalid = true // XXX: ?
 			}
 		}
 
 		if previous != invalid { // if invalid changed, send signal
 			send = true
-			dirty = true
+			obj.StateOK(false) // dirty
 		}
 
 		if invalid {
-			log.Printf("Waiting for: %v", svc) // waiting for svc to appear...
+			log.Printf("Waiting for: %s", svc) // waiting for svc to appear...
 			if activeSet {
 				activeSet = false
 				set.Remove(svc) // no return value should ever occur
@@ -165,25 +153,17 @@ func (obj *SvcRes) Watch(processChan chan event.Event) error {
 			case <-buschan: // XXX: wait for new units event to unstick
 				cuid.SetConverged(false)
 				// loop so that we can see the changed invalid signal
-				log.Printf("Svc[%v]->DaemonReload()", svc)
+				log.Printf("Svc[%s]->DaemonReload()", svc)
 
 			case event := <-obj.Events():
 				cuid.SetConverged(false)
 				if exit, send = obj.ReadEvent(&event); exit {
 					return nil // exit
 				}
-				if event.GetActivity() {
-					dirty = true
-				}
 
 			case <-cuid.ConvergedTimer():
 				cuid.SetConverged(true) // converged!
 				continue
-
-			case <-Startup(startup):
-				cuid.SetConverged(false)
-				send = true
-				dirty = true
 			}
 		} else {
 			if !activeSet {
@@ -191,7 +171,7 @@ func (obj *SvcRes) Watch(processChan chan event.Event) error {
 				set.Add(svc) // no return value should ever occur
 			}
 
-			log.Printf("Watching: %v", svc) // attempting to watch...
+			log.Printf("Watching: %s", svc) // attempting to watch...
 			obj.SetState(ResStateWatching)  // reset
 			select {
 			case event := <-subChannel:
@@ -203,52 +183,39 @@ func (obj *SvcRes) Watch(processChan chan event.Event) error {
 
 					switch event[svc].ActiveState {
 					case "active":
-						log.Printf("Svc[%v]->Started", svc)
+						log.Printf("Svc[%s]->Started", svc)
 					case "inactive":
-						log.Printf("Svc[%v]->Stopped", svc)
+						log.Printf("Svc[%s]->Stopped", svc)
 					case "reloading":
-						log.Printf("Svc[%v]->Reloading", svc)
+						log.Printf("Svc[%s]->Reloading", svc)
 					default:
 						log.Fatalf("Unknown svc state: %s", event[svc].ActiveState)
 					}
 				} else {
 					// svc stopped (and ActiveState is nil...)
-					log.Printf("Svc[%v]->Stopped", svc)
+					log.Printf("Svc[%s]->Stopped", svc)
 				}
 				send = true
-				dirty = true
+				obj.StateOK(false) // dirty
 
 			case err := <-subErrors:
 				cuid.SetConverged(false)
-				return fmt.Errorf("Unknown %s[%s] error: %v", obj.Kind(), obj.GetName(), err)
+				return errwrap.Wrapf(err, "Unknown %s[%s] error", obj.Kind(), obj.GetName())
 
 			case event := <-obj.Events():
 				cuid.SetConverged(false)
 				if exit, send = obj.ReadEvent(&event); exit {
 					return nil // exit
 				}
-				if event.GetActivity() {
-					dirty = true
-				}
 
 			case <-cuid.ConvergedTimer():
 				cuid.SetConverged(true) // converged!
 				continue
-
-			case <-Startup(startup):
-				cuid.SetConverged(false)
-				send = true
-				dirty = true
 			}
 		}
 
 		if send {
-			startup = true // startup finished
 			send = false
-			if dirty {
-				dirty = false
-				obj.isStateOK = false // something made state dirty
-			}
 			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
 				return err // we exit or bubble up a NACK...
 			}
@@ -258,34 +225,28 @@ func (obj *SvcRes) Watch(processChan chan event.Event) error {
 
 // CheckApply checks the resource state and applies the resource if the bool
 // input is true. It returns error info and if the state check passed or not.
-func (obj *SvcRes) CheckApply(apply bool) (checkok bool, err error) {
+func (obj *SvcRes) CheckApply(apply bool) (checkOK bool, err error) {
-	log.Printf("%v[%v]: CheckApply(%t)", obj.Kind(), obj.GetName(), apply)
-
-	if obj.isStateOK { // cache the state
-		return true, nil
-	}
-
 	if !systemdUtil.IsRunningSystemd() {
-		return false, errors.New("Systemd is not running.")
+		return false, fmt.Errorf("Systemd is not running.")
 	}
 
 	conn, err := systemd.NewSystemdConnection() // needs root access
 	if err != nil {
-		return false, fmt.Errorf("Failed to connect to systemd: %v", err)
+		return false, errwrap.Wrapf(err, "Failed to connect to systemd")
 	}
 	defer conn.Close()
 
-	var svc = fmt.Sprintf("%v.service", obj.Name) // systemd name
+	var svc = fmt.Sprintf("%s.service", obj.Name) // systemd name
 
 	loadstate, err := conn.GetUnitProperty(svc, "LoadState")
 	if err != nil {
-		return false, fmt.Errorf("Failed to get load state: %v", err)
+		return false, errwrap.Wrapf(err, "Failed to get load state")
 	}
 
 	// NOTE: we have to compare variants with other variants, they are really strings...
 	var notFound = (loadstate.Value == dbus.MakeVariant("not-found"))
 	if notFound {
-		return false, fmt.Errorf("Failed to find svc: %v", svc)
+		return false, errwrap.Wrapf(err, "Failed to find svc: %s", svc)
 	}
 
 	// XXX: check svc "enabled at boot" or not status...
@@ -293,14 +254,15 @@ func (obj *SvcRes) CheckApply(apply bool) (checkok bool, err error) {
 	//conn.GetUnitProperties(svc)
 	activestate, err := conn.GetUnitProperty(svc, "ActiveState")
 	if err != nil {
-		return false, fmt.Errorf("Failed to get active state: %v", err)
+		return false, errwrap.Wrapf(err, "Failed to get active state")
 	}
 
 	var running = (activestate.Value == dbus.MakeVariant("active"))
 	var stateOK = ((obj.State == "") || (obj.State == "running" && running) || (obj.State == "stopped" && !running))
-	var startupOK = true // XXX: DETECT AND SET
+	var startupOK = true        // XXX: DETECT AND SET
+	var refresh = obj.Refresh() // do we have a pending reload to apply?
 
-	if stateOK && startupOK {
+	if stateOK && startupOK && !refresh {
 		return true, nil // we are in the correct state
 	}
 
@@ -310,7 +272,7 @@ func (obj *SvcRes) CheckApply(apply bool) (checkok bool, err error) {
 	}
 
 	// apply portion
-	log.Printf("%v[%v]: Apply", obj.Kind(), obj.GetName())
+	log.Printf("%s[%s]: Apply", obj.Kind(), obj.GetName())
 	var files = []string{svc} // the svc represented in a list
 	if obj.Startup == "enabled" {
 		_, _, err = conn.EnableUnitFiles(files, false, true)
@@ -320,7 +282,7 @@ func (obj *SvcRes) CheckApply(apply bool) (checkok bool, err error) {
 	}
 
 	if err != nil {
-		return false, fmt.Errorf("Unable to change startup status: %v", err)
+		return false, errwrap.Wrapf(err, "Unable to change startup status")
 	}
 
 	// XXX: do we need to use a buffered channel here?
@@ -329,23 +291,36 @@ func (obj *SvcRes) CheckApply(apply bool) (checkok bool, err error) {
 	if obj.State == "running" {
 		_, err = conn.StartUnit(svc, "fail", result)
 		if err != nil {
-			return false, fmt.Errorf("Failed to start unit: %v", err)
+			return false, errwrap.Wrapf(err, "Failed to start unit")
 		}
+		if refresh {
+			log.Printf("%s[%s]: Skipping reload, due to pending start", obj.Kind(), obj.GetName())
+		}
+		refresh = false // we did a start, so a reload is not needed
 	} else if obj.State == "stopped" {
 		_, err = conn.StopUnit(svc, "fail", result)
 		if err != nil {
-			return false, fmt.Errorf("Failed to stop unit: %v", err)
+			return false, errwrap.Wrapf(err, "Failed to stop unit")
 		}
+		if refresh {
+			log.Printf("%s[%s]: Skipping reload, due to pending stop", obj.Kind(), obj.GetName())
+		}
+		refresh = false // we did a stop, so a reload is not needed
 	}
 
 	status := <-result
 	if &status == nil {
-		return false, errors.New("Systemd service action result is nil")
+		return false, fmt.Errorf("Systemd service action result is nil")
 	}
 	if status != "done" {
 		return false, fmt.Errorf("Unknown systemd return string: %v", status)
 	}
 
+	if refresh { // we need to reload the service
+		// XXX: run a svc reload here!
+		log.Printf("%s[%s]: Reloading...", obj.Kind(), obj.GetName())
+	}
+
 	// XXX: also set enabled on boot
 
 	return false, nil // success

resources/timer.go

@@ -33,6 +33,8 @@ func init() {
 type TimerRes struct {
 	BaseRes  `yaml:",inline"`
 	Interval int `yaml:"interval"` // Interval : Interval between runs
+
+	ticker *time.Ticker
 }
 
 // TimerUID is the UID struct for TimerRes.
@@ -65,55 +67,46 @@ func (obj *TimerRes) Validate() error {
 	return nil
 }
 
+// newTicker creates a new ticker
+func (obj *TimerRes) newTicker() *time.Ticker {
+	return time.NewTicker(time.Duration(obj.Interval) * time.Second)
+}
+
 // Watch is the primary listener for this resource and it outputs events.
 func (obj *TimerRes) Watch(processChan chan event.Event) error {
-	if obj.IsWatching() {
+	cuid := obj.Converger() // get the converger uid used to report status
-		return nil
-	}
-	obj.SetWatching(true)
-	defer obj.SetWatching(false)
-	cuid := obj.converger.Register()
-	defer cuid.Unregister()
 
-	var startup bool
+	// create a time.Ticker for the given interval
-	Startup := func(block bool) <-chan time.Time {
+	obj.ticker = obj.newTicker()
-		if block {
+	defer obj.ticker.Stop()
-			return nil // blocks forever
-			//return make(chan time.Time) // blocks forever
-		}
-		return time.After(time.Duration(500) * time.Millisecond) // 1/2 the resolution of converged timeout
-	}
 
-	// Create a time.Ticker for the given interval
+	// notify engine that we're running
-	ticker := time.NewTicker(time.Duration(obj.Interval) * time.Second)
+	if err := obj.Running(processChan); err != nil {
-	defer ticker.Stop()
+		return err // bubble up a NACK...
+	}
 
 	var send = false
 
 	for {
 		obj.SetState(ResStateWatching)
 		select {
-		case <-ticker.C: // received the timer event
+		case <-obj.ticker.C: // received the timer event
 			send = true
-			log.Printf("%v[%v]: received tick", obj.Kind(), obj.GetName())
+			log.Printf("%s[%s]: received tick", obj.Kind(), obj.GetName())
 
 		case event := <-obj.Events():
 			cuid.SetConverged(false)
 			if exit, _ := obj.ReadEvent(&event); exit {
 				return nil
 			}
+
 		case <-cuid.ConvergedTimer():
 			cuid.SetConverged(true)
 			continue
-
-		case <-Startup(startup):
-			cuid.SetConverged(false)
-			send = true
 		}
+
 		if send {
-			startup = true // startup finished
 			send = false
-			obj.isStateOK = false
 			if exit, err := obj.DoSend(processChan, "timer ticked"); exit || err != nil {
 				return err // we exit or bubble up a NACK...
 			}
@@ -121,6 +114,22 @@ func (obj *TimerRes) Watch(processChan chan event.Event) error {
 	}
 }
 
+// CheckApply method for Timer resource. Triggers a timer reset on notify.
+func (obj *TimerRes) CheckApply(apply bool) (bool, error) {
+	// because there are no checks to run, this resource has a less
+	// traditional pattern than what is seen in most resources...
+	if !obj.Refresh() { // this works for apply || !apply
+		return true, nil // state is always okay if no refresh to do
+	} else if !apply { // we had a refresh to do
+		return false, nil // therefore state is wrong
+	}
+
+	// reset the timer since apply && refresh
+	obj.ticker.Stop()
+	obj.ticker = obj.newTicker()
+	return false, nil
+}
+
 // GetUIDs includes all params to make a unique identification of this object.
 // Most resources only return one, although some resources can return multiple.
 func (obj *TimerRes) GetUIDs() []ResUID {
@@ -158,9 +167,3 @@ func (obj *TimerRes) Compare(res Res) bool {
 	}
 	return true
 }
-
-// CheckApply method for Timer resource. Does nothing, returns happy!
-func (obj *TimerRes) CheckApply(apply bool) (bool, error) {
-	log.Printf("%v[%v]: CheckApply(%t)", obj.Kind(), obj.GetName(), apply)
-	return true, nil // state is always okay
-}

resources/virt.go

@@ -22,10 +22,10 @@ import (
 	"fmt"
 	"log"
 	"math/rand"
+	"net/url"
 	"time"
 
 	"github.com/purpleidea/mgmt/event"
-	"github.com/purpleidea/mgmt/global"
 
 	errwrap "github.com/pkg/errors"
 	"github.com/rgbkrk/libvirt-go"
@@ -39,6 +39,19 @@ var (
 	libvirtInitialized = false
 )
 
+type virtURISchemeType int
+
+const (
+	defaultURI virtURISchemeType = iota
+	lxcURI
+)
+
+// VirtAuth is used to pass credentials to libvirt.
+type VirtAuth struct {
+	Username string `yaml:"username"`
+	Password string `yaml:"password"`
+}
+
 // VirtRes is a libvirt resource. A transient virt resource, which has its state
 // set to `shutoff` is one which does not exist. The parallel equivalent is a
 // file resource which removes a particular path.
@@ -49,18 +62,21 @@ type VirtRes struct {
 	Transient  bool               `yaml:"transient"` // defined (false) or undefined (true)
 	CPUs       uint16             `yaml:"cpus"`
 	Memory     uint64             `yaml:"memory"` // in KBytes
+	OSInit     string             `yaml:"osinit"` // init used by lxc
 	Boot       []string           `yaml:"boot"`   // boot order. values: fd, hd, cdrom, network
 	Disk       []diskDevice       `yaml:"disk"`
 	CDRom      []cdRomDevice      `yaml:"cdrom"`
 	Network    []networkDevice    `yaml:"network"`
 	Filesystem []filesystemDevice `yaml:"filesystem"`
+	Auth       *VirtAuth          `yaml:"auth"`
 
-	conn   libvirt.VirConnection
+	conn      libvirt.VirConnection
-	absent bool // cached state
+	absent    bool // cached state
+	uriScheme virtURISchemeType
 }
 
 // NewVirtRes is a constructor for this resource. It also calls Init() for you.
-func NewVirtRes(name string, uri, state string, transient bool, cpus uint16, memory uint64) (*VirtRes, error) {
+func NewVirtRes(name string, uri, state string, transient bool, cpus uint16, memory uint64, osinit string) (*VirtRes, error) {
 	obj := &VirtRes{
 		BaseRes: BaseRes{
 			Name: name,
@@ -70,6 +86,7 @@ func NewVirtRes(name string, uri, state string, transient bool, cpus uint16, mem
 		Transient: transient,
 		CPUs:      cpus,
 		Memory:    memory,
+		OSInit:    osinit,
 	}
 	return obj, obj.Init()
 }
@@ -82,6 +99,15 @@ func (obj *VirtRes) Init() error {
 		}
 		libvirtInitialized = true
 	}
+	var u *url.URL
+	var err error
+	if u, err = url.Parse(obj.URI); err != nil {
+		return errwrap.Wrapf(err, "%s[%s]: Parsing URI failed: %s", obj.Kind(), obj.GetName(), obj.URI)
+	}
+	switch u.Scheme {
+	case "lxc":
+		obj.uriScheme = lxcURI
+	}
 
 	obj.absent = (obj.Transient && obj.State == "shutoff") // machine shouldn't exist
 
@@ -94,26 +120,21 @@ func (obj *VirtRes) Validate() error {
 	return nil
 }
 
+func (obj *VirtRes) connect() (conn libvirt.VirConnection, err error) {
+	if obj.Auth != nil {
+		conn, err = libvirt.NewVirConnectionWithAuth(obj.URI, obj.Auth.Username, obj.Auth.Password)
+	}
+	if obj.Auth == nil || err != nil {
+		conn, err = libvirt.NewVirConnection(obj.URI)
+	}
+	return
+}
+
 // Watch is the primary listener for this resource and it outputs events.
 func (obj *VirtRes) Watch(processChan chan event.Event) error {
-	if obj.IsWatching() {
+	cuid := obj.Converger() // get the converger uid used to report status
-		return nil
-	}
-	obj.SetWatching(true)
-	defer obj.SetWatching(false)
-	cuid := obj.converger.Register()
-	defer cuid.Unregister()
 
-	var startup bool
+	conn, err := obj.connect()
-	Startup := func(block bool) <-chan time.Time {
-		if block {
-			return nil // blocks forever
-			//return make(chan time.Time) // blocks forever
-		}
-		return time.After(time.Duration(500) * time.Millisecond) // 1/2 the resolution of converged timeout
-	}
-
-	conn, err := libvirt.NewVirConnection(obj.URI)
 	if err != nil {
 		return fmt.Errorf("Connection to libvirt failed with: %s", err)
 	}
@@ -153,7 +174,7 @@ func (obj *VirtRes) Watch(processChan chan event.Event) error {
 				if domName == obj.GetName() {
 					eventChan <- lifecycleEvent.Event
 				}
-			} else if global.DEBUG {
+			} else if obj.debug {
 				log.Printf("%s[%s]: Event details isn't DomainLifecycleEvent", obj.Kind(), obj.GetName())
 			}
 			return 0
@@ -167,9 +188,13 @@ func (obj *VirtRes) Watch(processChan chan event.Event) error {
 	)
 	defer conn.DomainEventDeregister(callbackID)
 
+	// notify engine that we're running
+	if err := obj.Running(processChan); err != nil {
+		return err // bubble up a NACK...
+	}
+
 	var send = false
 	var exit = false
-	var dirty = false
 
 	for {
 		select {
@@ -178,37 +203,37 @@ func (obj *VirtRes) Watch(processChan chan event.Event) error {
 			switch event {
 			case libvirt.VIR_DOMAIN_EVENT_DEFINED:
 				if obj.Transient {
-					dirty = true
+					obj.StateOK(false) // dirty
 					send = true
 				}
 			case libvirt.VIR_DOMAIN_EVENT_UNDEFINED:
 				if !obj.Transient {
-					dirty = true
+					obj.StateOK(false) // dirty
 					send = true
 				}
 			case libvirt.VIR_DOMAIN_EVENT_STARTED:
 				fallthrough
 			case libvirt.VIR_DOMAIN_EVENT_RESUMED:
 				if obj.State != "running" {
-					dirty = true
+					obj.StateOK(false) // dirty
 					send = true
 				}
 			case libvirt.VIR_DOMAIN_EVENT_SUSPENDED:
 				if obj.State != "paused" {
-					dirty = true
+					obj.StateOK(false) // dirty
 					send = true
 				}
 			case libvirt.VIR_DOMAIN_EVENT_STOPPED:
 				fallthrough
 			case libvirt.VIR_DOMAIN_EVENT_SHUTDOWN:
 				if obj.State != "shutoff" {
-					dirty = true
+					obj.StateOK(false) // dirty
 					send = true
 				}
 			case libvirt.VIR_DOMAIN_EVENT_PMSUSPENDED:
 				fallthrough
 			case libvirt.VIR_DOMAIN_EVENT_CRASHED:
-				dirty = true
+				obj.StateOK(false) // dirty
 				send = true
 			}
 
@@ -225,21 +250,10 @@ func (obj *VirtRes) Watch(processChan chan event.Event) error {
 		case <-cuid.ConvergedTimer():
 			cuid.SetConverged(true) // converged!
 			continue
-
-		case <-Startup(startup):
-			cuid.SetConverged(false)
-			send = true
-			dirty = true
 		}
 
 		if send {
-			startup = true // startup finished
 			send = false
-			// only invalid state on certain types of events
-			if dirty {
-				dirty = false
-				obj.isStateOK = false // something made state dirty
-			}
 			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
 				return err // we exit or bubble up a NACK...
 			}
@@ -342,14 +356,8 @@ func (obj *VirtRes) domainCreate() (libvirt.VirDomain, bool, error) {
 // CheckApply checks the resource state and applies the resource if the bool
 // input is true. It returns error info and if the state check passed or not.
 func (obj *VirtRes) CheckApply(apply bool) (bool, error) {
-	log.Printf("%s[%s]: CheckApply(%t)", obj.Kind(), obj.GetName(), apply)
-
-	if obj.isStateOK { // cache the state
-		return true, nil
-	}
-
 	var err error
-	obj.conn, err = libvirt.NewVirConnection(obj.URI)
+	obj.conn, err = obj.connect()
 	if err != nil {
 		return false, fmt.Errorf("Connection to libvirt failed with: %s", err)
 	}
@@ -359,10 +367,9 @@ func (obj *VirtRes) CheckApply(apply bool) (bool, error) {
 	dom, err := obj.conn.LookupDomainByName(obj.GetName())
 	if err == nil {
 		// pass
-	} else if virErr, ok := err.(libvirt.VirError); ok && virErr.Domain == libvirt.VIR_FROM_QEMU && virErr.Code == libvirt.VIR_ERR_NO_DOMAIN {
+	} else if virErr, ok := err.(libvirt.VirError); ok && virErr.Code == libvirt.VIR_ERR_NO_DOMAIN {
 		// domain not found
 		if obj.absent {
-			obj.isStateOK = true
 			return true, nil
 		}
 
@@ -501,22 +508,49 @@ func (obj *VirtRes) CheckApply(apply bool) (bool, error) {
 		}
 	}
 
-	if apply || checkOK {
-		obj.isStateOK = true
-	}
 	return checkOK, nil // w00t
 }
 
+// Return the correct domain type based on the uri
+func (obj VirtRes) getDomainType() string {
+	switch obj.uriScheme {
+	case lxcURI:
+		return "<domain type='lxc'>"
+	default:
+		return "<domain type='kvm'>"
+	}
+}
+
+// Return the correct os type based on the uri
+func (obj VirtRes) getOSType() string {
+	switch obj.uriScheme {
+	case lxcURI:
+		return "<type>exe</type>"
+	default:
+		return "<type>hvm</type>"
+	}
+}
+
+func (obj VirtRes) getOSInit() string {
+	switch obj.uriScheme {
+	case lxcURI:
+		return fmt.Sprintf("<init>%s</init>", obj.OSInit)
+	default:
+		return ""
+	}
+}
+
 func (obj *VirtRes) getDomainXML() string {
 	var b string
-	b += "<domain type='kvm'>" // start domain
+	b += obj.getDomainType() // start domain
 
 	b += fmt.Sprintf("<name>%s</name>", obj.GetName())
 	b += fmt.Sprintf("<memory unit='KiB'>%d</memory>", obj.Memory)
 	b += fmt.Sprintf("<vcpu>%d</vcpu>", obj.CPUs)
 
 	b += "<os>"
-	b += "<type>hvm</type>"
+	b += obj.getOSType()
+	b += obj.getOSInit()
 	if obj.Boot != nil {
 		for _, boot := range obj.Boot {
 			b += fmt.Sprintf("<boot dev='%s'/>", boot)

test/test-gofmt.sh

@@ -9,9 +9,9 @@ ROOT=$(dirname "${BASH_SOURCE}")/..
 
 GO_VERSION=($(go version))
 
-if [[ -z $(echo "${GO_VERSION[2]}" | grep -E 'go1.2|go1.3|go1.4|go1.5|go1.6') ]]; then
+if [[ -z $(echo "${GO_VERSION[2]}" | grep -E 'go1.2|go1.3|go1.4|go1.5|go1.6|go1.7|go1.8|devel') ]]; then
-	echo "Unknown go version '${GO_VERSION}', skipping gofmt."
+	echo "Unknown go version '${GO_VERSION[2]}', failing gofmt."
-	exit 0
+	exit 1
 fi
 
 cd "${ROOT}"

yamlgraph/gapi.go

@@ -70,12 +70,12 @@ func (obj *GAPI) Graph() (*pgraph.Graph, error) {
 		return nil, fmt.Errorf("yamlgraph: ParseConfigFromFile returned nil")
 	}
 
-	g, err := config.NewGraphFromConfig(obj.data.Hostname, obj.data.EmbdEtcd, obj.data.Noop)
+	g, err := config.NewGraphFromConfig(obj.data.Hostname, obj.data.World, obj.data.Noop)
 	return g, err
 }
 
-// SwitchStream returns nil errors every time there could be a new graph.
+// Next returns nil errors every time there could be a new graph.
-func (obj *GAPI) SwitchStream() chan error {
+func (obj *GAPI) Next() chan error {
 	if obj.data.NoWatch {
 		return nil
 	}
@@ -88,7 +88,8 @@ func (obj *GAPI) SwitchStream() chan error {
 			ch <- fmt.Errorf("yamlgraph: GAPI is not initialized")
 			return
 		}
-		configChan := recwatch.ConfigWatch(*obj.File)
+		configWatcher := recwatch.NewConfigWatcher()
+		configChan := configWatcher.ConfigWatch(*obj.File) // simple
 		for {
 			select {
 			case err, ok := <-configChan: // returns nil events on ok!

yamlgraph/gconfig.go

@@ -26,8 +26,7 @@ import (
 	"reflect"
 	"strings"
 
-	"github.com/purpleidea/mgmt/etcd"
+	"github.com/purpleidea/mgmt/gapi"
-	"github.com/purpleidea/mgmt/global"
 	"github.com/purpleidea/mgmt/pgraph"
 	"github.com/purpleidea/mgmt/resources"
 	"github.com/purpleidea/mgmt/util"
@@ -56,14 +55,17 @@ type Edge struct {
 // Resources is the data structure of the set of resources.
 type Resources struct {
 	// in alphabetical order
-	Exec  []*resources.ExecRes  `yaml:"exec"`
+	Exec     []*resources.ExecRes     `yaml:"exec"`
-	File  []*resources.FileRes  `yaml:"file"`
+	File     []*resources.FileRes     `yaml:"file"`
-	Msg   []*resources.MsgRes   `yaml:"msg"`
+	Hostname []*resources.HostnameRes `yaml:"hostname"`
-	Noop  []*resources.NoopRes  `yaml:"noop"`
+	Msg      []*resources.MsgRes      `yaml:"msg"`
-	Pkg   []*resources.PkgRes   `yaml:"pkg"`
+	Noop     []*resources.NoopRes     `yaml:"noop"`
-	Svc   []*resources.SvcRes   `yaml:"svc"`
+	Nspawn   []*resources.NspawnRes   `yaml:"nspawn"`
-	Timer []*resources.TimerRes `yaml:"timer"`
+	Password []*resources.PasswordRes `yaml:"password"`
-	Virt  []*resources.VirtRes  `yaml:"virt"`
+	Pkg      []*resources.PkgRes      `yaml:"pkg"`
+	Svc      []*resources.SvcRes      `yaml:"svc"`
+	Timer    []*resources.TimerRes    `yaml:"timer"`
+	Virt     []*resources.VirtRes     `yaml:"virt"`
 }
 
 // GraphConfig is the data structure that describes a single graph to run.
@@ -89,7 +91,7 @@ func (c *GraphConfig) Parse(data []byte) error {
 
 // NewGraphFromConfig transforms a GraphConfig struct into a new graph.
 // FIXME: remove any possibly left over, now obsolete graph diff code from here!
-func (c *GraphConfig) NewGraphFromConfig(hostname string, embdEtcd *etcd.EmbdEtcd, noop bool) (*pgraph.Graph, error) {
+func (c *GraphConfig) NewGraphFromConfig(hostname string, world gapi.World, noop bool) (*pgraph.Graph, error) {
 	// hostname is the uuid for the host
 
 	var graph *pgraph.Graph          // new graph to return
@@ -114,9 +116,6 @@ func (c *GraphConfig) NewGraphFromConfig(hostname string, embdEtcd *etcd.EmbdEtc
 		slice := reflect.ValueOf(iface)
 		// XXX: should we just drop these everywhere and have the kind strings be all lowercase?
 		kind := util.FirstToUpper(name)
-		if global.DEBUG {
-			log.Printf("Config: Processing: %v...", kind)
-		}
 		for j := 0; j < slice.Len(); j++ { // loop through resources of same kind
 			x := slice.Index(j).Interface()
 			res, ok := x.(resources.Res) // convert to Res type
@@ -142,19 +141,19 @@ func (c *GraphConfig) NewGraphFromConfig(hostname string, embdEtcd *etcd.EmbdEtc
 				keep = append(keep, v)          // append
 
 			} else if !noop { // do not export any resources if noop
-				// store for addition to etcd storage...
+				// store for addition to backend storage...
 				res.SetName(res.GetName()[2:]) //slice off @@
 				res.SetKind(kind)              // cheap init
 				resourceList = append(resourceList, res)
 			}
 		}
 	}
-	// store in etcd
+	// store in backend (usually etcd)
-	if err := etcd.EtcdSetResources(embdEtcd, hostname, resourceList); err != nil {
+	if err := world.ResExport(resourceList); err != nil {
 		return nil, fmt.Errorf("Config: Could not export resources: %v", err)
 	}
 
-	// lookup from etcd
+	// lookup from backend (usually etcd)
 	var hostnameFilter []string // empty to get from everyone
 	kindFilter := []string{}
 	for _, t := range c.Collector {
@@ -162,11 +161,11 @@ func (c *GraphConfig) NewGraphFromConfig(hostname string, embdEtcd *etcd.EmbdEtc
 		kind := util.FirstToUpper(t.Kind)
 		kindFilter = append(kindFilter, kind)
 	}
-	// do all the graph look ups in one single step, so that if the etcd
+	// do all the graph look ups in one single step, so that if the backend
 	// database changes, we don't have a partial state of affairs...
 	if len(kindFilter) > 0 { // if kindFilter is empty, don't need to do lookups!
 		var err error
-		resourceList, err = etcd.EtcdGetResources(embdEtcd, hostnameFilter, kindFilter)
+		resourceList, err = world.ResCollect(hostnameFilter, kindFilter)
 		if err != nil {
 			return nil, fmt.Errorf("Config: Could not collect resources: %v", err)
 		}
@@ -177,7 +176,7 @@ func (c *GraphConfig) NewGraphFromConfig(hostname string, embdEtcd *etcd.EmbdEtc
 		for _, t := range c.Collector {
 			// XXX: should we just drop these everywhere and have the kind strings be all lowercase?
 			kind := util.FirstToUpper(t.Kind)
-			// use t.Kind and optionally t.Pattern to collect from etcd storage
+			// use t.Kind and optionally t.Pattern to collect from storage
 			log.Printf("Collect: %v; Pattern: %v", kind, t.Pattern)
 
 			// XXX: expand to more complex pattern matching here...