golint: Fix some golint issues

Let tip builds pass in travis too!

.ackrc

@@ -1,2 +1,3 @@
 --ignore-dir=old/
 --ignore-dir=tmp/
+--ignore-dir=vendor/

.gitignore

@@ -1,10 +1,11 @@
+.idea/
 .omv/
 .ssh/
 .vagrant/
-mgmt-documentation.pdf
 old/
 tmp/
 *_stringer.go
 mgmt
 mgmt.static
+mgmt.iml
 rpmbuild/

.gitmodules

@@ -10,3 +10,6 @@
 [submodule "vendor/gopkg.in/fsnotify.v1"]
 	path = vendor/gopkg.in/fsnotify.v1
 	url = https://gopkg.in/fsnotify.v1
+[submodule "vendor/github.com/purpleidea/go-systemd"]
+	path = vendor/github.com/purpleidea/go-systemd
+	url = https://github.com/purpleidea/go-systemd

.travis.yml

@@ -3,7 +3,8 @@ go:
   - 1.6
   - 1.7
   - tip
-sudo: false
+sudo: true
+dist: trusty
 before_install: 'git fetch --unshallow'
 install: 'make deps'
 script: 'make test'

Makefile

@@ -138,8 +138,8 @@ format: gofmt yamlfmt
 
 docs: $(PROGRAM)-documentation.pdf
 
-$(PROGRAM)-documentation.pdf: DOCUMENTATION.md
+$(PROGRAM)-documentation.pdf: docs/documentation.md
-	pandoc DOCUMENTATION.md -o '$(PROGRAM)-documentation.pdf'
+	pandoc docs/documentation.md -o docs/'$(PROGRAM)-documentation.pdf'
 
 #
 #	build aliases
@@ -184,7 +184,7 @@ $(SRPM): $(SPEC) $(SOURCE)
 $(SPEC): rpmbuild/ spec.in
 	@echo Running templater...
 	#cat spec.in > $(SPEC)
-	sed -e s/__PROGRAM__/$(PROGRAM)/ -e s/__VERSION__/$(VERSION)/ -e s/__RELEASE__/$(RELEASE)/ < spec.in > $(SPEC)
+	sed -e s/__PROGRAM__/$(PROGRAM)/g -e s/__VERSION__/$(VERSION)/g -e s/__RELEASE__/$(RELEASE)/g < spec.in > $(SPEC)
 	# append a changelog to the .spec file
 	git log --format="* %cd %aN <%aE>%n- (%h) %s%d%n" --date=local | sed -r 's/[0-9]+:[0-9]+:[0-9]+ //' >> $(SPEC)
 

README.md

@@ -4,7 +4,7 @@
 
 [![Go Report Card](https://goreportcard.com/badge/github.com/purpleidea/mgmt)](https://goreportcard.com/report/github.com/purpleidea/mgmt)
 [![Build Status](https://secure.travis-ci.org/purpleidea/mgmt.png?branch=master)](http://travis-ci.org/purpleidea/mgmt)
-[![Documentation](https://img.shields.io/docs/markdown.png)](DOCUMENTATION.md)
+[![Documentation](https://img.shields.io/docs/markdown.png)](docs/documentation.md)
 [![GoDoc](https://godoc.org/github.com/purpleidea/mgmt?status.svg)](https://godoc.org/github.com/purpleidea/mgmt)
 [![IRC](https://img.shields.io/irc/%23mgmtconfig.png)](https://webchat.freenode.net/?channels=#mgmtconfig)
 [![Jenkins](https://img.shields.io/jenkins/status.png)](https://ci.centos.org/job/purpleidea-mgmt/)
@@ -23,14 +23,25 @@ With your help you'll be able to influence our design and get us there sooner!
 
 ## Questions:
 Please join the [#mgmtconfig](https://webchat.freenode.net/?channels=#mgmtconfig) IRC community!
-If you have a well phrased question that might benefit others, consider asking it by sending a patch to the documentation [FAQ](https://github.com/purpleidea/mgmt/blob/master/DOCUMENTATION.md#usage-and-frequently-asked-questions) section. I'll merge your question, and a patch with the answer!
+If you have a well phrased question that might benefit others, consider asking it by sending a patch to the documentation [FAQ](https://github.com/purpleidea/mgmt/blob/master/docs/documentation.md#usage-and-frequently-asked-questions) section. I'll merge your question, and a patch with the answer!
 
 ## Quick start:
 * Make sure you have golang version 1.6 or greater installed.
-* Clone the repository recursively, eg: `git clone --recursive https://github.com/purpleidea/mgmt/`.
+* If you do not have a GOPATH yet, create one and export it:
-* Get the remaining golang dependencies on your own, or run `make deps` if you're comfortable with how we install them.
+```
+mkdir $HOME/gopath
+export GOPATH=$HOME/gopath
+```
+* You might also want to add the GOPATH to your `~/.bashrc` or `~/.profile`.
+* For more information you can read the [GOPATH documentation](https://golang.org/cmd/go/#hdr-GOPATH_environment_variable).
+* Next download the mgmt code base, and switch to that directory:
+```
+go get -u github.com/purpleidea/mgmt
+cd $GOPATH/src/github.com/purpleidea/mgmt
+```
+* Get the remaining golang deps with `go get ./...`, or run `make deps` if you're comfortable with how we install them.
 * Run `make build` to get a freshly built `mgmt` binary.
-* Run `time ./mgmt run --file examples/graph0.yaml --converged-timeout=5 --tmp-prefix` to try out a very simple example!
+* Run `time ./mgmt run --yaml examples/graph0.yaml --converged-timeout=5 --tmp-prefix` to try out a very simple example!
 * To run continuously in the default mode of operation, omit the `--converged-timeout` option.
 * Have fun hacking on our future technology!
 
@@ -38,7 +49,7 @@ If you have a well phrased question that might benefit others, consider asking i
 Please look in the [examples/](examples/) folder for more examples!
 
 ## Documentation:
-Please see: [DOCUMENTATION.md](DOCUMENTATION.md) or [PDF](https://pdfdoc-purpleidea.rhcloud.com/pdf/https://github.com/purpleidea/mgmt/blob/master/DOCUMENTATION.md).
+Please see: the manually created [documentation.md](docs/documentation.md) (also available as [PDF](https://pdfdoc-purpleidea.rhcloud.com/pdf/https://github.com/purpleidea/mgmt/blob/master/docs/documentation.md)) and the automatically generated [GoDoc documentation](https://godoc.org/github.com/purpleidea/mgmt).
 
 ## Roadmap:
 Please see: [TODO.md](TODO.md) for a list of upcoming work and TODO items.
@@ -53,7 +64,7 @@ Feel free to read my article on [debugging golang programs](https://ttboj.wordpr
 ## Dependencies:
 * golang 1.6 or higher (required, available in most distros)
 * golang libraries (required, available with `go get`)
-
+```
 go get github.com/coreos/etcd/client
 go get gopkg.in/yaml.v2
 go get gopkg.in/fsnotify.v1
@@ -61,11 +72,12 @@ Feel free to read my article on [debugging golang programs](https://ttboj.wordpr
 go get github.com/coreos/go-systemd/dbus
 go get github.com/coreos/go-systemd/util
 go get github.com/coreos/pkg/capnslog
-
+go get github.com/rgbkrk/libvirt-go
-* stringer (required for building), available as a package on some platforms, otherwise via `go get`
+```
-
+* stringer (optional for building), available as a package on some platforms, otherwise via `go get`
+```
 go get golang.org/x/tools/cmd/stringer
-
+```
 * pandoc (optional, for building a pdf of the documentation)
 * graphviz (optional, for building a visual representation of the graph)
 
@@ -90,6 +102,11 @@ We'd love to have your patches! Please send them by email, or as a pull request.
 * James Shubin; video: [Recording from DebConf16](http://meetings-archive.debian.net/pub/debian-meetings/2016/debconf16/Next_Generation_Config_Mgmt.webm) ([Slides](https://annex.debconf.org//debconf-share/debconf16/slides/15-next-generation-config-mgmt.pdf))
 * Felix Frank; blog: [Edging It All In (puppet and mgmt edges)](https://ffrank.github.io/features/2016/07/12/edging-it-all-in/)
 * Felix Frank; blog: [Translating All The Things (puppet to mgmt translation warnings)](https://ffrank.github.io/features/2016/08/19/translating-all-the-things/)
+* James Shubin; video: [Recording from systemd.conf 2016](https://www.youtube.com/watch?v=jB992Zb3nH0&html5=1)
+* James Shubin; blog: [Remote execution in mgmt](https://ttboj.wordpress.com/2016/10/07/remote-execution-in-mgmt/)
+* James Shubin; video: [Recording from High Load Strategy 2016](https://vimeo.com/191493409)
+* James Shubin; video: [Recording from NLUUG 2016](https://www.youtube.com/watch?v=MmpwOQAb_SE&html5=1)
+* James Shubin; blog: [Send/Recv in mgmt](https://ttboj.wordpress.com/2016/12/07/sendrecv-in-mgmt/)
 
 ##
 

converger/converger.go

@@ -27,26 +27,26 @@ import (
 )
 
 // TODO: we could make a new function that masks out the state of certain
-// UUID's, but at the moment the new Timer code has obsoleted the need...
+// UID's, but at the moment the new Timer code has obsoleted the need...
 
 // Converger is the general interface for implementing a convergence watcher
 type Converger interface { // TODO: need a better name
-	Register() ConvergerUUID
+	Register() ConvergerUID
-	IsConverged(ConvergerUUID) bool         // is the UUID converged ?
+	IsConverged(ConvergerUID) bool         // is the UID converged ?
-	SetConverged(ConvergerUUID, bool) error // set the converged state of the UUID
+	SetConverged(ConvergerUID, bool) error // set the converged state of the UID
-	Unregister(ConvergerUUID)
+	Unregister(ConvergerUID)
 	Start()
 	Pause()
 	Loop(bool)
-	ConvergedTimer(ConvergerUUID) <-chan time.Time
+	ConvergedTimer(ConvergerUID) <-chan time.Time
 	Status() map[uint64]bool
 	Timeout() int                // returns the timeout that this was created with
 	SetStateFn(func(bool) error) // sets the stateFn
 }
 
-// ConvergerUUID is the interface resources can use to notify with if converged
+// ConvergerUID is the interface resources can use to notify with if converged
 // you'll need to use part of the Converger interface to Register initially too
-type ConvergerUUID interface {
+type ConvergerUID interface {
 	ID() uint64   // get Id
 	Name() string // get a friendly name
 	SetName(string)
@@ -73,8 +73,8 @@ type converger struct {
 	status    map[uint64]bool
 }
 
-// convergerUUID is an implementation of the ConvergerUUID interface
+// convergerUID is an implementation of the ConvergerUID interface
-type convergerUUID struct {
+type convergerUID struct {
 	converger Converger
 	id        uint64
 	name      string // user defined, friendly name
@@ -95,13 +95,13 @@ func NewConverger(timeout int, stateFn func(bool) error) *converger {
 	}
 }
 
-// Register assigns a ConvergerUUID to the caller
+// Register assigns a ConvergerUID to the caller
-func (obj *converger) Register() ConvergerUUID {
+func (obj *converger) Register() ConvergerUID {
 	obj.mutex.Lock()
 	defer obj.mutex.Unlock()
 	obj.lastid++
 	obj.status[obj.lastid] = false // initialize as not converged
-	return &convergerUUID{
+	return &convergerUID{
 		converger: obj,
 		id:        obj.lastid,
 		name:      fmt.Sprintf("%d", obj.lastid), // some default
@@ -110,30 +110,30 @@ func (obj *converger) Register() ConvergerUUID {
 	}
 }
 
-// IsConverged gets the converged status of a uuid
+// IsConverged gets the converged status of a uid
-func (obj *converger) IsConverged(uuid ConvergerUUID) bool {
+func (obj *converger) IsConverged(uid ConvergerUID) bool {
-	if !uuid.IsValid() {
+	if !uid.IsValid() {
-		panic(fmt.Sprintf("Id of ConvergerUUID(%s) is nil!", uuid.Name()))
+		panic(fmt.Sprintf("Id of ConvergerUID(%s) is nil!", uid.Name()))
 	}
 	obj.mutex.RLock()
-	isConverged, found := obj.status[uuid.ID()] // lookup
+	isConverged, found := obj.status[uid.ID()] // lookup
 	obj.mutex.RUnlock()
 	if !found {
-		panic("Id of ConvergerUUID is unregistered!")
+		panic("Id of ConvergerUID is unregistered!")
 	}
 	return isConverged
 }
 
-// SetConverged updates the converger with the converged state of the UUID
+// SetConverged updates the converger with the converged state of the UID
-func (obj *converger) SetConverged(uuid ConvergerUUID, isConverged bool) error {
+func (obj *converger) SetConverged(uid ConvergerUID, isConverged bool) error {
-	if !uuid.IsValid() {
+	if !uid.IsValid() {
-		return fmt.Errorf("Id of ConvergerUUID(%s) is nil!", uuid.Name())
+		return fmt.Errorf("Id of ConvergerUID(%s) is nil!", uid.Name())
 	}
 	obj.mutex.Lock()
-	if _, found := obj.status[uuid.ID()]; !found {
+	if _, found := obj.status[uid.ID()]; !found {
-		panic("Id of ConvergerUUID is unregistered!")
+		panic("Id of ConvergerUID is unregistered!")
 	}
-	obj.status[uuid.ID()] = isConverged // set
+	obj.status[uid.ID()] = isConverged // set
 	obj.mutex.Unlock()                 // unlock *before* poke or deadlock!
 	if isConverged != obj.converged {  // only poke if it would be helpful
 		// run in a go routine so that we never block... just queue up!
@@ -143,7 +143,7 @@ func (obj *converger) SetConverged(uuid ConvergerUUID, isConverged bool) error {
 	return nil
 }
 
-// isConverged returns true if *every* registered uuid has converged
+// isConverged returns true if *every* registered uid has converged
 func (obj *converger) isConverged() bool {
 	obj.mutex.RLock() // take a read lock
 	defer obj.mutex.RUnlock()
@@ -155,16 +155,16 @@ func (obj *converger) isConverged() bool {
 	return true
 }
 
-// Unregister dissociates the ConvergedUUID from the converged checking
+// Unregister dissociates the ConvergedUID from the converged checking
-func (obj *converger) Unregister(uuid ConvergerUUID) {
+func (obj *converger) Unregister(uid ConvergerUID) {
-	if !uuid.IsValid() {
+	if !uid.IsValid() {
-		panic(fmt.Sprintf("Id of ConvergerUUID(%s) is nil!", uuid.Name()))
+		panic(fmt.Sprintf("Id of ConvergerUID(%s) is nil!", uid.Name()))
 	}
 	obj.mutex.Lock()
-	uuid.StopTimer() // ignore any errors
+	uid.StopTimer() // ignore any errors
-	delete(obj.status, uuid.ID())
+	delete(obj.status, uid.ID())
 	obj.mutex.Unlock()
-	uuid.InvalidateID()
+	uid.InvalidateID()
 }
 
 // Start causes a Converger object to start or resume running
@@ -245,18 +245,18 @@ func (obj *converger) Loop(startPaused bool) {
 
 // ConvergedTimer adds a timeout to a select call and blocks until then
 // TODO: this means we could eventually have per resource converged timeouts
-func (obj *converger) ConvergedTimer(uuid ConvergerUUID) <-chan time.Time {
+func (obj *converger) ConvergedTimer(uid ConvergerUID) <-chan time.Time {
 	// be clever: if i'm already converged, this timeout should block which
 	// avoids unnecessary new signals being sent! this avoids fast loops if
 	// we have a low timeout, or in particular a timeout == 0
-	if uuid.IsConverged() {
+	if uid.IsConverged() {
 		// blocks the case statement in select forever!
 		return util.TimeAfterOrBlock(-1)
 	}
 	return util.TimeAfterOrBlock(obj.timeout)
 }
 
-// Status returns a map of the converged status of each UUID.
+// Status returns a map of the converged status of each UID.
 func (obj *converger) Status() map[uint64]bool {
 	status := make(map[uint64]bool)
 	obj.mutex.RLock() // take a read lock
@@ -279,53 +279,53 @@ func (obj *converger) SetStateFn(stateFn func(bool) error) {
 	obj.stateFn = stateFn
 }
 
-// Id returns the unique id of this UUID object
+// Id returns the unique id of this UID object
-func (obj *convergerUUID) ID() uint64 {
+func (obj *convergerUID) ID() uint64 {
 	return obj.id
 }
 
-// Name returns a user defined name for the specific convergerUUID.
+// Name returns a user defined name for the specific convergerUID.
-func (obj *convergerUUID) Name() string {
+func (obj *convergerUID) Name() string {
 	return obj.name
 }
 
-// SetName sets a user defined name for the specific convergerUUID.
+// SetName sets a user defined name for the specific convergerUID.
-func (obj *convergerUUID) SetName(name string) {
+func (obj *convergerUID) SetName(name string) {
 	obj.name = name
 }
 
 // IsValid tells us if the id is valid or has already been destroyed
-func (obj *convergerUUID) IsValid() bool {
+func (obj *convergerUID) IsValid() bool {
 	return obj.id != 0 // an id of 0 is invalid
 }
 
 // InvalidateID marks the id as no longer valid
-func (obj *convergerUUID) InvalidateID() {
+func (obj *convergerUID) InvalidateID() {
 	obj.id = 0 // an id of 0 is invalid
 }
 
 // IsConverged is a helper function to the regular IsConverged method
-func (obj *convergerUUID) IsConverged() bool {
+func (obj *convergerUID) IsConverged() bool {
 	return obj.converger.IsConverged(obj)
 }
 
 // SetConverged is a helper function to the regular SetConverged notification
-func (obj *convergerUUID) SetConverged(isConverged bool) error {
+func (obj *convergerUID) SetConverged(isConverged bool) error {
 	return obj.converger.SetConverged(obj, isConverged)
 }
 
 // Unregister is a helper function to unregister myself
-func (obj *convergerUUID) Unregister() {
+func (obj *convergerUID) Unregister() {
 	obj.converger.Unregister(obj)
 }
 
 // ConvergedTimer is a helper around the regular ConvergedTimer method
-func (obj *convergerUUID) ConvergedTimer() <-chan time.Time {
+func (obj *convergerUID) ConvergedTimer() <-chan time.Time {
 	return obj.converger.ConvergedTimer(obj)
 }
 
 // StartTimer runs an invisible timer that automatically converges on timeout.
-func (obj *convergerUUID) StartTimer() (func() error, error) {
+func (obj *convergerUID) StartTimer() (func() error, error) {
 	obj.mutex.Lock()
 	if !obj.running {
 		obj.timer = make(chan struct{})
@@ -359,7 +359,7 @@ func (obj *convergerUUID) StartTimer() (func() error, error) {
 }
 
 // ResetTimer resets the counter to zero if using a StartTimer internally.
-func (obj *convergerUUID) ResetTimer() error {
+func (obj *convergerUID) ResetTimer() error {
 	obj.mutex.Lock()
 	defer obj.mutex.Unlock()
 	if obj.running {
@@ -370,7 +370,7 @@ func (obj *convergerUUID) ResetTimer() error {
 }
 
 // StopTimer stops the running timer permanently until a StartTimer is run.
-func (obj *convergerUUID) StopTimer() error {
+func (obj *convergerUID) StopTimer() error {
 	obj.mutex.Lock()
 	defer obj.mutex.Unlock()
 	if !obj.running {

docs/.gitignore

@@ -0,0 +1 @@
+mgmt-documentation.pdf

docs/documentation.md

@@ -23,7 +23,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 ####Available from:
 ####[https://github.com/purpleidea/mgmt/](https://github.com/purpleidea/mgmt/)
 
-####This documentation is available in: [Markdown](https://github.com/purpleidea/mgmt/blob/master/DOCUMENTATION.md) or [PDF](https://pdfdoc-purpleidea.rhcloud.com/pdf/https://github.com/purpleidea/mgmt/blob/master/DOCUMENTATION.md) format.
+####This documentation is available in: [Markdown](https://github.com/purpleidea/mgmt/blob/master/docs/documentation.md) or [PDF](https://pdfdoc-purpleidea.rhcloud.com/pdf/https://github.com/purpleidea/mgmt/blob/master/docs/documentation.md) format.
 
 ####Table of Contents
 
@@ -70,7 +70,7 @@ Older videos and other material [is available](https://github.com/purpleidea/mgm
 ##Setup
 
 During this prototype phase, the tool can be run out of the source directory.
-You'll probably want to use ```./run.sh run --file examples/graph1.yaml``` to
+You'll probably want to use ```./run.sh run --yaml examples/graph1.yaml``` to
 get started. Beware that this _can_ cause data loss. Understand what you're
 doing first, or perform these actions in a virtual environment such as the one
 provided by [Oh-My-Vagrant](https://github.com/purpleidea/oh-my-vagrant).
@@ -170,7 +170,8 @@ which need to exchange information that is only available at run time.
 
 ####Blog post
 
-An introductory blog post about this topic will follow soon.
+You can read the introductory blog post about this topic here:
+[https://ttboj.wordpress.com/2016/10/07/remote-execution-in-mgmt/](https://ttboj.wordpress.com/2016/10/07/remote-execution-in-mgmt/)
 
 ###Puppet support
 
@@ -217,11 +218,15 @@ parameter with the [Noop](#Noop) resource.
 
 * [Exec](#Exec): Execute shell commands on the system.
 * [File](#File): Manage files and directories.
+* [Hostname](#Hostname): Manages the hostname on the system.
 * [Msg](#Msg): Send log messages.
 * [Noop](#Noop): A simple resource that does nothing.
+* [Nspawn](#Nspawn): Manage systemd-machined nspawn containers.
+* [Password](#Password): Create random password strings.
 * [Pkg](#Pkg):  Manage system packages with PackageKit.
 * [Svc](#Svc): Manage system systemd services.
 * [Timer](#Timer): Manage system systemd services.
+* [Virt](#Virt): Manage virtual machines with libvirt.
 
 ###Exec
 
@@ -261,6 +266,30 @@ The force property is required if we want the file resource to be able to change
 a file into a directory or vice-versa. If such a change is needed, but the force
 property is not set to `true`, then this file resource will error.
 
+###Hostname
+
+The hostname resource manages static, transient/dynamic and pretty hostnames
+on the system and watches them for changes.
+
+#### static_hostname
+The static hostname is the one configured in /etc/hostname or a similar
+file.
+It is chosen by the local user. It is not always in sync with the current
+host name as returned by the gethostname() system call.
+
+#### transient_hostname
+The transient / dynamic hostname is the one configured via the kernel's
+sethostbyname().
+It can be different from the static hostname in case DHCP or mDNS have been
+configured to change the name based on network information.
+
+#### pretty_hostname
+The pretty hostname is a free-form UTF8 host name for presentation to the user.
+
+#### hostname
+Hostname is the fallback value for all 3 fields above, if only `hostname` is
+specified, it will set all 3 fields to this value.
+
 ###Msg
 
 The msg resource sends messages to the main log, or an external service such
@@ -271,6 +300,15 @@ as systemd's journal.
 The noop resource does absolutely nothing. It does have some utility in testing
 `mgmt` and also as a placeholder in the resource graph.
 
+###Nspawn
+
+The nspawn resource is used to manage systemd-machined style containers.
+
+###Password
+
+The password resource can generate a random string to be used as a password. It
+will re-generate the password if it receives a refresh notification.
+
 ###Pkg
 
 The pkg resource is used to manage system packages. This resource works on many
@@ -286,6 +324,10 @@ The service resource is still very WIP. Please help us my improving it!
 
 This resource needs better documentation. Please help us my improving it!
 
+###Virt
+
+The virt resource can manage virtual machines via libvirt.
+
 ##Usage and frequently asked questions
 (Send your questions as a patch to this FAQ! I'll review it, merge it, and
 respond by commit with the answer.)
@@ -333,7 +375,7 @@ starting up, and as a result, a default endpoint never gets added. The solution
 is to either reconcile the mistake, and if there is no important data saved, you
 can remove the etcd dataDir. This is typically `/var/lib/mgmt/etcd/member/`.
 
-###Why do resources have both a `Compare` method and an `IFF` (on the UUID) method?
+###Why do resources have both a `Compare` method and an `IFF` (on the UID) method?
 
 The `Compare()` methods are for determining if two resources are effectively the
 same, which is used to make graph change delta's efficient. This is when we want
@@ -342,9 +384,9 @@ vertices. Since we want to make this process efficient, we only update the parts
 that are different, and leave everything else alone. This `Compare()` method can
 tell us if two resources are the same.
 
-The `IFF()` method is part of the whole UUID system, which is for discerning if
+The `IFF()` method is part of the whole UID system, which is for discerning if a
-a resource meets the requirements another expects for an automatic edge. This is
+resource meets the requirements another expects for an automatic edge. This is
-because the automatic edge system assumes a unified UUID pattern to test for
+because the automatic edge system assumes a unified UID pattern to test for
 equality. In the future it might be helpful or sane to merge the two similar
 comparison functions although for now they are separate because they are
 actually answer different questions.
@@ -416,7 +458,7 @@ you can probably figure out most of it, as it's fairly intuitive.
 The main interface to the `mgmt` tool is the command line. For the most recent
 documentation, please run `mgmt --help`.
 
-####`--file <graph.yaml>`
+####`--yaml <graph.yaml>`
 Point to a graph file to run.
 
 ####`--converged-timeout <seconds>`

docs/puppet.md

@@ -14,7 +14,7 @@ This document goes into detail on how this works, and lists
 some pitfalls and limitations.
 
 For basic instructions on how to use the Puppet support, see
-the [main documentation](DOCUMENTATION.md#puppet-support).
+the [main documentation](documentation.md#puppet-support).
 
 ##Prerequisites
 

docs/resource-guide.md

@@ -0,0 +1,534 @@
+#mgmt
+
+<!--
+Mgmt
+Copyright (C) 2013-2016+ James Shubin and the project contributors
+Written by James Shubin <james@shubin.ca> and the project contributors
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+-->
+
+##mgmt resource guide by [James](https://ttboj.wordpress.com/)
+####Available from:
+####[https://github.com/purpleidea/mgmt/](https://github.com/purpleidea/mgmt/)
+
+####This documentation is available in: [Markdown](https://github.com/purpleidea/mgmt/blob/master/docs/resource-guide.md) or [PDF](https://pdfdoc-purpleidea.rhcloud.com/pdf/https://github.com/purpleidea/mgmt/blob/master/docs/resource-guide.md) format.
+
+####Table of Contents
+
+1. [Overview](#overview)
+2. [Theory - Resource theory in mgmt](#theory)
+3. [Resource API - Getting started with mgmt](#resource-api)
+	* [Init - Initialize the resource](#init)
+	* [CheckApply - Check and apply resource state](#checkapply)
+	* [Watch - Detect resource changes](#watch)
+	* [Compare - Compare resource with another](#compare)
+4. [Further considerations - More information about resource writing](#further-considerations)
+5. [Automatic edges - Adding automatic resources dependencies](#automatic-edges)
+6. [Automatic grouping - Grouping multiple resources into one](#automatic-grouping)
+7. [Send/Recv - Communication between resources](#send-recv)
+8. [Composite resources - Importing code from one resource into another](#composite-resources)
+9. [FAQ - Frequently asked questions](#frequently-asked-questions)
+10. [Suggestions - API change suggestions](#suggestions)
+11. [Authors - Authors and contact information](#authors)
+
+##Overview
+
+The `mgmt` tool has built-in resource primitives which make up the building
+blocks of any configuration. Each instance of a resource is mapped to a single
+vertex in the resource [graph](https://en.wikipedia.org/wiki/Directed_acyclic_graph).
+This guide is meant to instruct developers on how to write a brand new resource.
+Since `mgmt` and the core resources are written in golang, some prior golang
+knowledge is assumed.
+
+##Theory
+
+Resources in `mgmt` are similar to resources in other systems in that they are
+[idempotent](https://en.wikipedia.org/wiki/Idempotence). Our resources are
+uniquely different in that they can detect when their state has changed, and as
+a result can run to revert or repair this change instantly. For some background
+on this design, please read the
+[original article](https://ttboj.wordpress.com/2016/01/18/next-generation-configuration-mgmt/)
+on the subject.
+
+##Resource API
+
+To implement a resource in `mgmt` it must satisfy the
+[`Res`](https://github.com/purpleidea/mgmt/blob/master/resources/resources.go)
+interface. What follows are each of the method signatures and a description of
+each.
+
+###Init
+```golang
+Init() error
+```
+
+This is called to initialize the resource. If something goes wrong, it should
+return an error. It should set the resource `kind`, do any resource specific
+work, and finish by calling the `Init` method of the base resource.
+
+####Example
+```golang
+// Init initializes the Foo resource.
+func (obj *FooRes) Init() error {
+	obj.BaseRes.kind = "Foo" // must set capitalized resource kind
+	// run the resource specific initialization, and error if anything fails
+	if some_error {
+		return err // something went wrong!
+	}
+	return obj.BaseRes.Init() // call the base resource init
+}
+```
+
+###CheckApply
+```golang
+CheckApply(apply bool) (checkOK bool, err error)
+```
+
+`CheckApply` is where the real _work_ is done. Under normal circumstances, this
+function should check if the state of this resource is correct, and if so, it
+should return: `(true, nil)`. If the `apply` variable is set to `true`, then
+this means that we should then proceed to run the changes required to bring the
+resource into the correct state. If the `apply` variable is set to `false`, then
+the resource is operating in _noop_ mode and _no operations_ should be executed!
+
+After having executed the necessary operations to bring the resource back into
+the desired state, or after having detected that the state was incorrect, but
+that changes can't be made because `apply` is `false`, you should then return
+`(false, nil)`.
+
+You must cause the resource to converge during a single execution of this
+function. If you cannot, then you must return an error! The exception to this
+rule is that if an external force changes the state of the resource while it is
+being remedied, it is possible to return from this function even though the
+resource isn't now converged. This is not a bug, as the resources `Watch`
+facility will detect the change, ultimately resulting in a subsequent call to
+`CheckApply`.
+
+####Example
+```golang
+// CheckApply does the idempotent work of checking and applying resource state.
+func (obj *FooRes) CheckApply(apply bool) (bool, error) {
+	// check the state
+	if state_is_okay { return true, nil } // done early! :)
+	// state was bad
+	if !apply { return false, nil } // don't apply; !stateok, nil
+	// do the apply!
+	return false, nil // after success applying
+	if any_error { return false, err } // anytime there's an err!
+}
+```
+
+The `CheckApply` function is called by the `mgmt` engine when it believes a call
+is necessary. Under certain conditions when a `Watch` call does not invalidate
+the state of the resource, and no refresh call was sent, its execution might be
+skipped. This is an engine optimization, and not a bug. It is mentioned here in
+the documentation in case you are confused as to why a debug message you've
+added to the code isn't always printed.
+
+####Refresh notifications
+Some resources may choose to support receiving refresh notifications. In general
+these should be avoided if possible, but nevertheless, they do make sense in
+certain situations. Resources that support these need to verify if one was sent
+during the CheckApply phase of execution. This is accomplished by calling the
+`Refresh() bool` method of the resource, and inspecting the return value. This
+is only necessary if you plan to perform a refresh action. Refresh actions
+should still respect the `apply` variable, and no system changes should be made
+if it is `false`. Refresh notifications are generated by any resource when an
+action is applied by that resource and are transmitted through graph edges which
+have enabled their propagation. Resources that currently perform some refresh
+action include `svc`, `timer`, and `password`.
+
+####Paired execution
+For many resources it is not uncommon to see `CheckApply` run twice in rapid
+succession. This is usually not a pathological occurrence, but rather a healthy
+pattern which is a consequence of the event system. When the state of the
+resource is incorrect, `CheckApply` will run to remedy the state. In response to
+having just changed the state, it is usually the case that this repair will
+trigger the `Watch` code! In response, a second `CheckApply` is triggered, which
+will likely find the state to now be correct.
+
+####Summary
+* Anytime an error occurs during `CheckApply`, you should return `(false, err)`.
+* If the state is correct and no changes are needed, return `(true, nil)`.
+* You should only make changes to the system if `apply` is set to `true`.
+* After checking the state and possibly applying the fix, return `(false, nil)`.
+* Returning `(true, err)` is a programming error and will cause a `Fatal`.
+
+###Watch
+```golang
+Watch(chan Event) error
+```
+
+`Watch` is a main loop that runs and sends messages when it detects that the
+state of the resource might have changed. To send a message you should write to
+the input `Event` channel using the `DoSend` helper method. The Watch function
+should run continuously until a shutdown message is received. If at any time
+something goes wrong, you should return an error, and the `mgmt` engine will
+handle possibly restarting the main loop based on the `retry` meta parameters.
+
+It is better to send an event notification which turns out to be spurious, than
+to miss a possible event. Resources which can miss events are incorrect and need
+to be re-engineered so that this isn't the case. If you have an idea for a
+resource which would fit this criteria, but you can't find a solution, please
+contact the `mgmt` maintainers so that this problem can be investigated and a
+possible system level engineering fix can be found.
+
+You may have trouble deciding how much resource state checking should happen in
+the `Watch` loop versus deferring it all to the `CheckApply` method. You may
+want to put some simple fast path checking in `Watch` to avoid generating
+obviously spurious events, but in general it's best to keep the `Watch` method
+as simple as possible. Contact the `mgmt` maintainers if you're not sure.
+
+If the resource is activated in `polling` mode, the `Watch` method will not get
+executed. As a result, the resource must still work even if the main loop is not
+running.
+
+####Select
+The lifetime of most resources `Watch` method should be spent in an infinite
+loop that is bounded by a `select` call. The `select` call is the point where
+our method hands back control to the engine (and the kernel) so that we can
+sleep until something of interest wakes us up. In this loop we must process
+events from the engine via the `<-obj.Events()` call, wait for the converged
+timeout with `<-cuid.ConvergedTimer()`, and receive events for our resource
+itself!
+
+####Events
+If we receive an internal event from the `<-obj.Events()` method, we can read it
+with the ReadEvent helper function. This function tells us if we should shutdown
+our resource, and if we should generate an event. When we want to send an event,
+we use the `DoSend` helper function. It is also important to mark the resource
+state as `dirty` if we believe it might have changed. We do this with the
+`StateOK(false)` function.
+
+####Startup
+Once the `Watch` function has finished starting up successfully, it is important
+to generate one event to notify the `mgmt` engine that we're now listening
+successfully, so that it can run an initial `CheckApply` to ensure we're safely
+tracking a healthy state and that we didn't miss anything when `Watch` was down
+or from before `mgmt` was running. It does this by calling the `Running` method.
+
+####Converged
+The engine might be asked to shutdown when the entire state of the system has
+not seen any changes for some duration of time. In order for the engine to be
+able to make this determination, each resource must report its converged state.
+To do this, the `Watch` method should get the `ConvergedUID` handle that has
+been prepared for it by the engine. This is done by calling the `Converger`
+method on the resource object. The result can be used to set the converged
+status with `SetConverged`, and to notify when the particular timeout has been
+reached by waiting on `ConvergedTimer`.
+
+Instead of interacting with the `ConvergedUID` with these two methods, we can
+instead use the `StartTimer` and `ResetTimer` methods which accomplish the same
+thing, but provide a `select`-free interface for different coding situations.
+
+####Example
+```golang
+// Watch is the listener and main loop for this resource.
+func (obj *FooRes) Watch(processChan chan event.Event) error {
+	cuid := obj.Converger() // get the converger uid used to report status
+
+	// setup the Foo resource
+	var err error
+	if err, obj.foo = OpenFoo(); err != nil {
+		return err // we couldn't startup
+	}
+	defer obj.whatever.CloseFoo() // shutdown our
+
+	// notify engine that we're running
+	if err := obj.Running(processChan); err != nil {
+		return err // bubble up a NACK...
+	}
+
+	var send = false // send event?
+	var exit = false
+	for {
+		obj.SetState(ResStateWatching) // reset
+		select {
+		case event := <-obj.Events():
+			cuid.SetConverged(false)
+			// we avoid sending events on unpause
+			if exit, send = obj.ReadEvent(&event); exit {
+				return nil // exit
+			}
+
+		// the actual events!
+		case event := <-obj.foo.Events:
+			if is_an_event {
+				send = true // used below
+				cuid.SetConverged(false)
+				obj.StateOK(false) // dirty
+			}
+
+		// event errors
+		case err := <-obj.foo.Errors:
+			cuuid.SetConverged(false)
+			return err // will cause a retry or permanent failure
+
+		case <-cuid.ConvergedTimer():
+			cuid.SetConverged(true) // converged!
+			continue
+		}
+
+		// do all our event sending all together to avoid duplicate msgs
+		if send {
+			send = false
+			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
+				return err // we exit or bubble up a NACK...
+			}
+		}
+	}
+}
+```
+
+####Summary
+* Remember to call the appropriate `converger` methods throughout the resource.
+* Remember to call `Startup` when the `Watch` is running successfully.
+* Remember to process internal events and shutdown promptly if asked to.
+* Ensure the design of your resource is well thought out.
+* Have a look at the existing resources for a rough idea of how this all works.
+
+###Compare
+```golang
+Compare(Res) bool
+```
+
+Each resource must have a `Compare` method. This takes as input another resource
+and must return whether they are identical or not. This is used for identifying
+if an existing resource can be used in place of a new one with a similar set of
+parameters. In particular, when switching from one graph to a new (possibly
+identical) graph, this avoids recomputing the state for resources which don't
+change or that are sufficiently similar that they don't need to be swapped out.
+
+In general if all the resource properties are identical, then they usually don't
+need to be changed. On occasion, not all of them need to be compared, in
+particular if they store some generated state, or if they aren't significant in
+some way.
+
+####Example
+```golang
+// Compare two resources and return if they are equivalent.
+func (obj *FooRes) Compare(res Res) bool {
+	switch res.(type) {
+	case *FooRes: // only compare to other resources of the Foo kind!
+		res := res.(*FileRes)
+		if !obj.BaseRes.Compare(res) { // call base Compare
+			return false
+		}
+		if obj.Name != res.Name {
+			return false
+		}
+		if obj.whatever != res.whatever {
+			return false
+		}
+		if obj.Flag != res.Flag {
+			return false
+		}
+	default:
+		return false // different kind of resource
+	}
+	return true // they must match!
+}
+```
+
+###Validate
+```golang
+Validate() error
+```
+
+This method is used to validate if the populated resource struct is a valid
+representation of the resource kind. If it does not conform to the resource
+specifications, it should generate an error. If you notice that this method is
+quite large, it might be an indication that you might want to reconsider the
+parameter list and interface to this resource.
+
+###GetUIDs
+```golang
+GetUIDs() []ResUID
+```
+
+The `GetUIDs` method returns a list of `ResUID` interfaces that represent the
+particular resource uniquely. This is used with the AutoEdges API to determine
+if another resource can match a dependency to this one.
+
+###AutoEdges
+```golang
+AutoEdges() AutoEdge
+```
+
+This returns a struct that implements the `AutoEdge` interface. This struct
+is used to match other resources that might be relevant dependencies for this
+resource.
+
+###CollectPattern
+```golang
+CollectPattern() string
+```
+
+This is currently a stub and will be updated once the DSL is further along.
+
+##Further considerations
+There is some additional information that any resource writer will need to know.
+Each issue is listed separately below!
+
+###Resource struct
+Each resource will implement methods as pointer receivers on a resource struct.
+The resource struct must include an anonymous reference to the `BaseRes` struct.
+The naming convention for resources is that they end with a `Res` suffix. If
+you'd like your resource to be accessible by the `YAML` graph API (GAPI), then
+you'll need to include the appropriate YAML fields as shown below.
+
+####Example
+```golang
+type FooRes struct {
+	BaseRes `yaml:",inline"` // base properties
+
+	Whatever string `yaml:"whatever"` // you pick!
+	Bar int // no yaml, used as public output value for send/recv
+	Baz bool `yaml:"baz"` // something else
+
+	something string // some private field
+}
+```
+
+###YAML
+In addition to labelling your resource struct with YAML fields, you must also
+add an entry to the internal `GraphConfig` struct. It is a fairly straight
+forward one line patch.
+
+```golang
+type GraphConfig struct {
+// [snip...]
+	Resources struct {
+		Noop []*resources.NoopRes `yaml:"noop"`
+		File []*resources.FileRes `yaml:"file"`
+		// [snip...]
+		Foo []*resources.FooRes `yaml:"foo"` // tada :)
+	}
+}
+```
+
+###Gob registration
+All resources must be registered with the `golang` _gob_ module so that they can
+be encoded and decoded. Make sure to include the following code snippet for this
+to work.
+
+```golang
+import "encoding/gob"
+func init() { // special golang method that runs once
+	gob.Register(&FooRes{}) // substitude your resource here
+}
+```
+
+##Automatic edges
+Automatic edges in `mgmt` are well described in [this article](https://ttboj.wordpress.com/2016/03/14/automatic-edges-in-mgmt/).
+The best example of this technique can be seen in the `svc` resource.
+Unfortunately no further documentation about this subject has been written. To
+expand this section, please send a patch! Please contact us if you'd like to
+work on a resource that uses this feature, or to add it to an existing one!
+
+##Automatic grouping
+Automatic grouping in `mgmt` is well described in [this article](https://ttboj.wordpress.com/2016/03/30/automatic-grouping-in-mgmt/).
+The best example of this technique can be seen in the `pkg` resource.
+Unfortunately no further documentation about this subject has been written. To
+expand this section, please send a patch! Please contact us if you'd like to
+work on a resource that uses this feature, or to add it to an existing one!
+
+
+##Send/Recv
+In `mgmt` there is a novel concept called _Send/Recv_. For some background,
+please [read the introductory article](https://ttboj.wordpress.com/2016/12/07/sendrecv-in-mgmt/).
+When using this feature, the engine will automatically send the user specified
+value to the intended destination without requiring any resource specific code.
+Any time that one of the destination values is changed, the engine automatically
+marks the resource state as `dirty`. To detect if a particular value was
+received, and if it changed (during this invocation of CheckApply) from the
+previous value, you can query the Recv parameter. It will contain a `map` of all
+the keys which can be received on, and the value has a `Changed` property which
+will indicate whether the value was updated on this particular `CheckApply`
+invocation. The type of the sending key must match that of the receiving one.
+This can _only_ be done inside of the `CheckApply` function!
+
+```golang
+// inside CheckApply, probably near the top
+if val, exists := obj.Recv["SomeKey"]; exists {
+	log.Printf("SomeKey was sent to us from: %s[%s].%s", val.Res.Kind(), val.Res.GetName(), val.Key)
+	if val.Changed {
+		log.Printf("SomeKey was just updated!")
+		// you may want to invalidate some local cache
+	}
+}
+```
+
+Astute readers will note that there isn't anything that prevents a user from
+sending an identically typed value to some arbitrary (public) key that the
+resource author hadn't considered! While this is true, resources should probably
+work within this problem space anyways. The rule of thumb is that any public
+parameter which is normally used in a resource can be used safely.
+
+One subtle scenario is that if a resource creates a local cache or stores a
+computation that depends on the value of a public parameter and will require
+invalidation should that public parameter change, then you must detect that
+scenario and invalidate the cache when it occurs. This *must* be processed
+before there is a possibility of failure in CheckApply, because if we fail (and
+possibly run again) the subsequent send->recv transfer might not have a new
+value to copy, and therefore we won't see this notification of change.
+Therefore, it is important to process these promptly, if they must not be lost,
+such as for cache invalidation.
+
+Remember, `Send/Recv` only changes your resource code if you cache state.
+
+##Composite resources
+Composite resources are resources which embed one or more existing resources.
+This is useful to prevent code duplication in higher level resource scenarios.
+The best example of this technique can be seen in the `nspawn` resource which
+can be seen to partially embed a `svc` resource, but without its `Watch`.
+Unfortunately no further documentation about this subject has been written. To
+expand this section, please send a patch! Please contact us if you'd like to
+work on a resource that uses this feature, or to add it to an existing one!
+
+##Frequently asked questions
+(Send your questions as a patch to this FAQ! I'll review it, merge it, and
+respond by commit with the answer.)
+
+###Can I write resources in a different language?
+Currently `golang` is the only supported language for built-in resources. We
+might consider allowing external resources to be imported in the future. This
+will likely require a language that can expose a C-like API, such as `python` or
+`ruby`. Custom `golang` resources are already possible when using mgmt as a lib.
+Higher level resource collections will be possible once the `mgmt` DSL is ready.
+
+###What new resource primitives need writing?
+There are still many ideas for new resources that haven't been written yet. If
+you'd like to contribute one, please contact us and tell us about your idea!
+
+###Where can I find more information about mgmt?
+Additional blog posts, videos and other material [is available!](https://github.com/purpleidea/mgmt/#on-the-web).
+
+##Suggestions
+If you have any ideas for API changes or other improvements to resource writing,
+please let us know! We're still pre 1.0 and pre 0.1 and happy to break API in
+order to get it right!
+
+##Authors
+
+Copyright (C) 2013-2016+ James Shubin and the project contributors
+
+Please see the
+[AUTHORS](https://github.com/purpleidea/mgmt/tree/master/AUTHORS) file
+for more information.
+
+* [github](https://github.com/purpleidea/)
+* [&#64;purpleidea](https://twitter.com/#!/purpleidea)
+* [https://ttboj.wordpress.com/](https://ttboj.wordpress.com/)

etcd/etcd.go

@@ -36,11 +36,12 @@
 // * The elected leader should decide who to nominate/unnominate to keep the right number of servers.
 //
 // Smoke testing:
-// ./mgmt run --file examples/etcd1a.yaml --hostname h1
+// mkdir /tmp/mgmt{A..E}
-// ./mgmt run --file examples/etcd1b.yaml --hostname h2 --tmp-prefix --seeds http://127.0.0.1:2379 --client-urls http://127.0.0.1:2381 --server-urls http://127.0.0.1:2382
+// ./mgmt run --yaml examples/etcd1a.yaml --hostname h1 --tmp-prefix
-// ./mgmt run --file examples/etcd1c.yaml --hostname h3 --tmp-prefix --seeds http://127.0.0.1:2379 --client-urls http://127.0.0.1:2383 --server-urls http://127.0.0.1:2384
+// ./mgmt run --yaml examples/etcd1b.yaml --hostname h2 --tmp-prefix --seeds http://127.0.0.1:2379 --client-urls http://127.0.0.1:2381 --server-urls http://127.0.0.1:2382
+// ./mgmt run --yaml examples/etcd1c.yaml --hostname h3 --tmp-prefix --seeds http://127.0.0.1:2379 --client-urls http://127.0.0.1:2383 --server-urls http://127.0.0.1:2384
 // ETCDCTL_API=3 etcdctl --endpoints 127.0.0.1:2379 put /_mgmt/idealClusterSize 3
-// ./mgmt run --file examples/etcd1d.yaml --hostname h4 --tmp-prefix --seeds http://127.0.0.1:2379 --client-urls http://127.0.0.1:2385 --server-urls http://127.0.0.1:2386
+// ./mgmt run --yaml examples/etcd1d.yaml --hostname h4 --tmp-prefix --seeds http://127.0.0.1:2379 --client-urls http://127.0.0.1:2385 --server-urls http://127.0.0.1:2386
 // ETCDCTL_API=3 etcdctl --endpoints 127.0.0.1:2379 member list
 // ETCDCTL_API=3 etcdctl --endpoints 127.0.0.1:2381 put /_mgmt/idealClusterSize 5
 // ETCDCTL_API=3 etcdctl --endpoints 127.0.0.1:2381 member list
@@ -63,7 +64,6 @@ import (
 
 	"github.com/purpleidea/mgmt/converger"
 	"github.com/purpleidea/mgmt/event"
-	"github.com/purpleidea/mgmt/global"
 	"github.com/purpleidea/mgmt/resources"
 	"github.com/purpleidea/mgmt/util"
 
@@ -153,6 +153,13 @@ type TN struct {
 	data    *etcd.TxnResponse
 }
 
+// Flags are some constant flags which are used throughout the program.
+type Flags struct {
+	Debug   bool // add additional log messages
+	Trace   bool // add execution flow log messages
+	Verbose bool // add extra log message output
+}
+
 // EmbdEtcd provides the embedded server and client etcd functionality
 type EmbdEtcd struct { // EMBeddeD etcd
 	// etcd client connection related
@@ -189,6 +196,7 @@ type EmbdEtcd struct { // EMBeddeD etcd
 	delq    chan *DL // delete queue
 	txnq    chan *TN // txn queue
 
+	flags     Flags
 	prefix    string              // folder prefix to use for misc storage
 	converger converger.Converger // converged tracking
 
@@ -199,7 +207,7 @@ type EmbdEtcd struct { // EMBeddeD etcd
 }
 
 // NewEmbdEtcd creates the top level embedded etcd struct client and server obj
-func NewEmbdEtcd(hostname string, seeds, clientURLs, serverURLs etcdtypes.URLs, noServer bool, idealClusterSize uint16, prefix string, converger converger.Converger) *EmbdEtcd {
+func NewEmbdEtcd(hostname string, seeds, clientURLs, serverURLs etcdtypes.URLs, noServer bool, idealClusterSize uint16, flags Flags, prefix string, converger converger.Converger) *EmbdEtcd {
 	endpoints := make(etcdtypes.URLsMap)
 	if hostname == seedSentinel { // safety
 		return nil
@@ -228,6 +236,7 @@ func NewEmbdEtcd(hostname string, seeds, clientURLs, serverURLs etcdtypes.URLs,
 
 		idealClusterSize: idealClusterSize,
 		converger:        converger,
+		flags:            flags,
 		prefix:           prefix,
 		dataDir:          path.Join(prefix, "etcd"),
 	}
@@ -272,7 +281,7 @@ func (obj *EmbdEtcd) GetConfig() etcd.Config {
 // Connect connects the client to a server, and then builds the *API structs.
 // If reconnect is true, it will force a reconnect with new config endpoints.
 func (obj *EmbdEtcd) Connect(reconnect bool) error {
-	if global.DEBUG {
+	if obj.flags.Debug {
 		log.Println("Etcd: Connect...")
 	}
 	obj.cLock.Lock()
@@ -307,7 +316,7 @@ func (obj *EmbdEtcd) Connect(reconnect bool) error {
 			if emax > maxClientConnectRetries {
 				log.Printf("Etcd: The dataDir (%s) might be inconsistent or corrupt.", obj.dataDir)
 				log.Printf("Etcd: Please see: %s", "https://github.com/purpleidea/mgmt/blob/master/DOCUMENTATION.md#what-does-the-error-message-about-an-inconsistent-datadir-mean")
-				obj.cError = fmt.Errorf("Can't find an available endpoint.")
+				obj.cError = fmt.Errorf("can't find an available endpoint")
 				return obj.cError
 			}
 			err = &CtxDelayErr{time.Duration(emax) * time.Second, "No endpoints available yet!"} // retry with backoff...
@@ -528,29 +537,29 @@ func (obj *EmbdEtcd) CtxError(ctx context.Context, err error) (context.Context,
 	var isTimeout = false
 	var iter int // = 0
 	if ctxerr, ok := ctx.Value(ctxErr).(error); ok {
-		if global.DEBUG {
+		if obj.flags.Debug {
 			log.Printf("Etcd: CtxError: err(%v), ctxerr(%v)", err, ctxerr)
 		}
 		if i, ok := ctx.Value(ctxIter).(int); ok {
 			iter = i + 1 // load and increment
-			if global.DEBUG {
+			if obj.flags.Debug {
 				log.Printf("Etcd: CtxError: Iter: %v", iter)
 			}
 		}
 		isTimeout = err == context.DeadlineExceeded
-		if global.DEBUG {
+		if obj.flags.Debug {
 			log.Printf("Etcd: CtxError: isTimeout: %v", isTimeout)
 		}
 		if !isTimeout {
 			iter = 0 // reset timer
 		}
 		err = ctxerr // restore error
-	} else if global.DEBUG {
+	} else if obj.flags.Debug {
 		log.Printf("Etcd: CtxError: No value found")
 	}
 	ctxHelper := func(tmin, texp, tmax int) context.Context {
 		t := expBackoff(tmin, texp, iter, tmax)
-		if global.DEBUG {
+		if obj.flags.Debug {
 			log.Printf("Etcd: CtxError: Timeout: %v", t)
 		}
 
@@ -637,13 +646,13 @@ func (obj *EmbdEtcd) CtxError(ctx context.Context, err error) (context.Context,
 		fallthrough
 	case isGrpc(grpc.ErrClientConnClosing):
 
-		if global.DEBUG {
+		if obj.flags.Debug {
 			log.Printf("Etcd: CtxError: Error(%T): %+v", err, err)
 			log.Printf("Etcd: Endpoints are: %v", obj.client.Endpoints())
 			log.Printf("Etcd: Client endpoints are: %v", obj.endpoints)
 		}
 
-		if global.DEBUG {
+		if obj.flags.Debug {
 			log.Printf("Etcd: CtxError: Locking...")
 		}
 		obj.rLock.Lock()
@@ -664,7 +673,7 @@ func (obj *EmbdEtcd) CtxError(ctx context.Context, err error) (context.Context,
 			obj.ctxErr = fmt.Errorf("Etcd: Permanent connect error: %v", err)
 			return ctx, obj.ctxErr
 		}
-		if global.DEBUG {
+		if obj.flags.Debug {
 			log.Printf("Etcd: CtxError: Unlocking...")
 		}
 		obj.rLock.Unlock()
@@ -691,24 +700,24 @@ func (obj *EmbdEtcd) CtxError(ctx context.Context, err error) (context.Context,
 
 // CbLoop is the loop where callback execution is serialized
 func (obj *EmbdEtcd) CbLoop() {
-	cuuid := obj.converger.Register()
+	cuid := obj.converger.Register()
-	cuuid.SetName("Etcd: CbLoop")
+	cuid.SetName("Etcd: CbLoop")
-	defer cuuid.Unregister()
+	defer cuid.Unregister()
 	if e := obj.Connect(false); e != nil {
 		return // fatal
 	}
 	// we use this timer because when we ignore un-converge events and loop,
 	// we reset the ConvergedTimer case statement, ruining the timeout math!
-	cuuid.StartTimer()
+	cuid.StartTimer()
 	for {
 		ctx := context.Background() // TODO: inherit as input argument?
 		select {
 		// etcd watcher event
 		case re := <-obj.wevents:
 			if !re.skipConv { // if we want to count it...
-				cuuid.ResetTimer() // activity!
+				cuid.ResetTimer() // activity!
 			}
-			if global.TRACE {
+			if obj.flags.Trace {
 				log.Printf("Trace: Etcd: CbLoop: Event: StartLoop")
 			}
 			for {
@@ -716,11 +725,11 @@ func (obj *EmbdEtcd) CbLoop() {
 					//re.resp.NACK() // nope!
 					break
 				}
-				if global.TRACE {
+				if obj.flags.Trace {
 					log.Printf("Trace: Etcd: CbLoop: rawCallback()")
 				}
 				err := rawCallback(ctx, re)
-				if global.TRACE {
+				if obj.flags.Trace {
 					log.Printf("Trace: Etcd: CbLoop: rawCallback(): %v", err)
 				}
 				if err == nil {
@@ -732,14 +741,14 @@ func (obj *EmbdEtcd) CbLoop() {
 					break // TODO: it's bad, break or return?
 				}
 			}
-			if global.TRACE {
+			if obj.flags.Trace {
 				log.Printf("Trace: Etcd: CbLoop: Event: FinishLoop")
 			}
 
 		// exit loop commit
 		case <-obj.exitTimeout:
 			log.Println("Etcd: Exiting callback loop!")
-			cuuid.StopTimer() // clean up nicely
+			cuid.StopTimer() // clean up nicely
 			return
 		}
 	}
@@ -747,24 +756,24 @@ func (obj *EmbdEtcd) CbLoop() {
 
 // Loop is the main loop where everything is serialized
 func (obj *EmbdEtcd) Loop() {
-	cuuid := obj.converger.Register()
+	cuid := obj.converger.Register()
-	cuuid.SetName("Etcd: Loop")
+	cuid.SetName("Etcd: Loop")
-	defer cuuid.Unregister()
+	defer cuid.Unregister()
 	if e := obj.Connect(false); e != nil {
 		return // fatal
 	}
-	cuuid.StartTimer()
+	cuid.StartTimer()
 	for {
 		ctx := context.Background() // TODO: inherit as input argument?
 		// priority channel...
 		select {
 		case aw := <-obj.awq:
-			cuuid.ResetTimer() // activity!
+			cuid.ResetTimer() // activity!
-			if global.TRACE {
+			if obj.flags.Trace {
 				log.Printf("Trace: Etcd: Loop: PriorityAW: StartLoop")
 			}
 			obj.loopProcessAW(ctx, aw)
-			if global.TRACE {
+			if obj.flags.Trace {
 				log.Printf("Trace: Etcd: Loop: PriorityAW: FinishLoop")
 			}
 			continue // loop to drain the priority channel first!
@@ -775,19 +784,19 @@ func (obj *EmbdEtcd) Loop() {
 		select {
 		// add watcher
 		case aw := <-obj.awq:
-			cuuid.ResetTimer() // activity!
+			cuid.ResetTimer() // activity!
-			if global.TRACE {
+			if obj.flags.Trace {
 				log.Printf("Trace: Etcd: Loop: AW: StartLoop")
 			}
 			obj.loopProcessAW(ctx, aw)
-			if global.TRACE {
+			if obj.flags.Trace {
 				log.Printf("Trace: Etcd: Loop: AW: FinishLoop")
 			}
 
 		// set kv pair
 		case kv := <-obj.setq:
-			cuuid.ResetTimer() // activity!
+			cuid.ResetTimer() // activity!
-			if global.TRACE {
+			if obj.flags.Trace {
 				log.Printf("Trace: Etcd: Loop: Set: StartLoop")
 			}
 			for {
@@ -804,16 +813,16 @@ func (obj *EmbdEtcd) Loop() {
 					break // TODO: it's bad, break or return?
 				}
 			}
-			if global.TRACE {
+			if obj.flags.Trace {
 				log.Printf("Trace: Etcd: Loop: Set: FinishLoop")
 			}
 
 		// get value
 		case gq := <-obj.getq:
 			if !gq.skipConv {
-				cuuid.ResetTimer() // activity!
+				cuid.ResetTimer() // activity!
 			}
-			if global.TRACE {
+			if obj.flags.Trace {
 				log.Printf("Trace: Etcd: Loop: Get: StartLoop")
 			}
 			for {
@@ -831,14 +840,14 @@ func (obj *EmbdEtcd) Loop() {
 					break // TODO: it's bad, break or return?
 				}
 			}
-			if global.TRACE {
+			if obj.flags.Trace {
 				log.Printf("Trace: Etcd: Loop: Get: FinishLoop")
 			}
 
 		// delete value
 		case dl := <-obj.delq:
-			cuuid.ResetTimer() // activity!
+			cuid.ResetTimer() // activity!
-			if global.TRACE {
+			if obj.flags.Trace {
 				log.Printf("Trace: Etcd: Loop: Delete: StartLoop")
 			}
 			for {
@@ -856,14 +865,14 @@ func (obj *EmbdEtcd) Loop() {
 					break // TODO: it's bad, break or return?
 				}
 			}
-			if global.TRACE {
+			if obj.flags.Trace {
 				log.Printf("Trace: Etcd: Loop: Delete: FinishLoop")
 			}
 
 		// run txn
 		case tn := <-obj.txnq:
-			cuuid.ResetTimer() // activity!
+			cuid.ResetTimer() // activity!
-			if global.TRACE {
+			if obj.flags.Trace {
 				log.Printf("Trace: Etcd: Loop: Txn: StartLoop")
 			}
 			for {
@@ -881,7 +890,7 @@ func (obj *EmbdEtcd) Loop() {
 					break // TODO: it's bad, break or return?
 				}
 			}
-			if global.TRACE {
+			if obj.flags.Trace {
 				log.Printf("Trace: Etcd: Loop: Txn: FinishLoop")
 			}
 
@@ -897,7 +906,7 @@ func (obj *EmbdEtcd) Loop() {
 		// exit loop commit
 		case <-obj.exitTimeout:
 			log.Println("Etcd: Exiting loop!")
-			cuuid.StopTimer() // clean up nicely
+			cuid.StopTimer() // clean up nicely
 			return
 		}
 	}
@@ -935,7 +944,7 @@ func (obj *EmbdEtcd) Set(key, value string, opts ...etcd.OpOption) error {
 
 // rawSet actually implements the key set operation
 func (obj *EmbdEtcd) rawSet(ctx context.Context, kv *KV) error {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: rawSet()")
 	}
 	// key is the full key path
@@ -944,7 +953,7 @@ func (obj *EmbdEtcd) rawSet(ctx context.Context, kv *KV) error {
 	response, err := obj.client.KV.Put(ctx, kv.key, kv.value, kv.opts...)
 	obj.rLock.RUnlock()
 	log.Printf("Etcd: Set(%s): %v", kv.key, response) // w00t... bonus
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: rawSet(): %v", err)
 	}
 	return err
@@ -969,7 +978,7 @@ func (obj *EmbdEtcd) ComplexGet(path string, skipConv bool, opts ...etcd.OpOptio
 }
 
 func (obj *EmbdEtcd) rawGet(ctx context.Context, gq *GQ) (result map[string]string, err error) {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: rawGet()")
 	}
 	obj.rLock.RLock()
@@ -985,7 +994,7 @@ func (obj *EmbdEtcd) rawGet(ctx context.Context, gq *GQ) (result map[string]stri
 		result[bytes.NewBuffer(x.Key).String()] = bytes.NewBuffer(x.Value).String()
 	}
 
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: rawGet(): %v", result)
 	}
 	return
@@ -1003,7 +1012,7 @@ func (obj *EmbdEtcd) Delete(path string, opts ...etcd.OpOption) (int64, error) {
 }
 
 func (obj *EmbdEtcd) rawDelete(ctx context.Context, dl *DL) (count int64, err error) {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: rawDelete()")
 	}
 	count = -1
@@ -1013,7 +1022,7 @@ func (obj *EmbdEtcd) rawDelete(ctx context.Context, dl *DL) (count int64, err er
 	if err == nil {
 		count = response.Deleted
 	}
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: rawDelete(): %v", err)
 	}
 	return
@@ -1031,13 +1040,13 @@ func (obj *EmbdEtcd) Txn(ifcmps []etcd.Cmp, thenops, elseops []etcd.Op) (*etcd.T
 }
 
 func (obj *EmbdEtcd) rawTxn(ctx context.Context, tn *TN) (*etcd.TxnResponse, error) {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: rawTxn()")
 	}
 	obj.rLock.RLock()
 	response, err := obj.client.KV.Txn(ctx).If(tn.ifcmps...).Then(tn.thenops...).Else(tn.elseops...).Commit()
 	obj.rLock.RUnlock()
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: rawTxn(): %v, %v", response, err)
 	}
 	return response, err
@@ -1071,7 +1080,7 @@ func (obj *EmbdEtcd) rawAddWatcher(ctx context.Context, aw *AW) (func(), error)
 			err := response.Err()
 			isCanceled := response.Canceled || err == context.Canceled
 			if response.Header.Revision == 0 { // by inspection
-				if global.DEBUG {
+				if obj.flags.Debug {
 					log.Printf("Etcd: Watch: Received empty message!") // switched client connection
 				}
 				isCanceled = true
@@ -1085,14 +1094,14 @@ func (obj *EmbdEtcd) rawAddWatcher(ctx context.Context, aw *AW) (func(), error)
 			}
 
 			if err == nil { // watch from latest good revision
-				rev = response.Header.Revision // TODO +1 ?
+				rev = response.Header.Revision // TODO: +1 ?
 				useRev = true
 				if !locked {
 					retry = false
 				}
 				locked = false
 			} else {
-				if global.DEBUG {
+				if obj.flags.Debug {
 					log.Printf("Etcd: Watch: Error: %v", err) // probably fixable
 				}
 				// this new context is the fix for a tricky set
@@ -1141,9 +1150,6 @@ func rawCallback(ctx context.Context, re *RE) error {
 			// NOTE: the callback must *not* block!
 			// FIXME: do we need to pass ctx in via *RE, or in the callback signature ?
 			err = callback(re) // run the callback
-			if global.TRACE {
-				log.Printf("Trace: Etcd: rawCallback(): %v", err)
-			}
 			if !re.errCheck || err == nil {
 				return nil
 			}
@@ -1159,7 +1165,7 @@ func rawCallback(ctx context.Context, re *RE) error {
 // FIXME: we might need to respond to member change/disconnect/shutdown events,
 // see: https://github.com/coreos/etcd/issues/5277
 func (obj *EmbdEtcd) volunteerCallback(re *RE) error {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: volunteerCallback()")
 		defer log.Printf("Trace: Etcd: volunteerCallback(): Finished!")
 	}
@@ -1347,7 +1353,7 @@ func (obj *EmbdEtcd) volunteerCallback(re *RE) error {
 // nominateCallback runs to respond to the nomination list change events
 // functionally, it controls the starting and stopping of the server process
 func (obj *EmbdEtcd) nominateCallback(re *RE) error {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: nominateCallback()")
 		defer log.Printf("Trace: Etcd: nominateCallback(): Finished!")
 	}
@@ -1396,10 +1402,10 @@ func (obj *EmbdEtcd) nominateCallback(re *RE) error {
 	_, exists := obj.nominated[obj.hostname]
 	// FIXME: can we get rid of the len(obj.nominated) == 0 ?
 	newCluster := len(obj.nominated) == 0 || (len(obj.nominated) == 1 && exists)
-	if global.DEBUG {
+	if obj.flags.Debug {
 		log.Printf("Etcd: nominateCallback(): newCluster: %v; exists: %v; obj.server == nil: %t", newCluster, exists, obj.server == nil)
 	}
-	// XXX check if i have actually volunteered first of all...
+	// XXX: check if i have actually volunteered first of all...
 	if obj.server == nil && (newCluster || exists) {
 
 		log.Printf("Etcd: StartServer(newCluster: %t): %+v", newCluster, obj.nominated)
@@ -1408,8 +1414,12 @@ func (obj *EmbdEtcd) nominateCallback(re *RE) error {
 			obj.nominated, // other peer members and urls or empty map
 		)
 		if err != nil {
+			var retries uint
+			if re != nil {
+				retries = re.retries
+			}
 			// retry maxStartServerRetries times, then permanently fail
-			return &CtxRetriesErr{maxStartServerRetries - re.retries, fmt.Sprintf("Etcd: StartServer: Error: %+v", err)}
+			return &CtxRetriesErr{maxStartServerRetries - retries, fmt.Sprintf("Etcd: StartServer: Error: %+v", err)}
 		}
 
 		if len(obj.endpoints) == 0 {
@@ -1426,7 +1436,7 @@ func (obj *EmbdEtcd) nominateCallback(re *RE) error {
 			// XXX: just put this wherever for now so we don't block
 			// nominate self so "member" list is correct for peers to see
 			EtcdNominate(obj, obj.hostname, obj.serverURLs)
-			// XXX if this fails, where will we retry this part ?
+			// XXX: if this fails, where will we retry this part ?
 		}
 
 		// advertise client urls
@@ -1434,7 +1444,7 @@ func (obj *EmbdEtcd) nominateCallback(re *RE) error {
 			// XXX: don't advertise local addresses! 127.0.0.1:2381 doesn't really help remote hosts
 			// XXX: but sometimes this is what we want... hmmm how do we decide? filter on callback?
 			EtcdAdvertiseEndpoints(obj, curls)
-			// XXX if this fails, where will we retry this part ?
+			// XXX: if this fails, where will we retry this part ?
 
 			// force this to remove sentinel before we reconnect...
 			obj.endpointCallback(nil)
@@ -1495,7 +1505,7 @@ func (obj *EmbdEtcd) nominateCallback(re *RE) error {
 
 // endpointCallback runs to respond to the endpoint list change events
 func (obj *EmbdEtcd) endpointCallback(re *RE) error {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: endpointCallback()")
 		defer log.Printf("Trace: Etcd: endpointCallback(): Finished!")
 	}
@@ -1561,7 +1571,7 @@ func (obj *EmbdEtcd) endpointCallback(re *RE) error {
 
 // idealClusterSizeCallback runs to respond to the ideal cluster size changes
 func (obj *EmbdEtcd) idealClusterSizeCallback(re *RE) error {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: idealClusterSizeCallback()")
 		defer log.Printf("Trace: Etcd: idealClusterSizeCallback(): Finished!")
 	}
@@ -1650,7 +1660,7 @@ func (obj *EmbdEtcd) StartServer(newCluster bool, peerURLsMap etcdtypes.URLsMap)
 	} else {
 		cfg.ClusterState = embed.ClusterStateFlagExisting
 	}
-	//cfg.ForceNewCluster = newCluster // TODO ?
+	//cfg.ForceNewCluster = newCluster // TODO: ?
 
 	log.Printf("Etcd: StartServer: Starting server...")
 	obj.server, err = embed.StartEtcd(cfg)
@@ -1667,6 +1677,14 @@ func (obj *EmbdEtcd) StartServer(newCluster bool, peerURLsMap etcdtypes.URLsMap)
 		obj.serverwg.Add(1)      // add for the DestroyServer()
 		obj.DestroyServer()
 		return e
+	// TODO: should we wait for this notification elsewhere?
+	case <-obj.server.Server.StopNotify(): // it's going down now...
+		e := fmt.Errorf("Etcd: StartServer: Received stop notification.")
+		log.Printf(e.Error())
+		obj.server.Server.Stop() // trigger a shutdown
+		obj.serverwg.Add(1)      // add for the DestroyServer()
+		obj.DestroyServer()
+		return e
 	}
 	//log.Fatal(<-obj.server.Err())	XXX
 	log.Printf("Etcd: StartServer: Server running...")
@@ -1700,7 +1718,7 @@ func (obj *EmbdEtcd) DestroyServer() error {
 
 // EtcdNominate nominates a particular client to be a server (peer)
 func EtcdNominate(obj *EmbdEtcd, hostname string, urls etcdtypes.URLs) error {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: EtcdNominate(%v): %v", hostname, urls.String())
 		defer log.Printf("Trace: Etcd: EtcdNominate(%v): Finished!", hostname)
 	}
@@ -1742,7 +1760,7 @@ func EtcdNominated(obj *EmbdEtcd) (etcdtypes.URLsMap, error) {
 			return nil, fmt.Errorf("Etcd: Nominated: Data format error!: %v", err)
 		}
 		nominated[name] = urls // add to map
-		if global.DEBUG {
+		if obj.flags.Debug {
 			log.Printf("Etcd: Nominated(%v): %v", name, val)
 		}
 	}
@@ -1751,7 +1769,7 @@ func EtcdNominated(obj *EmbdEtcd) (etcdtypes.URLsMap, error) {
 
 // EtcdVolunteer offers yourself up to be a server if needed
 func EtcdVolunteer(obj *EmbdEtcd, urls etcdtypes.URLs) error {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: EtcdVolunteer(%v): %v", obj.hostname, urls.String())
 		defer log.Printf("Trace: Etcd: EtcdVolunteer(%v): Finished!", obj.hostname)
 	}
@@ -1774,7 +1792,7 @@ func EtcdVolunteer(obj *EmbdEtcd, urls etcdtypes.URLs) error {
 
 // EtcdVolunteers returns a urls map of available etcd server volunteers
 func EtcdVolunteers(obj *EmbdEtcd) (etcdtypes.URLsMap, error) {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: EtcdVolunteers()")
 		defer log.Printf("Trace: Etcd: EtcdVolunteers(): Finished!")
 	}
@@ -1797,7 +1815,7 @@ func EtcdVolunteers(obj *EmbdEtcd) (etcdtypes.URLsMap, error) {
 			return nil, fmt.Errorf("Etcd: Volunteers: Data format error!: %v", err)
 		}
 		volunteers[name] = urls // add to map
-		if global.DEBUG {
+		if obj.flags.Debug {
 			log.Printf("Etcd: Volunteer(%v): %v", name, val)
 		}
 	}
@@ -1806,7 +1824,7 @@ func EtcdVolunteers(obj *EmbdEtcd) (etcdtypes.URLsMap, error) {
 
 // EtcdAdvertiseEndpoints advertises the list of available client endpoints
 func EtcdAdvertiseEndpoints(obj *EmbdEtcd, urls etcdtypes.URLs) error {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: EtcdAdvertiseEndpoints(%v): %v", obj.hostname, urls.String())
 		defer log.Printf("Trace: Etcd: EtcdAdvertiseEndpoints(%v): Finished!", obj.hostname)
 	}
@@ -1829,7 +1847,7 @@ func EtcdAdvertiseEndpoints(obj *EmbdEtcd, urls etcdtypes.URLs) error {
 
 // EtcdEndpoints returns a urls map of available etcd server endpoints
 func EtcdEndpoints(obj *EmbdEtcd) (etcdtypes.URLsMap, error) {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: EtcdEndpoints()")
 		defer log.Printf("Trace: Etcd: EtcdEndpoints(): Finished!")
 	}
@@ -1852,7 +1870,7 @@ func EtcdEndpoints(obj *EmbdEtcd) (etcdtypes.URLsMap, error) {
 			return nil, fmt.Errorf("Etcd: Endpoints: Data format error!: %v", err)
 		}
 		endpoints[name] = urls // add to map
-		if global.DEBUG {
+		if obj.flags.Debug {
 			log.Printf("Etcd: Endpoint(%v): %v", name, val)
 		}
 	}
@@ -1861,7 +1879,7 @@ func EtcdEndpoints(obj *EmbdEtcd) (etcdtypes.URLsMap, error) {
 
 // EtcdSetHostnameConverged sets whether a specific hostname is converged.
 func EtcdSetHostnameConverged(obj *EmbdEtcd, hostname string, isConverged bool) error {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: EtcdSetHostnameConverged(%s): %v", hostname, isConverged)
 		defer log.Printf("Trace: Etcd: EtcdSetHostnameConverged(%v): Finished!", hostname)
 	}
@@ -1875,7 +1893,7 @@ func EtcdSetHostnameConverged(obj *EmbdEtcd, hostname string, isConverged bool)
 
 // EtcdHostnameConverged returns a map of every hostname's converged state.
 func EtcdHostnameConverged(obj *EmbdEtcd) (map[string]bool, error) {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: EtcdHostnameConverged()")
 		defer log.Printf("Trace: Etcd: EtcdHostnameConverged(): Finished!")
 	}
@@ -1920,7 +1938,7 @@ func EtcdAddHostnameConvergedWatcher(obj *EmbdEtcd, callbackFn func(map[string]b
 
 // EtcdSetClusterSize sets the ideal target cluster size of etcd peers
 func EtcdSetClusterSize(obj *EmbdEtcd, value uint16) error {
-	if global.TRACE {
+	if obj.flags.Trace {
 		log.Printf("Trace: Etcd: EtcdSetClusterSize(): %v", value)
 		defer log.Printf("Trace: Etcd: EtcdSetClusterSize(): Finished!")
 	}
@@ -1954,7 +1972,7 @@ func EtcdGetClusterSize(obj *EmbdEtcd) (uint16, error) {
 
 // EtcdMemberAdd adds a member to the cluster.
 func EtcdMemberAdd(obj *EmbdEtcd, peerURLs etcdtypes.URLs) (*etcd.MemberAddResponse, error) {
-	//obj.Connect(false) // TODO ?
+	//obj.Connect(false) // TODO: ?
 	ctx := context.Background()
 	var response *etcd.MemberAddResponse
 	var err error
@@ -1979,7 +1997,7 @@ func EtcdMemberAdd(obj *EmbdEtcd, peerURLs etcdtypes.URLs) (*etcd.MemberAddRespo
 // if there was an error. This is because it might have run without error, but
 // the member wasn't found, for example.
 func EtcdMemberRemove(obj *EmbdEtcd, mID uint64) (bool, error) {
-	//obj.Connect(false) // TODO ?
+	//obj.Connect(false) // TODO: ?
 	ctx := context.Background()
 	for {
 		if obj.exiting { // the exit signal has been sent!
@@ -2005,7 +2023,7 @@ func EtcdMemberRemove(obj *EmbdEtcd, mID uint64) (bool, error) {
 // The member ID's are the keys, because an empty names means unstarted!
 // TODO: consider queueing this through the main loop with CtxError(ctx, err)
 func EtcdMembers(obj *EmbdEtcd) (map[uint64]string, error) {
-	//obj.Connect(false) // TODO ?
+	//obj.Connect(false) // TODO: ?
 	ctx := context.Background()
 	var response *etcd.MemberListResponse
 	var err error
@@ -2014,7 +2032,7 @@ func EtcdMembers(obj *EmbdEtcd) (map[uint64]string, error) {
 			return nil, fmt.Errorf("Exiting...")
 		}
 		obj.rLock.RLock()
-		if global.TRACE {
+		if obj.flags.Trace {
 			log.Printf("Trace: Etcd: EtcdMembers(): Endpoints are: %v", obj.client.Endpoints())
 		}
 		response, err = obj.client.MemberList(ctx)
@@ -2036,7 +2054,7 @@ func EtcdMembers(obj *EmbdEtcd) (map[uint64]string, error) {
 
 // EtcdLeader returns the current leader of the etcd server cluster
 func EtcdLeader(obj *EmbdEtcd) (string, error) {
-	//obj.Connect(false) // TODO ?
+	//obj.Connect(false) // TODO: ?
 	var err error
 	membersMap := make(map[uint64]string)
 	if membersMap, err = EtcdMembers(obj); err != nil {
@@ -2109,7 +2127,7 @@ func EtcdWatch(obj *EmbdEtcd) chan bool {
 
 // EtcdSetResources exports all of the resources which we pass in to etcd
 func EtcdSetResources(obj *EmbdEtcd, hostname string, resourceList []resources.Res) error {
-	// key structure is /$NS/exported/$hostname/resources/$uuid = $data
+	// key structure is /$NS/exported/$hostname/resources/$uid = $data
 
 	var kindFilter []string // empty to get from everyone
 	hostnameFilter := []string{hostname}
@@ -2130,8 +2148,8 @@ func EtcdSetResources(obj *EmbdEtcd, hostname string, resourceList []resources.R
 		if res.Kind() == "" {
 			log.Fatalf("Etcd: SetResources: Error: Empty kind: %v", res.GetName())
 		}
-		uuid := fmt.Sprintf("%s/%s", res.Kind(), res.GetName())
+		uid := fmt.Sprintf("%s/%s", res.Kind(), res.GetName())
-		path := fmt.Sprintf("/%s/exported/%s/resources/%s", NS, hostname, uuid)
+		path := fmt.Sprintf("/%s/exported/%s/resources/%s", NS, hostname, uid)
 		if data, err := resources.ResToB64(res); err == nil {
 			ifs = append(ifs, etcd.Compare(etcd.Value(path), "=", data)) // desired state
 			ops = append(ops, etcd.OpPut(path, data))
@@ -2155,8 +2173,8 @@ func EtcdSetResources(obj *EmbdEtcd, hostname string, resourceList []resources.R
 		if res.Kind() == "" {
 			log.Fatalf("Etcd: SetResources: Error: Empty kind: %v", res.GetName())
 		}
-		uuid := fmt.Sprintf("%s/%s", res.Kind(), res.GetName())
+		uid := fmt.Sprintf("%s/%s", res.Kind(), res.GetName())
-		path := fmt.Sprintf("/%s/exported/%s/resources/%s", NS, hostname, uuid)
+		path := fmt.Sprintf("/%s/exported/%s/resources/%s", NS, hostname, uid)
 
 		if match(res, resourceList) { // if we match, no need to delete!
 			continue
@@ -2182,9 +2200,9 @@ func EtcdSetResources(obj *EmbdEtcd, hostname string, resourceList []resources.R
 // If the kindfilter or hostnameFilter is empty, then it assumes no filtering...
 // TODO: Expand this with a more powerful filter based on what we eventually
 // support in our collect DSL. Ideally a server side filter like WithFilter()
-// We could do this if the pattern was /$NS/exported/$kind/$hostname/$uuid = $data
+// We could do this if the pattern was /$NS/exported/$kind/$hostname/$uid = $data
 func EtcdGetResources(obj *EmbdEtcd, hostnameFilter, kindFilter []string) ([]resources.Res, error) {
-	// key structure is /$NS/exported/$hostname/resources/$uuid = $data
+	// key structure is /$NS/exported/$hostname/resources/$uid = $data
 	path := fmt.Sprintf("/%s/exported/", NS)
 	resourceList := []resources.Res{}
 	keyMap, err := obj.Get(path, etcd.WithPrefix(), etcd.WithSort(etcd.SortByKey, etcd.SortAscend))
@@ -2266,9 +2284,7 @@ func ApplyDeltaEvents(re *RE, urlsmap etcdtypes.URLsMap) (etcdtypes.URLsMap, err
 			if _, exists := urlsmap[key]; !exists {
 				// this can happen if we retry an operation b/w
 				// a reconnect so ignore if we are reconnecting
-				if global.DEBUG {
 				log.Printf("Etcd: ApplyDeltaEvents: Inconsistent key: %v", key)
-				}
 				return nil, errApplyDeltaEventsInconsistent
 			}
 			delete(urlsmap, key)

etcd/world.go

@@ -0,0 +1,43 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package etcd
+
+import (
+	"github.com/purpleidea/mgmt/resources"
+)
+
+// World is an etcd backed implementation of the World interface.
+type World struct {
+	Hostname string // uuid for the consumer of these
+	EmbdEtcd *EmbdEtcd
+}
+
+// ResExport exports a list of resources under our hostname namespace.
+// Subsequent calls replace the previously set collection atomically.
+func (obj *World) ResExport(resourceList []resources.Res) error {
+	return EtcdSetResources(obj.EmbdEtcd, obj.Hostname, resourceList)
+}
+
+// ResCollect gets the collection of exported resources which match the filter.
+// It does this atomically so that a call always returns a complete collection.
+func (obj *World) ResCollect(hostnameFilter, kindFilter []string) ([]resources.Res, error) {
+	// XXX: should we be restricted to retrieving resources that were
+	// exported with a tag that allows or restricts our hostname? We could
+	// enforce that here if the underlying API supported it... Add this?
+	return EtcdGetResources(obj.EmbdEtcd, hostnameFilter, kindFilter)
+}

examples/hostname.yaml

@@ -0,0 +1,7 @@
+---
+graph: mygraph
+resources:
+  hostname:
+    - name: Hostname Watcher @ TestHost
+      hostname: test.hostname.example.com
+edges: []

examples/lib/libmgmt1.go

@@ -0,0 +1,188 @@
+// libmgmt example
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/purpleidea/mgmt/gapi"
+	mgmt "github.com/purpleidea/mgmt/lib"
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/resources"
+	"github.com/purpleidea/mgmt/yamlgraph"
+)
+
+// MyGAPI implements the main GAPI interface.
+type MyGAPI struct {
+	Name     string // graph name
+	Interval uint   // refresh interval, 0 to never refresh
+
+	data        gapi.Data
+	initialized bool
+	closeChan   chan struct{}
+	wg          sync.WaitGroup // sync group for tunnel go routines
+}
+
+// NewMyGAPI creates a new MyGAPI struct and calls Init().
+func NewMyGAPI(data gapi.Data, name string, interval uint) (*MyGAPI, error) {
+	obj := &MyGAPI{
+		Name:     name,
+		Interval: interval,
+	}
+	return obj, obj.Init(data)
+}
+
+// Init initializes the MyGAPI struct.
+func (obj *MyGAPI) Init(data gapi.Data) error {
+	if obj.initialized {
+		return fmt.Errorf("Already initialized!")
+	}
+	if obj.Name == "" {
+		return fmt.Errorf("The graph name must be specified!")
+	}
+	obj.data = data // store for later
+	obj.closeChan = make(chan struct{})
+	obj.initialized = true
+	return nil
+}
+
+// Graph returns a current Graph.
+func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
+	if !obj.initialized {
+		return nil, fmt.Errorf("libmgmt: MyGAPI is not initialized")
+	}
+
+	n1, err := resources.NewNoopRes("noop1")
+	if err != nil {
+		return nil, fmt.Errorf("Can't create resource: %v", err)
+	}
+
+	// we can still build a graph via the yaml method
+	gc := &yamlgraph.GraphConfig{
+		Graph: obj.Name,
+		Resources: yamlgraph.Resources{ // must redefine anonymous struct :(
+			// in alphabetical order
+			Exec:  []*resources.ExecRes{},
+			File:  []*resources.FileRes{},
+			Msg:   []*resources.MsgRes{},
+			Noop:  []*resources.NoopRes{n1},
+			Pkg:   []*resources.PkgRes{},
+			Svc:   []*resources.SvcRes{},
+			Timer: []*resources.TimerRes{},
+			Virt:  []*resources.VirtRes{},
+		},
+		//Collector: []collectorResConfig{},
+		//Edges:     []Edge{},
+		Comment: "comment!",
+	}
+
+	g, err := gc.NewGraphFromConfig(obj.data.Hostname, obj.data.World, obj.data.Noop)
+	return g, err
+}
+
+// Next returns nil errors every time there could be a new graph.
+func (obj *MyGAPI) Next() chan error {
+	if obj.data.NoWatch || obj.Interval <= 0 {
+		return nil
+	}
+	ch := make(chan error)
+	obj.wg.Add(1)
+	go func() {
+		defer obj.wg.Done()
+		defer close(ch) // this will run before the obj.wg.Done()
+		if !obj.initialized {
+			ch <- fmt.Errorf("libmgmt: MyGAPI is not initialized")
+			return
+		}
+
+		// arbitrarily change graph every interval seconds
+		ticker := time.NewTicker(time.Duration(obj.Interval) * time.Second)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ticker.C:
+				log.Printf("libmgmt: Generating new graph...")
+				ch <- nil // trigger a run
+			case <-obj.closeChan:
+				return
+			}
+		}
+	}()
+	return ch
+}
+
+// Close shuts down the MyGAPI.
+func (obj *MyGAPI) Close() error {
+	if !obj.initialized {
+		return fmt.Errorf("libmgmt: MyGAPI is not initialized")
+	}
+	close(obj.closeChan)
+	obj.wg.Wait()
+	obj.initialized = false // closed = true
+	return nil
+}
+
+// Run runs an embedded mgmt server.
+func Run() error {
+
+	obj := &mgmt.Main{}
+	obj.Program = "libmgmt" // TODO: set on compilation
+	obj.Version = "0.0.1"   // TODO: set on compilation
+	obj.TmpPrefix = true
+	obj.IdealClusterSize = -1
+	obj.ConvergedTimeout = -1
+	obj.Noop = true
+
+	obj.GAPI = &MyGAPI{ // graph API
+		Name:     "libmgmt", // TODO: set on compilation
+		Interval: 15,        // arbitrarily change graph every 15 seconds
+	}
+
+	if err := obj.Init(); err != nil {
+		return err
+	}
+
+	// install the exit signal handler
+	exit := make(chan struct{})
+	defer close(exit)
+	go func() {
+		signals := make(chan os.Signal, 1)
+		signal.Notify(signals, os.Interrupt) // catch ^C
+		//signal.Notify(signals, os.Kill) // catch signals
+		signal.Notify(signals, syscall.SIGTERM)
+
+		select {
+		case sig := <-signals: // any signal will do
+			if sig == os.Interrupt {
+				log.Println("Interrupted by ^C")
+				obj.Exit(nil)
+				return
+			}
+			log.Println("Interrupted by signal")
+			obj.Exit(fmt.Errorf("Killed by %v", sig))
+			return
+		case <-exit:
+			return
+		}
+	}()
+
+	if err := obj.Run(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func main() {
+	log.Printf("Hello!")
+	if err := Run(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+		return
+	}
+	log.Printf("Goodbye!")
+}

examples/lib/libmgmt2.go

@@ -0,0 +1,188 @@
+// libmgmt example
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"strconv"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/purpleidea/mgmt/gapi"
+	mgmt "github.com/purpleidea/mgmt/lib"
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/resources"
+)
+
+// MyGAPI implements the main GAPI interface.
+type MyGAPI struct {
+	Name     string // graph name
+	Count    uint   // number of resources to create
+	Interval uint   // refresh interval, 0 to never refresh
+
+	data        gapi.Data
+	initialized bool
+	closeChan   chan struct{}
+	wg          sync.WaitGroup // sync group for tunnel go routines
+}
+
+// NewMyGAPI creates a new MyGAPI struct and calls Init().
+func NewMyGAPI(data gapi.Data, name string, interval uint, count uint) (*MyGAPI, error) {
+	obj := &MyGAPI{
+		Name:     name,
+		Count:    count,
+		Interval: interval,
+	}
+	return obj, obj.Init(data)
+}
+
+// Init initializes the MyGAPI struct.
+func (obj *MyGAPI) Init(data gapi.Data) error {
+	if obj.initialized {
+		return fmt.Errorf("Already initialized!")
+	}
+	if obj.Name == "" {
+		return fmt.Errorf("The graph name must be specified!")
+	}
+	obj.data = data // store for later
+	obj.closeChan = make(chan struct{})
+	obj.initialized = true
+	return nil
+}
+
+// Graph returns a current Graph.
+func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
+	if !obj.initialized {
+		return nil, fmt.Errorf("libmgmt: MyGAPI is not initialized")
+	}
+
+	g := pgraph.NewGraph(obj.Name)
+	var vertex *pgraph.Vertex
+	for i := uint(0); i < obj.Count; i++ {
+		n, err := resources.NewNoopRes(fmt.Sprintf("noop%d", i))
+		if err != nil {
+			return nil, fmt.Errorf("Can't create resource: %v", err)
+		}
+		v := pgraph.NewVertex(n)
+		g.AddVertex(v)
+		if i > 0 {
+			g.AddEdge(vertex, v, pgraph.NewEdge(fmt.Sprintf("e%d", i)))
+		}
+		vertex = v // save
+	}
+
+	//g, err := config.NewGraphFromConfig(obj.data.Hostname, obj.data.World, obj.data.Noop)
+	return g, nil
+}
+
+// Next returns nil errors every time there could be a new graph.
+func (obj *MyGAPI) Next() chan error {
+	if obj.data.NoWatch || obj.Interval <= 0 {
+		return nil
+	}
+	ch := make(chan error)
+	obj.wg.Add(1)
+	go func() {
+		defer obj.wg.Done()
+		defer close(ch) // this will run before the obj.wg.Done()
+		if !obj.initialized {
+			ch <- fmt.Errorf("libmgmt: MyGAPI is not initialized")
+			return
+		}
+
+		// arbitrarily change graph every interval seconds
+		ticker := time.NewTicker(time.Duration(obj.Interval) * time.Second)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ticker.C:
+				log.Printf("libmgmt: Generating new graph...")
+				ch <- nil // trigger a run
+			case <-obj.closeChan:
+				return
+			}
+		}
+	}()
+	return ch
+}
+
+// Close shuts down the MyGAPI.
+func (obj *MyGAPI) Close() error {
+	if !obj.initialized {
+		return fmt.Errorf("libmgmt: MyGAPI is not initialized")
+	}
+	close(obj.closeChan)
+	obj.wg.Wait()
+	obj.initialized = false // closed = true
+	return nil
+}
+
+// Run runs an embedded mgmt server.
+func Run(count uint) error {
+
+	obj := &mgmt.Main{}
+	obj.Program = "libmgmt" // TODO: set on compilation
+	obj.Version = "0.0.1"   // TODO: set on compilation
+	obj.TmpPrefix = true
+	obj.IdealClusterSize = -1
+	obj.ConvergedTimeout = -1
+	obj.Noop = true
+
+	obj.GAPI = &MyGAPI{ // graph API
+		Name:     "libmgmt", // TODO: set on compilation
+		Count:    count,     // number of vertices to add
+		Interval: 15,        // arbitrarily change graph every 15 seconds
+	}
+
+	if err := obj.Init(); err != nil {
+		return err
+	}
+
+	// install the exit signal handler
+	exit := make(chan struct{})
+	defer close(exit)
+	go func() {
+		signals := make(chan os.Signal, 1)
+		signal.Notify(signals, os.Interrupt) // catch ^C
+		//signal.Notify(signals, os.Kill) // catch signals
+		signal.Notify(signals, syscall.SIGTERM)
+
+		select {
+		case sig := <-signals: // any signal will do
+			if sig == os.Interrupt {
+				log.Println("Interrupted by ^C")
+				obj.Exit(nil)
+				return
+			}
+			log.Println("Interrupted by signal")
+			obj.Exit(fmt.Errorf("Killed by %v", sig))
+			return
+		case <-exit:
+			return
+		}
+	}()
+
+	if err := obj.Run(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func main() {
+	log.Printf("Hello!")
+	var count uint = 1 // default
+	if len(os.Args) == 2 {
+		if i, err := strconv.Atoi(os.Args[1]); err == nil && i > 0 {
+			count = uint(i)
+		}
+	}
+	if err := Run(count); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+		return
+	}
+	log.Printf("Goodbye!")
+}

examples/lib/libmgmt3.go

@@ -0,0 +1,225 @@
+// libmgmt example of send->recv
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/purpleidea/mgmt/gapi"
+	mgmt "github.com/purpleidea/mgmt/lib"
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/resources"
+)
+
+// MyGAPI implements the main GAPI interface.
+type MyGAPI struct {
+	Name     string // graph name
+	Interval uint   // refresh interval, 0 to never refresh
+
+	data        gapi.Data
+	initialized bool
+	closeChan   chan struct{}
+	wg          sync.WaitGroup // sync group for tunnel go routines
+}
+
+// NewMyGAPI creates a new MyGAPI struct and calls Init().
+func NewMyGAPI(data gapi.Data, name string, interval uint) (*MyGAPI, error) {
+	obj := &MyGAPI{
+		Name:     name,
+		Interval: interval,
+	}
+	return obj, obj.Init(data)
+}
+
+// Init initializes the MyGAPI struct.
+func (obj *MyGAPI) Init(data gapi.Data) error {
+	if obj.initialized {
+		return fmt.Errorf("Already initialized!")
+	}
+	if obj.Name == "" {
+		return fmt.Errorf("The graph name must be specified!")
+	}
+	obj.data = data // store for later
+	obj.closeChan = make(chan struct{})
+	obj.initialized = true
+	return nil
+}
+
+// Graph returns a current Graph.
+func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
+	if !obj.initialized {
+		return nil, fmt.Errorf("libmgmt: MyGAPI is not initialized")
+	}
+
+	g := pgraph.NewGraph(obj.Name)
+
+	content := "Delete me to trigger a notification!\n"
+	f0 := &resources.FileRes{
+		BaseRes: resources.BaseRes{
+			Name: "README",
+		},
+		Path:    "/tmp/mgmt/README",
+		Content: &content,
+		State:   "present",
+	}
+
+	v0 := pgraph.NewVertex(f0)
+	g.AddVertex(v0)
+
+	p1 := &resources.PasswordRes{
+		BaseRes: resources.BaseRes{
+			Name: "password1",
+		},
+		Length: 8,    // generated string will have this many characters
+		Saved:  true, // this causes passwords to be stored in plain text!
+	}
+	v1 := pgraph.NewVertex(p1)
+	g.AddVertex(v1)
+
+	f1 := &resources.FileRes{
+		BaseRes: resources.BaseRes{
+			Name: "file1",
+			// send->recv!
+			Recv: map[string]*resources.Send{
+				"Content": {Res: p1, Key: "Password"},
+			},
+		},
+		Path: "/tmp/mgmt/secret",
+		//Content:  p1.Password, // won't work
+		State: "present",
+	}
+
+	v2 := pgraph.NewVertex(f1)
+	g.AddVertex(v2)
+
+	n1 := &resources.NoopRes{
+		BaseRes: resources.BaseRes{
+			Name: "noop1",
+		},
+	}
+
+	v3 := pgraph.NewVertex(n1)
+	g.AddVertex(v3)
+
+	e0 := pgraph.NewEdge("e0")
+	e0.Notify = true // send a notification from v0 to v1
+	g.AddEdge(v0, v1, e0)
+
+	g.AddEdge(v1, v2, pgraph.NewEdge("e1"))
+
+	e2 := pgraph.NewEdge("e2")
+	e2.Notify = true // send a notification from v2 to v3
+	g.AddEdge(v2, v3, e2)
+
+	//g, err := config.NewGraphFromConfig(obj.data.Hostname, obj.data.World, obj.data.Noop)
+	return g, nil
+}
+
+// Next returns nil errors every time there could be a new graph.
+func (obj *MyGAPI) Next() chan error {
+	if obj.data.NoWatch || obj.Interval <= 0 {
+		return nil
+	}
+	ch := make(chan error)
+	obj.wg.Add(1)
+	go func() {
+		defer obj.wg.Done()
+		defer close(ch) // this will run before the obj.wg.Done()
+		if !obj.initialized {
+			ch <- fmt.Errorf("libmgmt: MyGAPI is not initialized")
+			return
+		}
+
+		// arbitrarily change graph every interval seconds
+		ticker := time.NewTicker(time.Duration(obj.Interval) * time.Second)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ticker.C:
+				log.Printf("libmgmt: Generating new graph...")
+				ch <- nil // trigger a run
+			case <-obj.closeChan:
+				return
+			}
+		}
+	}()
+	return ch
+}
+
+// Close shuts down the MyGAPI.
+func (obj *MyGAPI) Close() error {
+	if !obj.initialized {
+		return fmt.Errorf("libmgmt: MyGAPI is not initialized")
+	}
+	close(obj.closeChan)
+	obj.wg.Wait()
+	obj.initialized = false // closed = true
+	return nil
+}
+
+// Run runs an embedded mgmt server.
+func Run() error {
+
+	obj := &mgmt.Main{}
+	obj.Program = "libmgmt" // TODO: set on compilation
+	obj.Version = "0.0.1"   // TODO: set on compilation
+	obj.TmpPrefix = true    // disable for easy debugging
+	//prefix := "/tmp/testprefix/"
+	//obj.Prefix = &p // enable for easy debugging
+	obj.IdealClusterSize = -1
+	obj.ConvergedTimeout = -1
+	obj.Noop = false // FIXME: careful!
+
+	obj.GAPI = &MyGAPI{ // graph API
+		Name:     "libmgmt", // TODO: set on compilation
+		Interval: 60 * 10,   // arbitrarily change graph every 15 seconds
+	}
+
+	if err := obj.Init(); err != nil {
+		return err
+	}
+
+	// install the exit signal handler
+	exit := make(chan struct{})
+	defer close(exit)
+	go func() {
+		signals := make(chan os.Signal, 1)
+		signal.Notify(signals, os.Interrupt) // catch ^C
+		//signal.Notify(signals, os.Kill) // catch signals
+		signal.Notify(signals, syscall.SIGTERM)
+
+		select {
+		case sig := <-signals: // any signal will do
+			if sig == os.Interrupt {
+				log.Println("Interrupted by ^C")
+				obj.Exit(nil)
+				return
+			}
+			log.Println("Interrupted by signal")
+			obj.Exit(fmt.Errorf("Killed by %v", sig))
+			return
+		case <-exit:
+			return
+		}
+	}()
+
+	if err := obj.Run(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func main() {
+	log.Printf("Hello!")
+	if err := Run(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+		return
+	}
+	log.Printf("Goodbye!")
+}

examples/nspawn1.yaml

@@ -0,0 +1,7 @@
+---
+graph: mygraph
+resources:
+  nspawn:
+  - name: mgmt-nspawn1
+    state: running
+edges: []

examples/nspawn2.yaml

@@ -0,0 +1,7 @@
+---
+graph: mygraph
+resources:
+  nspawn:
+  - name: mgmt-nspawn2
+    state: stopped
+edges: []

examples/remote2a.yaml

@@ -0,0 +1,20 @@
+---
+graph: mygraph
+comment: remote noop example
+resources:
+  file:
+  - name: file1a
+    path: "/tmp/file1a"
+    content: |
+      i am file1a
+    state: exists
+  - name: "@@file2a"
+    path: "/tmp/file2a"
+    content: |
+      i am file2a, exported from host a
+    state: exists
+collect:
+- kind: file
+  pattern: "/tmp/"
+edges: []
+remote: ssh://root:vagrant@192.168.121.201:22

examples/remote2b.yaml

@@ -0,0 +1,20 @@
+---
+graph: mygraph
+comment: remote noop example
+resources:
+  file:
+  - name: file1b
+    path: "/tmp/file1b"
+    content: |
+      i am file1b
+    state: exists
+  - name: "@@file2b"
+    path: "/tmp/file2b"
+    content: |
+      i am file2b, exported from host b
+    state: exists
+collect:
+- kind: file
+  pattern: "/tmp/"
+edges: []
+remote: ssh://root:vagrant@192.168.121.202:22

examples/virt1.yaml

@@ -0,0 +1,11 @@
+---
+graph: mygraph
+resources:
+  virt:
+  - name: mgmt1
+    uri: 'qemu:///session'
+    cpus: 1
+    memory: 524288
+    state: shutoff
+    transient: true
+edges: []

examples/virt2.yaml

@@ -0,0 +1,11 @@
+---
+graph: mygraph
+resources:
+  virt:
+  - name: mgmt2
+    uri: 'qemu:///session'
+    cpus: 1
+    memory: 524288
+    state: shutoff
+    transient: false
+edges: []

examples/virt3.yaml

@@ -0,0 +1,11 @@
+---
+graph: mygraph
+resources:
+  virt:
+  - name: mgmt3
+    uri: 'qemu:///session'
+    cpus: 1
+    memory: 524288
+    state: running
+    transient: false
+edges: []

gapi/gapi.go

@@ -0,0 +1,50 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// Package gapi defines the interface that graph API generators must meet.
+package gapi
+
+import (
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/resources"
+)
+
+// World is an interface to the rest of the different graph state. It allows
+// the GAPI to store state and exchange information throughout the cluster. It
+// is the interface each machine uses to communicate with the rest of the world.
+type World interface { // TODO: is there a better name for this interface?
+	ResExport([]resources.Res) error
+	// FIXME: should this method take a "filter" data struct instead of many args?
+	ResCollect(hostnameFilter, kindFilter []string) ([]resources.Res, error)
+}
+
+// Data is the set of input values passed into the GAPI structs via Init.
+type Data struct {
+	Hostname string // uuid for the host, required for GAPI
+	World    World
+	Noop     bool
+	NoWatch  bool
+	// NOTE: we can add more fields here if needed by GAPI endpoints
+}
+
+// GAPI is a Graph API that represents incoming graphs and change streams.
+type GAPI interface {
+	Init(Data) error               // initializes the GAPI and passes in useful data
+	Graph() (*pgraph.Graph, error) // returns the most recent pgraph
+	Next() chan error              // returns a stream of switch events
+	Close() error                  // shutdown the GAPI
+}

global/global.go

@@ -1,26 +0,0 @@
-// Mgmt
-// Copyright (C) 2013-2016+ James Shubin and the project contributors
-// Written by James Shubin <james@shubin.ca> and the project contributors
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-// Package global holds some global variables that are used throughout the code.
-package global
-
-// These constants are used throughout the program.
-const (
-	DEBUG   = false // add additional log messages
-	TRACE   = false // add execution flow log messages
-	VERBOSE = false // add extra log message output
-)

lib/cli.go

@@ -0,0 +1,328 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package lib
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/purpleidea/mgmt/puppet"
+	"github.com/purpleidea/mgmt/yamlgraph"
+
+	"github.com/urfave/cli"
+)
+
+// run is the main run target.
+func run(c *cli.Context) error {
+
+	obj := &Main{}
+
+	obj.Program = c.App.Name
+	obj.Version = c.App.Version
+	if val, exists := c.App.Metadata["flags"]; exists {
+		if flags, ok := val.(Flags); ok {
+			obj.Flags = flags
+		}
+	}
+
+	if h := c.String("hostname"); c.IsSet("hostname") && h != "" {
+		obj.Hostname = &h
+	}
+
+	if s := c.String("prefix"); c.IsSet("prefix") && s != "" {
+		obj.Prefix = &s
+	}
+	obj.TmpPrefix = c.Bool("tmp-prefix")
+	obj.AllowTmpPrefix = c.Bool("allow-tmp-prefix")
+
+	if _ = c.String("code"); c.IsSet("code") {
+		if obj.GAPI != nil {
+			return fmt.Errorf("Can't combine code GAPI with existing GAPI.")
+		}
+		// TODO: implement DSL GAPI
+		//obj.GAPI = &dsl.GAPI{
+		//	Code: &s,
+		//}
+		return fmt.Errorf("The Code GAPI is not implemented yet!") // TODO: DSL
+	}
+	if y := c.String("yaml"); c.IsSet("yaml") {
+		if obj.GAPI != nil {
+			return fmt.Errorf("Can't combine YAML GAPI with existing GAPI.")
+		}
+		obj.GAPI = &yamlgraph.GAPI{
+			File: &y,
+		}
+	}
+	if p := c.String("puppet"); c.IsSet("puppet") {
+		if obj.GAPI != nil {
+			return fmt.Errorf("Can't combine puppet GAPI with existing GAPI.")
+		}
+		obj.GAPI = &puppet.GAPI{
+			PuppetParam: &p,
+			PuppetConf:  c.String("puppet-conf"),
+		}
+	}
+	obj.Remotes = c.StringSlice("remote") // FIXME: GAPI-ify somehow?
+
+	obj.NoWatch = c.Bool("no-watch")
+	obj.Noop = c.Bool("noop")
+	obj.Graphviz = c.String("graphviz")
+	obj.GraphvizFilter = c.String("graphviz-filter")
+	obj.ConvergedTimeout = c.Int("converged-timeout")
+	obj.MaxRuntime = uint(c.Int("max-runtime"))
+
+	obj.Seeds = c.StringSlice("seeds")
+	obj.ClientURLs = c.StringSlice("client-urls")
+	obj.ServerURLs = c.StringSlice("server-urls")
+	obj.IdealClusterSize = c.Int("ideal-cluster-size")
+	obj.NoServer = c.Bool("no-server")
+
+	obj.CConns = uint16(c.Int("cconns"))
+	obj.AllowInteractive = c.Bool("allow-interactive")
+	obj.SSHPrivIDRsa = c.String("ssh-priv-id-rsa")
+	obj.NoCaching = c.Bool("no-caching")
+	obj.Depth = uint16(c.Int("depth"))
+
+	obj.NoPgp = c.Bool("no-pgp")
+
+	if kp := c.String("pgp-key-path"); c.IsSet("pgp-key-path") {
+		obj.PgpKeyPath = &kp
+	}
+
+	if us := c.String("pgp-identity"); c.IsSet("pgp-identity") {
+		obj.PgpIdentity = &us
+	}
+
+	if err := obj.Init(); err != nil {
+		return err
+	}
+
+	// install the exit signal handler
+	exit := make(chan struct{})
+	defer close(exit)
+	go func() {
+		signals := make(chan os.Signal, 1)
+		signal.Notify(signals, os.Interrupt) // catch ^C
+		//signal.Notify(signals, os.Kill) // catch signals
+		signal.Notify(signals, syscall.SIGTERM)
+
+		select {
+		case sig := <-signals: // any signal will do
+			if sig == os.Interrupt {
+				log.Println("Interrupted by ^C")
+				obj.Exit(nil)
+				return
+			}
+			log.Println("Interrupted by signal")
+			obj.Exit(fmt.Errorf("Killed by %v", sig))
+			return
+		case <-exit:
+			return
+		}
+	}()
+
+	if err := obj.Run(); err != nil {
+		return err
+		//return cli.NewExitError(err.Error(), 1) // TODO: ?
+		//return cli.NewExitError("", 1) // TODO: ?
+	}
+	return nil
+}
+
+// CLI is the entry point for using mgmt normally from the CLI.
+func CLI(program, version string, flags Flags) error {
+
+	// test for sanity
+	if program == "" || version == "" {
+		return fmt.Errorf("Program was not compiled correctly. Please see Makefile.")
+	}
+	app := cli.NewApp()
+	app.Name = program // App.name and App.version pass these values through
+	app.Version = version
+	app.Usage = "next generation config management"
+	app.Metadata = map[string]interface{}{ // additional flags
+		"flags": flags,
+	}
+	//app.Action = ... // without a default action, help runs
+
+	app.Commands = []cli.Command{
+		{
+			Name:    "run",
+			Aliases: []string{"r"},
+			Usage:   "run",
+			Action:  run,
+			Flags: []cli.Flag{
+				// useful for testing multiple instances on same machine
+				cli.StringFlag{
+					Name:  "hostname",
+					Value: "",
+					Usage: "hostname to use",
+				},
+
+				cli.StringFlag{
+					Name:   "prefix",
+					Usage:  "specify a path to the working prefix directory",
+					EnvVar: "MGMT_PREFIX",
+				},
+				cli.BoolFlag{
+					Name:  "tmp-prefix",
+					Usage: "request a pseudo-random, temporary prefix to be used",
+				},
+				cli.BoolFlag{
+					Name:  "allow-tmp-prefix",
+					Usage: "allow creation of a new temporary prefix if main prefix is unavailable",
+				},
+
+				cli.StringFlag{
+					Name:  "code, c",
+					Value: "",
+					Usage: "code definition to run",
+				},
+				cli.StringFlag{
+					Name:  "yaml",
+					Value: "",
+					Usage: "yaml graph definition to run",
+				},
+				cli.StringFlag{
+					Name:  "puppet, p",
+					Value: "",
+					Usage: "load graph from puppet, optionally takes a manifest or path to manifest file",
+				},
+				cli.StringFlag{
+					Name:  "puppet-conf",
+					Value: "",
+					Usage: "the path to an alternate puppet.conf file",
+				},
+				cli.StringSliceFlag{
+					Name:  "remote",
+					Value: &cli.StringSlice{},
+					Usage: "list of remote graph definitions to run",
+				},
+
+				cli.BoolFlag{
+					Name:  "no-watch",
+					Usage: "do not update graph on stream switch events",
+				},
+				cli.BoolFlag{
+					Name:  "noop",
+					Usage: "globally force all resources into no-op mode",
+				},
+				cli.StringFlag{
+					Name:  "graphviz, g",
+					Value: "",
+					Usage: "output file for graphviz data",
+				},
+				cli.StringFlag{
+					Name:  "graphviz-filter, gf",
+					Value: "dot", // directed graph default
+					Usage: "graphviz filter to use",
+				},
+				cli.IntFlag{
+					Name:   "converged-timeout, t",
+					Value:  -1,
+					Usage:  "exit after approximately this many seconds in a converged state",
+					EnvVar: "MGMT_CONVERGED_TIMEOUT",
+				},
+				cli.IntFlag{
+					Name:   "max-runtime",
+					Value:  0,
+					Usage:  "exit after a maximum of approximately this many seconds",
+					EnvVar: "MGMT_MAX_RUNTIME",
+				},
+
+				// if empty, it will startup a new server
+				cli.StringSliceFlag{
+					Name:   "seeds, s",
+					Value:  &cli.StringSlice{}, // empty slice
+					Usage:  "default etc client endpoint",
+					EnvVar: "MGMT_SEEDS",
+				},
+				// port 2379 and 4001 are common
+				cli.StringSliceFlag{
+					Name:   "client-urls",
+					Value:  &cli.StringSlice{},
+					Usage:  "list of URLs to listen on for client traffic",
+					EnvVar: "MGMT_CLIENT_URLS",
+				},
+				// port 2380 and 7001 are common
+				cli.StringSliceFlag{
+					Name:   "server-urls, peer-urls",
+					Value:  &cli.StringSlice{},
+					Usage:  "list of URLs to listen on for server (peer) traffic",
+					EnvVar: "MGMT_SERVER_URLS",
+				},
+				cli.IntFlag{
+					Name:   "ideal-cluster-size",
+					Value:  -1,
+					Usage:  "ideal number of server peers in cluster; only read by initial server",
+					EnvVar: "MGMT_IDEAL_CLUSTER_SIZE",
+				},
+				cli.BoolFlag{
+					Name:  "no-server",
+					Usage: "do not let other servers peer with me",
+				},
+
+				cli.IntFlag{
+					Name:   "cconns",
+					Value:  0,
+					Usage:  "number of maximum concurrent remote ssh connections to run; 0 for unlimited",
+					EnvVar: "MGMT_CCONNS",
+				},
+				cli.BoolFlag{
+					Name:  "allow-interactive",
+					Usage: "allow interactive prompting, such as for remote passwords",
+				},
+				cli.StringFlag{
+					Name:   "ssh-priv-id-rsa",
+					Value:  "~/.ssh/id_rsa",
+					Usage:  "default path to ssh key file, set empty to never touch",
+					EnvVar: "MGMT_SSH_PRIV_ID_RSA",
+				},
+				cli.BoolFlag{
+					Name:  "no-caching",
+					Usage: "don't allow remote caching of remote execution binary",
+				},
+				cli.IntFlag{
+					Name:   "depth",
+					Hidden: true, // internal use only
+					Value:  0,
+					Usage:  "specify depth in remote hierarchy",
+				},
+				cli.BoolFlag{
+					Name:  "no-pgp",
+					Usage: "don't create pgp keys",
+				},
+				cli.StringFlag{
+					Name:  "pgp-key-path",
+					Value: "",
+					Usage: "path for instance key pair",
+				},
+				cli.StringFlag{
+					Name:  "pgp-identity",
+					Value: "",
+					Usage: "default identity used for generation",
+				},
+			},
+		},
+	}
+	app.EnableBashCompletion = true
+	return app.Run(os.Args)
+}

lib/main.go

@@ -0,0 +1,561 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package lib
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+	"time"
+
+	"github.com/purpleidea/mgmt/converger"
+	"github.com/purpleidea/mgmt/etcd"
+	"github.com/purpleidea/mgmt/gapi"
+	"github.com/purpleidea/mgmt/pgp"
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/recwatch"
+	"github.com/purpleidea/mgmt/remote"
+	"github.com/purpleidea/mgmt/resources"
+	"github.com/purpleidea/mgmt/util"
+
+	etcdtypes "github.com/coreos/etcd/pkg/types"
+	"github.com/coreos/pkg/capnslog"
+	multierr "github.com/hashicorp/go-multierror"
+	errwrap "github.com/pkg/errors"
+)
+
+// Flags are some constant flags which are used throughout the program.
+type Flags struct {
+	Debug   bool // add additional log messages
+	Trace   bool // add execution flow log messages
+	Verbose bool // add extra log message output
+}
+
+// Main is the main struct for running the mgmt logic.
+type Main struct {
+	Program string // the name of this program, usually set at compile time
+	Version string // the version of this program, usually set at compile time
+
+	Flags Flags // static global flags that are set at compile time
+
+	Hostname *string // hostname to use; nil if undefined
+
+	Prefix         *string // prefix passed in; nil if undefined
+	TmpPrefix      bool    // request a pseudo-random, temporary prefix to be used
+	AllowTmpPrefix bool    // allow creation of a new temporary prefix if main prefix is unavailable
+
+	GAPI    gapi.GAPI // graph API interface struct
+	Remotes []string  // list of remote graph definitions to run
+
+	NoWatch          bool   // do not update graph on watched graph definition file changes
+	Noop             bool   // globally force all resources into no-op mode
+	Graphviz         string // output file for graphviz data
+	GraphvizFilter   string // graphviz filter to use
+	ConvergedTimeout int    // exit after approximately this many seconds in a converged state; -1 to disable
+	MaxRuntime       uint   // exit after a maximum of approximately this many seconds
+
+	Seeds            []string // default etc client endpoint
+	ClientURLs       []string // list of URLs to listen on for client traffic
+	ServerURLs       []string // list of URLs to listen on for server (peer) traffic
+	IdealClusterSize int      // ideal number of server peers in cluster; only read by initial server
+	NoServer         bool     // do not let other servers peer with me
+
+	CConns           uint16 // number of maximum concurrent remote ssh connections to run, 0 for unlimited
+	AllowInteractive bool   // allow interactive prompting, such as for remote passwords
+	SSHPrivIDRsa     string // default path to ssh key file, set empty to never touch
+	NoCaching        bool   // don't allow remote caching of remote execution binary
+	Depth            uint16 // depth in remote hierarchy; for internal use only
+
+	seeds            etcdtypes.URLs // processed seeds value
+	clientURLs       etcdtypes.URLs // processed client urls value
+	serverURLs       etcdtypes.URLs // processed server urls value
+	idealClusterSize uint16         // processed ideal cluster size value
+
+	NoPgp       bool    // disallow pgp functionality
+	PgpKeyPath  *string // import a pre-made key pair
+	PgpIdentity *string
+	pgpKeys     *pgp.PGP // agent key pair
+
+	exit chan error // exit signal
+}
+
+// Init initializes the main struct after it performs some validation.
+func (obj *Main) Init() error {
+
+	if obj.Program == "" || obj.Version == "" {
+		return fmt.Errorf("You must set the Program and Version strings!")
+	}
+
+	if obj.Prefix != nil && obj.TmpPrefix {
+		return fmt.Errorf("Choosing a prefix and the request for a tmp prefix is illogical!")
+	}
+
+	obj.idealClusterSize = uint16(obj.IdealClusterSize)
+	if obj.IdealClusterSize < 0 { // value is undefined, set to the default
+		obj.idealClusterSize = etcd.DefaultIdealClusterSize
+	}
+
+	if obj.idealClusterSize < 1 {
+		return fmt.Errorf("IdealClusterSize should be at least one!")
+	}
+
+	if obj.NoServer && len(obj.Remotes) > 0 {
+		// TODO: in this case, we won't be able to tunnel stuff back to
+		// here, so if we're okay with every remote graph running in an
+		// isolated mode, then this is okay. Improve on this if there's
+		// someone who really wants to be able to do this.
+		return fmt.Errorf("The Server is required when using Remotes!")
+	}
+
+	if obj.CConns < 0 {
+		return fmt.Errorf("The CConns value should be at least zero!")
+	}
+
+	if obj.ConvergedTimeout >= 0 && obj.CConns > 0 && len(obj.Remotes) > int(obj.CConns) {
+		return fmt.Errorf("You can't converge if you have more remotes than available connections!")
+	}
+
+	if obj.Depth < 0 { // user should not be using this argument manually
+		return fmt.Errorf("Negative values for Depth are not permitted!")
+	}
+
+	// transform the url list inputs into etcd typed lists
+	var err error
+	obj.seeds, err = etcdtypes.NewURLs(
+		util.FlattenListWithSplit(obj.Seeds, []string{",", ";", " "}),
+	)
+	if err != nil && len(obj.Seeds) > 0 {
+		return fmt.Errorf("Seeds didn't parse correctly!")
+	}
+	obj.clientURLs, err = etcdtypes.NewURLs(
+		util.FlattenListWithSplit(obj.ClientURLs, []string{",", ";", " "}),
+	)
+	if err != nil && len(obj.ClientURLs) > 0 {
+		return fmt.Errorf("ClientURLs didn't parse correctly!")
+	}
+	obj.serverURLs, err = etcdtypes.NewURLs(
+		util.FlattenListWithSplit(obj.ServerURLs, []string{",", ";", " "}),
+	)
+	if err != nil && len(obj.ServerURLs) > 0 {
+		return fmt.Errorf("ServerURLs didn't parse correctly!")
+	}
+
+	obj.exit = make(chan error)
+	return nil
+}
+
+// Exit causes a safe shutdown. This is often attached to the ^C signal handler.
+func (obj *Main) Exit(err error) {
+	obj.exit <- err // trigger an exit!
+}
+
+// Run is the main execution entrypoint to run mgmt.
+func (obj *Main) Run() error {
+
+	var start = time.Now().UnixNano()
+
+	var flags int
+	if obj.Flags.Debug || true { // TODO: remove || true
+		flags = log.LstdFlags | log.Lshortfile
+	}
+	flags = (flags - log.Ldate) // remove the date for now
+	log.SetFlags(flags)
+
+	// un-hijack from capnslog...
+	log.SetOutput(os.Stderr)
+	if obj.Flags.Verbose {
+		capnslog.SetFormatter(capnslog.NewLogFormatter(os.Stderr, "(etcd) ", flags))
+	} else {
+		capnslog.SetFormatter(capnslog.NewNilFormatter())
+	}
+
+	log.Printf("This is: %s, version: %s", obj.Program, obj.Version)
+	log.Printf("Main: Start: %v", start)
+
+	hostname, err := os.Hostname() // a sensible default
+	// allow passing in the hostname, instead of using the system setting
+	if h := obj.Hostname; h != nil && *h != "" { // override by cli
+		hostname = *h
+	} else if err != nil {
+		return errwrap.Wrapf(err, "Can't get default hostname!")
+	}
+	if hostname == "" { // safety check
+		return fmt.Errorf("Hostname cannot be empty!")
+	}
+
+	var prefix = fmt.Sprintf("/var/lib/%s/", obj.Program) // default prefix
+	if p := obj.Prefix; p != nil {
+		prefix = *p
+	}
+	// make sure the working directory prefix exists
+	if obj.TmpPrefix || os.MkdirAll(prefix, 0770) != nil {
+		if obj.TmpPrefix || obj.AllowTmpPrefix {
+			var err error
+			if prefix, err = ioutil.TempDir("", obj.Program+"-"+hostname+"-"); err != nil {
+				return fmt.Errorf("Main: Error: Can't create temporary prefix!")
+			}
+			log.Println("Main: Warning: Working prefix directory is temporary!")
+
+		} else {
+			return fmt.Errorf("Main: Error: Can't create prefix!")
+		}
+	}
+	log.Printf("Main: Working prefix is: %s", prefix)
+	pgraphPrefix := fmt.Sprintf("%s/", path.Join(prefix, "pgraph")) // pgraph namespace
+	if err := os.MkdirAll(pgraphPrefix, 0770); err != nil {
+		return errwrap.Wrapf(err, "Can't create pgraph prefix")
+	}
+
+	if !obj.NoPgp {
+		pgpPrefix := fmt.Sprintf("%s/", path.Join(prefix, "pgp"))
+		if err := os.MkdirAll(pgpPrefix, 0770); err != nil {
+			return errwrap.Wrapf(err, "Can't create pgp prefix")
+		}
+
+		pgpKeyringPath := path.Join(pgpPrefix, pgp.DefaultKeyringFile) // default path
+
+		if p := obj.PgpKeyPath; p != nil {
+			pgpKeyringPath = *p
+		}
+
+		var err error
+		if obj.pgpKeys, err = pgp.Import(pgpKeyringPath); err != nil && !os.IsNotExist(err) {
+			return errwrap.Wrapf(err, "Can't import pgp key")
+		}
+
+		if obj.pgpKeys == nil {
+
+			identity := fmt.Sprintf("%s <%s> %s", obj.Program, "root@"+hostname, "generated by "+obj.Program)
+			if p := obj.PgpIdentity; p != nil {
+				identity = *p
+			}
+
+			name, comment, email, err := pgp.ParseIdentity(identity)
+			if err != nil {
+				return errwrap.Wrapf(err, "Can't parse user string")
+
+			}
+
+			// TODO: Make hash configurable
+			if obj.pgpKeys, err = pgp.Generate(name, comment, email, nil); err != nil {
+				return errwrap.Wrapf(err, "Can't creating pgp key")
+			}
+
+			if err := obj.pgpKeys.SaveKey(pgpKeyringPath); err != nil {
+				return errwrap.Wrapf(err, "Can't save pgp key")
+			}
+		}
+
+		// TODO: Import admin key
+	}
+
+	var G, oldGraph *pgraph.Graph
+
+	// exit after `max-runtime` seconds for no reason at all...
+	if i := obj.MaxRuntime; i > 0 {
+		go func() {
+			time.Sleep(time.Duration(i) * time.Second)
+			obj.Exit(nil)
+		}()
+	}
+
+	// setup converger
+	converger := converger.NewConverger(
+		obj.ConvergedTimeout,
+		nil, // stateFn gets added in by EmbdEtcd
+	)
+	go converger.Loop(true) // main loop for converger, true to start paused
+
+	// embedded etcd
+	if len(obj.seeds) == 0 {
+		log.Printf("Main: Seeds: No seeds specified!")
+	} else {
+		log.Printf("Main: Seeds(%d): %v", len(obj.seeds), obj.seeds)
+	}
+	EmbdEtcd := etcd.NewEmbdEtcd(
+		hostname,
+		obj.seeds,
+		obj.clientURLs,
+		obj.serverURLs,
+		obj.NoServer,
+		obj.idealClusterSize,
+		etcd.Flags{
+			Debug:   obj.Flags.Debug,
+			Trace:   obj.Flags.Trace,
+			Verbose: obj.Flags.Verbose,
+		},
+		prefix,
+		converger,
+	)
+	if EmbdEtcd == nil {
+		// TODO: verify EmbdEtcd is not nil below...
+		obj.Exit(fmt.Errorf("Main: Etcd: Creation failed!"))
+	} else if err := EmbdEtcd.Startup(); err != nil { // startup (returns when etcd main loop is running)
+		obj.Exit(fmt.Errorf("Main: Etcd: Startup failed: %v", err))
+	}
+	convergerStateFn := func(b bool) error {
+		// exit if we are using the converged timeout and we are the
+		// root node. otherwise, if we are a child node in a remote
+		// execution hierarchy, we should only notify our converged
+		// state and wait for the parent to trigger the exit.
+		if t := obj.ConvergedTimeout; obj.Depth == 0 && t >= 0 {
+			if b {
+				log.Printf("Converged for %d seconds, exiting!", t)
+				obj.Exit(nil) // trigger an exit!
+			}
+			return nil
+		}
+		// send our individual state into etcd for others to see
+		return etcd.EtcdSetHostnameConverged(EmbdEtcd, hostname, b) // TODO: what should happen on error?
+	}
+	if EmbdEtcd != nil {
+		converger.SetStateFn(convergerStateFn)
+	}
+
+	var gapiChan chan error // stream events are nil errors
+	if obj.GAPI != nil {
+		data := gapi.Data{
+			Hostname: hostname,
+			// NOTE: alternate implementations can be substituted in
+			World: &etcd.World{
+				Hostname: hostname,
+				EmbdEtcd: EmbdEtcd,
+			},
+			Noop:    obj.Noop,
+			NoWatch: obj.NoWatch,
+		}
+		if err := obj.GAPI.Init(data); err != nil {
+			obj.Exit(fmt.Errorf("Main: GAPI: Init failed: %v", err))
+		} else if !obj.NoWatch {
+			gapiChan = obj.GAPI.Next() // stream of graph switch events!
+		}
+	}
+
+	exitchan := make(chan struct{}) // exit on close
+	go func() {
+		startChan := make(chan struct{}) // start signal
+		go func() { startChan <- struct{}{} }()
+
+		log.Println("Etcd: Starting...")
+		etcdChan := etcd.EtcdWatch(EmbdEtcd)
+		first := true // first loop or not
+		for {
+			log.Println("Main: Waiting...")
+			select {
+			case <-startChan: // kick the loop once at start
+				// pass
+
+			case b := <-etcdChan:
+				if !b { // ignore the message
+					continue
+				}
+				// everything else passes through to cause a compile!
+
+			case err, ok := <-gapiChan:
+				if !ok { // channel closed
+					if obj.Flags.Debug {
+						log.Printf("Main: GAPI exited")
+					}
+					gapiChan = nil // disable it
+					continue
+				}
+				if err != nil {
+					obj.Exit(err) // trigger exit
+					continue
+					//return // TODO: return or wait for exitchan?
+				}
+				if obj.NoWatch { // extra safety for bad GAPI's
+					log.Printf("Main: GAPI stream should be quiet with NoWatch!") // fix the GAPI!
+					continue                                                      // no stream events should be sent
+				}
+
+			case <-exitchan:
+				return
+			}
+
+			if obj.GAPI == nil {
+				log.Printf("Config: GAPI is empty!")
+				continue
+			}
+
+			// we need the vertices to be paused to work on them, so
+			// run graph vertex LOCK...
+			if !first { // TODO: we can flatten this check out I think
+				converger.Pause() // FIXME: add sync wait?
+				G.Pause()         // sync
+
+				//G.UnGroup() // FIXME: implement me if needed!
+			}
+
+			// make the graph from yaml, lib, puppet->yaml, or dsl!
+			newGraph, err := obj.GAPI.Graph() // generate graph!
+			if err != nil {
+				log.Printf("Config: Error creating new graph: %v", err)
+				// unpause!
+				if !first {
+					G.Start(first)    // sync
+					converger.Start() // after G.Start()
+				}
+				continue
+			}
+			newGraph.Flags = pgraph.Flags{Debug: obj.Flags.Debug}
+			// pass in the information we need
+			newGraph.AssociateData(&resources.Data{
+				Converger: converger,
+				Prefix:    pgraphPrefix,
+				Debug:     obj.Flags.Debug,
+			})
+
+			// apply the global noop parameter if requested
+			if obj.Noop {
+				for _, m := range newGraph.GraphMetas() {
+					m.Noop = obj.Noop
+				}
+			}
+
+			// FIXME: make sure we "UnGroup()" any semi-destructive
+			// changes to the resources so our efficient GraphSync
+			// will be able to re-use and cmp to the old graph.
+			newFullGraph, err := newGraph.GraphSync(oldGraph)
+			if err != nil {
+				log.Printf("Config: Error running graph sync: %v", err)
+				// unpause!
+				if !first {
+					G.Start(first)    // sync
+					converger.Start() // after G.Start()
+				}
+				continue
+			}
+			oldGraph = newFullGraph // save old graph
+			G = oldGraph.Copy()     // copy to active graph
+
+			G.AutoEdges() // add autoedges; modifies the graph
+			G.AutoGroup() // run autogroup; modifies the graph
+			// TODO: do we want to do a transitive reduction?
+			// FIXME: run a type checker that verifies all the send->recv relationships
+
+			log.Printf("Graph: %v", G) // show graph
+			if obj.GraphvizFilter != "" {
+				if err := G.ExecGraphviz(obj.GraphvizFilter, obj.Graphviz); err != nil {
+					log.Printf("Graphviz: %v", err)
+				} else {
+					log.Printf("Graphviz: Successfully generated graph!")
+				}
+			}
+			// G.Start(...) needs to be synchronous or wait,
+			// because if half of the nodes are started and
+			// some are not ready yet and the EtcdWatch
+			// loops, we'll cause G.Pause(...) before we
+			// even got going, thus causing nil pointer errors
+			G.Start(first)    // sync
+			converger.Start() // after G.Start()
+			first = false
+		}
+	}()
+
+	configWatcher := recwatch.NewConfigWatcher()
+	configWatcher.Flags = recwatch.Flags{Debug: obj.Flags.Debug}
+	events := configWatcher.Events()
+	if !obj.NoWatch {
+		configWatcher.Add(obj.Remotes...) // add all the files...
+	} else {
+		events = nil // signal that no-watch is true
+	}
+	go func() {
+		select {
+		case err := <-configWatcher.Error():
+			obj.Exit(err) // trigger an exit!
+
+		case <-exitchan:
+			return
+		}
+	}()
+
+	// initialize the add watcher, which calls the f callback on map changes
+	convergerCb := func(f func(map[string]bool) error) (func(), error) {
+		return etcd.EtcdAddHostnameConvergedWatcher(EmbdEtcd, f)
+	}
+
+	// build remotes struct for remote ssh
+	remotes := remote.NewRemotes(
+		EmbdEtcd.LocalhostClientURLs().StringSlice(),
+		[]string{etcd.DefaultClientURL},
+		obj.Noop,
+		obj.Remotes, // list of files
+		events,      // watch for file changes
+		obj.CConns,
+		obj.AllowInteractive,
+		obj.SSHPrivIDRsa,
+		!obj.NoCaching,
+		obj.Depth,
+		prefix,
+		converger,
+		convergerCb,
+		remote.Flags{
+			Program: obj.Program,
+			Debug:   obj.Flags.Debug,
+		},
+	)
+
+	// TODO: is there any benefit to running the remotes above in the loop?
+	// wait for etcd to be running before we remote in, which we do above!
+	go remotes.Run()
+
+	if obj.GAPI == nil {
+		converger.Start() // better start this for empty graphs
+	}
+	log.Println("Main: Running...")
+
+	reterr := <-obj.exit // wait for exit signal
+
+	log.Println("Destroy...")
+
+	if obj.GAPI != nil {
+		if err := obj.GAPI.Close(); err != nil {
+			err = errwrap.Wrapf(err, "GAPI closed poorly!")
+			reterr = multierr.Append(reterr, err) // list of errors
+		}
+	}
+
+	configWatcher.Close()                  // stop sending file changes to remotes
+	if err := remotes.Exit(); err != nil { // tell all the remote connections to shutdown; waits!
+		err = errwrap.Wrapf(err, "Remote exited poorly!")
+		reterr = multierr.Append(reterr, err) // list of errors
+	}
+
+	// tell inner main loop to exit
+	close(exitchan)
+
+	G.Exit() // tell all the children to exit, and waits for them to do so
+
+	// cleanup etcd main loop last so it can process everything first
+	if err := EmbdEtcd.Destroy(); err != nil { // shutdown and cleanup etcd
+		err = errwrap.Wrapf(err, "Etcd exited poorly!")
+		reterr = multierr.Append(reterr, err) // list of errors
+	}
+
+	if obj.Flags.Debug {
+		log.Printf("Main: Graph: %v", G)
+	}
+
+	// TODO: wait for each vertex to exit...
+	log.Println("Goodbye!")
+	return reterr
+}

main.go

@@ -19,576 +19,33 @@ package main
 
 import (
 	"fmt"
-	"io/ioutil"
-	"log"
 	"os"
-	"os/signal"
-	"sync"
-	"syscall"
-	"time"
 
-	"github.com/purpleidea/mgmt/converger"
+	mgmt "github.com/purpleidea/mgmt/lib"
-	"github.com/purpleidea/mgmt/etcd"
+)
-	"github.com/purpleidea/mgmt/gconfig"
-	"github.com/purpleidea/mgmt/global"
-	"github.com/purpleidea/mgmt/pgraph"
-	"github.com/purpleidea/mgmt/puppet"
-	"github.com/purpleidea/mgmt/recwatch"
-	"github.com/purpleidea/mgmt/remote"
-	"github.com/purpleidea/mgmt/util"
 
-	etcdtypes "github.com/coreos/etcd/pkg/types"
+// These constants are some global variables that are used throughout the code.
-	"github.com/coreos/pkg/capnslog"
+const (
-	"github.com/urfave/cli"
+	DEBUG   = false // add additional log messages
+	TRACE   = false // add execution flow log messages
+	VERBOSE = false // add extra log message output
 )
 
 // set at compile time
 var (
 	program string
 	version string
-	prefix  = fmt.Sprintf("/var/lib/%s/", program)
 )
 
-// signal handler
-func waitForSignal(exit chan error) error {
-	signals := make(chan os.Signal, 1)
-	signal.Notify(signals, os.Interrupt) // catch ^C
-	//signal.Notify(signals, os.Kill) // catch signals
-	signal.Notify(signals, syscall.SIGTERM)
-
-	select {
-	case sig := <-signals: // any signal will do
-		if sig == os.Interrupt {
-			log.Println("Interrupted by ^C")
-			return nil
-		} else {
-			log.Println("Interrupted by signal")
-			return fmt.Errorf("Killed by %v", sig)
-		}
-	case err := <-exit: // or a manual signal
-		log.Println("Interrupted by exit signal")
-		return err
-	}
-}
-
-// run is the main run target.
-func run(c *cli.Context) error {
-	var start = time.Now().UnixNano()
-	log.Printf("This is: %v, version: %v", program, version)
-	log.Printf("Main: Start: %v", start)
-
-	hostname, _ := os.Hostname()
-	// allow passing in the hostname, instead of using --hostname
-	if c.IsSet("file") {
-		if config := gconfig.ParseConfigFromFile(c.String("file")); config != nil {
-			if h := config.Hostname; h != "" {
-				hostname = h
-			}
-		}
-	}
-	if c.IsSet("hostname") { // override by cli
-		if h := c.String("hostname"); h != "" {
-			hostname = h
-		}
-	}
-	noop := c.Bool("noop")
-
-	seeds, err := etcdtypes.NewURLs(
-		util.FlattenListWithSplit(c.StringSlice("seeds"), []string{",", ";", " "}),
-	)
-	if err != nil && len(c.StringSlice("seeds")) > 0 {
-		log.Printf("Main: Error: seeds didn't parse correctly!")
-		return cli.NewExitError("", 1)
-	}
-	clientURLs, err := etcdtypes.NewURLs(
-		util.FlattenListWithSplit(c.StringSlice("client-urls"), []string{",", ";", " "}),
-	)
-	if err != nil && len(c.StringSlice("client-urls")) > 0 {
-		log.Printf("Main: Error: clientURLs didn't parse correctly!")
-		return cli.NewExitError("", 1)
-	}
-	serverURLs, err := etcdtypes.NewURLs(
-		util.FlattenListWithSplit(c.StringSlice("server-urls"), []string{",", ";", " "}),
-	)
-	if err != nil && len(c.StringSlice("server-urls")) > 0 {
-		log.Printf("Main: Error: serverURLs didn't parse correctly!")
-		return cli.NewExitError("", 1)
-	}
-
-	idealClusterSize := uint16(c.Int("ideal-cluster-size"))
-	if idealClusterSize < 1 {
-		log.Printf("Main: Error: idealClusterSize should be at least one!")
-		return cli.NewExitError("", 1)
-	}
-
-	if c.IsSet("file") && c.IsSet("puppet") {
-		log.Println("Main: Error: the --file and --puppet parameters cannot be used together!")
-		return cli.NewExitError("", 1)
-	}
-
-	if c.Bool("no-server") && len(c.StringSlice("remote")) > 0 {
-		// TODO: in this case, we won't be able to tunnel stuff back to
-		// here, so if we're okay with every remote graph running in an
-		// isolated mode, then this is okay. Improve on this if there's
-		// someone who really wants to be able to do this.
-		log.Println("Main: Error: the --no-server and --remote parameters cannot be used together!")
-		return cli.NewExitError("", 1)
-	}
-
-	cConns := uint16(c.Int("cconns"))
-	if cConns < 0 {
-		log.Printf("Main: Error: --cconns should be at least zero!")
-		return cli.NewExitError("", 1)
-	}
-
-	if c.IsSet("converged-timeout") && cConns > 0 && len(c.StringSlice("remote")) > c.Int("cconns") {
-		log.Printf("Main: Error: combining --converged-timeout with more remotes than available connections will never converge!")
-		return cli.NewExitError("", 1)
-	}
-
-	depth := uint16(c.Int("depth"))
-	if depth < 0 { // user should not be using this argument manually
-		log.Printf("Main: Error: negative values for --depth are not permitted!")
-		return cli.NewExitError("", 1)
-	}
-
-	if c.IsSet("prefix") && c.Bool("tmp-prefix") {
-		log.Println("Main: Error: combining --prefix and the request for a tmp prefix is illogical!")
-		return cli.NewExitError("", 1)
-	}
-	if s := c.String("prefix"); c.IsSet("prefix") && s != "" {
-		prefix = s
-	}
-
-	// make sure the working directory prefix exists
-	if c.Bool("tmp-prefix") || os.MkdirAll(prefix, 0770) != nil {
-		if c.Bool("tmp-prefix") || c.Bool("allow-tmp-prefix") {
-			if prefix, err = ioutil.TempDir("", program+"-"); err != nil {
-				log.Printf("Main: Error: Can't create temporary prefix!")
-				return cli.NewExitError("", 1)
-			}
-			log.Println("Main: Warning: Working prefix directory is temporary!")
-
-		} else {
-			log.Printf("Main: Error: Can't create prefix!")
-			return cli.NewExitError("", 1)
-		}
-	}
-	log.Printf("Main: Working prefix is: %s", prefix)
-
-	var wg sync.WaitGroup
-	exit := make(chan error) // exit signal
-	var G, fullGraph *pgraph.Graph
-
-	// exit after `max-runtime` seconds for no reason at all...
-	if i := c.Int("max-runtime"); i > 0 {
-		go func() {
-			time.Sleep(time.Duration(i) * time.Second)
-			exit <- nil
-		}()
-	}
-
-	// setup converger
-	converger := converger.NewConverger(
-		c.Int("converged-timeout"),
-		nil, // stateFn gets added in by EmbdEtcd
-	)
-	go converger.Loop(true) // main loop for converger, true to start paused
-
-	// embedded etcd
-	if len(seeds) == 0 {
-		log.Printf("Main: Seeds: No seeds specified!")
-	} else {
-		log.Printf("Main: Seeds(%v): %v", len(seeds), seeds)
-	}
-	EmbdEtcd := etcd.NewEmbdEtcd(
-		hostname,
-		seeds,
-		clientURLs,
-		serverURLs,
-		c.Bool("no-server"),
-		idealClusterSize,
-		prefix,
-		converger,
-	)
-	if EmbdEtcd == nil {
-		// TODO: verify EmbdEtcd is not nil below...
-		exit <- fmt.Errorf("Main: Etcd: Creation failed!")
-	} else if err := EmbdEtcd.Startup(); err != nil { // startup (returns when etcd main loop is running)
-		exit <- fmt.Errorf("Main: Etcd: Startup failed: %v", err)
-	}
-	convergerStateFn := func(b bool) error {
-		// exit if we are using the converged-timeout and we are the
-		// root node. otherwise, if we are a child node in a remote
-		// execution hierarchy, we should only notify our converged
-		// state and wait for the parent to trigger the exit.
-		if depth == 0 && c.Int("converged-timeout") >= 0 {
-			if b {
-				log.Printf("Converged for %d seconds, exiting!", c.Int("converged-timeout"))
-				exit <- nil // trigger an exit!
-			}
-			return nil
-		}
-		// send our individual state into etcd for others to see
-		return etcd.EtcdSetHostnameConverged(EmbdEtcd, hostname, b) // TODO: what should happen on error?
-	}
-	if EmbdEtcd != nil {
-		converger.SetStateFn(convergerStateFn)
-	}
-
-	exitchan := make(chan struct{}) // exit on close
-	go func() {
-		startchan := make(chan struct{}) // start signal
-		go func() { startchan <- struct{}{} }()
-		file := c.String("file")
-		var configchan chan error
-		var puppetchan <-chan time.Time
-		if !c.Bool("no-watch") && c.IsSet("file") {
-			configchan = recwatch.ConfigWatch(file)
-		} else if c.IsSet("puppet") {
-			interval := puppet.PuppetInterval(c.String("puppet-conf"))
-			puppetchan = time.Tick(time.Duration(interval) * time.Second)
-		}
-		log.Println("Etcd: Starting...")
-		etcdchan := etcd.EtcdWatch(EmbdEtcd)
-		first := true // first loop or not
-		for {
-			log.Println("Main: Waiting...")
-			select {
-			case <-startchan: // kick the loop once at start
-				// pass
-
-			case b := <-etcdchan:
-				if !b { // ignore the message
-					continue
-				}
-				// everything else passes through to cause a compile!
-
-			case <-puppetchan:
-				// nothing, just go on
-
-			case e := <-configchan:
-				if c.Bool("no-watch") {
-					continue // not ready to read config
-				}
-				if e != nil {
-					exit <- e // trigger exit
-					continue
-					//return // TODO: return or wait for exitchan?
-				}
-			// XXX: case compile_event: ...
-			// ...
-			case <-exitchan:
-				return
-			}
-
-			var config *gconfig.GraphConfig
-			if c.IsSet("file") {
-				config = gconfig.ParseConfigFromFile(file)
-			} else if c.IsSet("puppet") {
-				config = puppet.ParseConfigFromPuppet(c.String("puppet"), c.String("puppet-conf"))
-			}
-			if config == nil {
-				log.Printf("Config: Parse failure")
-				continue
-			}
-
-			if config.Hostname != "" && config.Hostname != hostname {
-				log.Printf("Config: Hostname changed, ignoring config!")
-				continue
-			}
-			config.Hostname = hostname // set it in case it was ""
-
-			// run graph vertex LOCK...
-			if !first { // TODO: we can flatten this check out I think
-				converger.Pause() // FIXME: add sync wait?
-				G.Pause()         // sync
-			}
-
-			// build graph from yaml file on events (eg: from etcd)
-			// we need the vertices to be paused to work on them
-			if newFullgraph, err := config.NewGraphFromConfig(fullGraph, EmbdEtcd, noop); err == nil { // keep references to all original elements
-				fullGraph = newFullgraph
-			} else {
-				log.Printf("Config: Error making new graph from config: %v", err)
-				// unpause!
-				if !first {
-					G.Start(&wg, first) // sync
-					converger.Start()   // after G.Start()
-				}
-				continue
-			}
-
-			G = fullGraph.Copy() // copy to active graph
-			// XXX: do etcd transaction out here...
-			G.AutoEdges() // add autoedges; modifies the graph
-			G.AutoGroup() // run autogroup; modifies the graph
-			// TODO: do we want to do a transitive reduction?
-
-			log.Printf("Graph: %v", G) // show graph
-			err := G.ExecGraphviz(c.String("graphviz-filter"), c.String("graphviz"))
-			if err != nil {
-				log.Printf("Graphviz: %v", err)
-			} else {
-				log.Printf("Graphviz: Successfully generated graph!")
-			}
-			G.AssociateData(converger)
-			// G.Start(...) needs to be synchronous or wait,
-			// because if half of the nodes are started and
-			// some are not ready yet and the EtcdWatch
-			// loops, we'll cause G.Pause(...) before we
-			// even got going, thus causing nil pointer errors
-			G.Start(&wg, first) // sync
-			converger.Start()   // after G.Start()
-			first = false
-		}
-	}()
-
-	configWatcher := recwatch.NewConfigWatcher()
-	events := configWatcher.Events()
-	if !c.Bool("no-watch") {
-		configWatcher.Add(c.StringSlice("remote")...) // add all the files...
-	} else {
-		events = nil // signal that no-watch is true
-	}
-	go func() {
-		select {
-		case err := <-configWatcher.Error():
-			exit <- err // trigger an exit!
-
-		case <-exitchan:
-			return
-		}
-	}()
-
-	// initialize the add watcher, which calls the f callback on map changes
-	convergerCb := func(f func(map[string]bool) error) (func(), error) {
-		return etcd.EtcdAddHostnameConvergedWatcher(EmbdEtcd, f)
-	}
-
-	// build remotes struct for remote ssh
-	remotes := remote.NewRemotes(
-		EmbdEtcd.LocalhostClientURLs().StringSlice(),
-		[]string{etcd.DefaultClientURL},
-		noop,
-		c.StringSlice("remote"), // list of files
-		events,                  // watch for file changes
-		cConns,
-		c.Bool("allow-interactive"),
-		c.String("ssh-priv-id-rsa"),
-		!c.Bool("no-caching"),
-		depth,
-		prefix,
-		converger,
-		convergerCb,
-		program,
-	)
-
-	// TODO: is there any benefit to running the remotes above in the loop?
-	// wait for etcd to be running before we remote in, which we do above!
-	go remotes.Run()
-
-	if !c.IsSet("file") && !c.IsSet("puppet") {
-		converger.Start() // better start this for empty graphs
-	}
-	log.Println("Main: Running...")
-
-	err = waitForSignal(exit) // pass in exit channel to watch
-
-	log.Println("Destroy...")
-
-	configWatcher.Close() // stop sending file changes to remotes
-	remotes.Exit()        // tell all the remote connections to shutdown; waits!
-
-	G.Exit() // tell all the children to exit
-
-	// tell inner main loop to exit
-	close(exitchan)
-
-	// cleanup etcd main loop last so it can process everything first
-	if err := EmbdEtcd.Destroy(); err != nil { // shutdown and cleanup etcd
-		log.Printf("Etcd exited poorly with: %v", err)
-	}
-
-	if global.DEBUG {
-		log.Printf("Graph: %v", G)
-	}
-
-	wg.Wait() // wait for primary go routines to exit
-
-	// TODO: wait for each vertex to exit...
-	log.Println("Goodbye!")
-	return err
-}
-
 func main() {
-	var flags int
+	flags := mgmt.Flags{
-	if global.DEBUG || true { // TODO: remove || true
+		Debug:   DEBUG,
-		flags = log.LstdFlags | log.Lshortfile
+		Trace:   TRACE,
+		Verbose: VERBOSE,
 	}
-	flags = (flags - log.Ldate) // remove the date for now
+	if err := mgmt.CLI(program, version, flags); err != nil {
-	log.SetFlags(flags)
+		fmt.Println(err)
-
+		os.Exit(1)
-	// un-hijack from capnslog...
+		return
-	log.SetOutput(os.Stderr)
-	if global.VERBOSE {
-		capnslog.SetFormatter(capnslog.NewLogFormatter(os.Stderr, "(etcd) ", flags))
-	} else {
-		capnslog.SetFormatter(capnslog.NewNilFormatter())
 	}
-
-	// test for sanity
-	if program == "" || version == "" {
-		log.Fatal("Program was not compiled correctly. Please see Makefile.")
-	}
-	app := cli.NewApp()
-	app.Name = program
-	app.Usage = "next generation config management"
-	app.Version = version
-	//app.Action = ... // without a default action, help runs
-
-	app.Commands = []cli.Command{
-		{
-			Name:    "run",
-			Aliases: []string{"r"},
-			Usage:   "run",
-			Action:  run,
-			Flags: []cli.Flag{
-				cli.StringFlag{
-					Name:   "file, f",
-					Value:  "",
-					Usage:  "graph definition to run",
-					EnvVar: "MGMT_FILE",
-				},
-				cli.BoolFlag{
-					Name:  "no-watch",
-					Usage: "do not update graph on watched graph definition file changes",
-				},
-				cli.StringFlag{
-					Name:  "code, c",
-					Value: "",
-					Usage: "code definition to run",
-				},
-				cli.StringFlag{
-					Name:  "graphviz, g",
-					Value: "",
-					Usage: "output file for graphviz data",
-				},
-				cli.StringFlag{
-					Name:  "graphviz-filter, gf",
-					Value: "dot", // directed graph default
-					Usage: "graphviz filter to use",
-				},
-				// useful for testing multiple instances on same machine
-				cli.StringFlag{
-					Name:  "hostname",
-					Value: "",
-					Usage: "hostname to use",
-				},
-				// if empty, it will startup a new server
-				cli.StringSliceFlag{
-					Name:   "seeds, s",
-					Value:  &cli.StringSlice{}, // empty slice
-					Usage:  "default etc client endpoint",
-					EnvVar: "MGMT_SEEDS",
-				},
-				// port 2379 and 4001 are common
-				cli.StringSliceFlag{
-					Name:   "client-urls",
-					Value:  &cli.StringSlice{},
-					Usage:  "list of URLs to listen on for client traffic",
-					EnvVar: "MGMT_CLIENT_URLS",
-				},
-				// port 2380 and 7001 are common
-				cli.StringSliceFlag{
-					Name:   "server-urls, peer-urls",
-					Value:  &cli.StringSlice{},
-					Usage:  "list of URLs to listen on for server (peer) traffic",
-					EnvVar: "MGMT_SERVER_URLS",
-				},
-				cli.BoolFlag{
-					Name:  "no-server",
-					Usage: "do not let other servers peer with me",
-				},
-				cli.IntFlag{
-					Name:   "ideal-cluster-size",
-					Value:  etcd.DefaultIdealClusterSize,
-					Usage:  "ideal number of server peers in cluster, only read by initial server",
-					EnvVar: "MGMT_IDEAL_CLUSTER_SIZE",
-				},
-				cli.IntFlag{
-					Name:   "converged-timeout, t",
-					Value:  -1,
-					Usage:  "exit after approximately this many seconds in a converged state",
-					EnvVar: "MGMT_CONVERGED_TIMEOUT",
-				},
-				cli.IntFlag{
-					Name:   "max-runtime",
-					Value:  0,
-					Usage:  "exit after a maximum of approximately this many seconds",
-					EnvVar: "MGMT_MAX_RUNTIME",
-				},
-				cli.BoolFlag{
-					Name:  "noop",
-					Usage: "globally force all resources into no-op mode",
-				},
-				cli.StringFlag{
-					Name:  "puppet, p",
-					Value: "",
-					Usage: "load graph from puppet, optionally takes a manifest or path to manifest file",
-				},
-				cli.StringFlag{
-					Name:  "puppet-conf",
-					Value: "",
-					Usage: "supply the path to an alternate puppet.conf file to use",
-				},
-				cli.StringSliceFlag{
-					Name:  "remote",
-					Value: &cli.StringSlice{},
-					Usage: "list of remote graph definitions to run",
-				},
-				cli.BoolFlag{
-					Name:  "allow-interactive",
-					Usage: "allow interactive prompting, such as for remote passwords",
-				},
-				cli.StringFlag{
-					Name:   "ssh-priv-id-rsa",
-					Value:  "~/.ssh/id_rsa",
-					Usage:  "default path to ssh key file, set empty to never touch",
-					EnvVar: "MGMT_SSH_PRIV_ID_RSA",
-				},
-				cli.IntFlag{
-					Name:   "cconns",
-					Value:  0,
-					Usage:  "number of maximum concurrent remote ssh connections to run, 0 for unlimited",
-					EnvVar: "MGMT_CCONNS",
-				},
-				cli.BoolFlag{
-					Name:  "no-caching",
-					Usage: "don't allow remote caching of remote execution binary",
-				},
-				cli.IntFlag{
-					Name:   "depth",
-					Hidden: true, // internal use only
-					Value:  0,
-					Usage:  "specify depth in remote hierarchy",
-				},
-				cli.StringFlag{
-					Name:   "prefix",
-					Usage:  "specify a path to the working prefix directory",
-					EnvVar: "MGMT_PREFIX",
-				},
-				cli.BoolFlag{
-					Name:  "tmp-prefix",
-					Usage: "request a pseudo-random, temporary prefix to be used",
-				},
-				cli.BoolFlag{
-					Name:  "allow-tmp-prefix",
-					Usage: "allow creation of a new temporary prefix if main prefix is unavailable",
-				},
-			},
-		},
-	}
-	app.EnableBashCompletion = true
-	app.Run(os.Args)
 }

misc/make-deps.sh

@@ -11,13 +11,30 @@ fi
 
 sudo_command=$(which sudo)
 
-if [ $travis -eq 0 ]; then
 YUM=`which yum 2>/dev/null`
+DNF=`which dnf 2>/dev/null`
 APT=`which apt-get 2>/dev/null`
+
+# if DNF is available use it
+if [ -x "$DNF" ]; then
+	YUM=$DNF
+fi
+
 if [ -z "$YUM" -a -z "$APT" ]; then
 	echo "The package managers can't be found."
 	exit 1
 fi
+
+if [ ! -z "$YUM" ]; then
+	$sudo_command $YUM install -y libvirt-devel
+
+fi
+if [ ! -z "$APT" ]; then
+	$sudo_command $APT install -y libvirt-dev || true
+	$sudo_command $APT install -y libpcap0.8-dev || true
+fi
+
+if [ $travis -eq 0 ]; then
 	if [ ! -z "$YUM" ]; then
 		# some go dependencies are stored in mercurial
 		$sudo_command $YUM install -y golang golang-googlecode-tools-stringer hg
@@ -29,7 +46,6 @@ if [ $travis -eq 0 ]; then
 		# one of these two golang tools packages should work on debian
 		$sudo_command $APT install -y golang-golang-x-tools || true
 		$sudo_command $APT install -y golang-go.tools || true
-		$sudo_command $APT install -y libpcap0.8-dev || true
 	fi
 fi
 
@@ -39,7 +55,7 @@ if go version | grep 'go1\.[0123]\.'; then
 	exit 1
 fi
 
-go get ./...	# get all the go dependencies
+go get -d ./...	# get all the go dependencies
 [ -e "$GOBIN/mgmt" ] && rm -f "$GOBIN/mgmt"	# the `go get` version has no -X
 # vet is built-in in go 1.6 - we check for go vet command
 go vet 1> /dev/null 2>&1

pgp/pgp.go

@@ -0,0 +1,230 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgp
+
+import (
+	"bufio"
+	"bytes"
+	"crypto"
+	"encoding/base64"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+
+	errwrap "github.com/pkg/errors"
+	"golang.org/x/crypto/openpgp"
+	"golang.org/x/crypto/openpgp/packet"
+)
+
+// DefaultKeyringFile is the default file name for keyrings.
+const DefaultKeyringFile = "keyring.pgp"
+
+// CONFIG set default Hash.
+var CONFIG packet.Config
+
+func init() {
+	CONFIG.DefaultHash = crypto.SHA256
+}
+
+// PGP contains base entity.
+type PGP struct {
+	Entity *openpgp.Entity
+}
+
+// Import private key from defined path.
+func Import(privKeyPath string) (*PGP, error) {
+
+	privKeyFile, err := os.Open(privKeyPath)
+	if err != nil {
+		return nil, err
+	}
+	defer privKeyFile.Close()
+
+	file := packet.NewReader(bufio.NewReader(privKeyFile))
+	entity, err := openpgp.ReadEntity(file)
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "can't read entity from path")
+	}
+
+	obj := &PGP{
+		Entity: entity,
+	}
+
+	log.Printf("PGP: Imported key: %s", obj.Entity.PrivateKey.KeyIdShortString())
+	return obj, nil
+}
+
+// Generate creates new key pair. This key pair must be saved or it will be lost.
+func Generate(name, comment, email string, hash *crypto.Hash) (*PGP, error) {
+	if hash != nil {
+		CONFIG.DefaultHash = *hash
+	}
+	// generate a new public/private key pair
+	entity, err := openpgp.NewEntity(name, comment, email, &CONFIG)
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "can't generate entity")
+	}
+
+	obj := &PGP{
+		Entity: entity,
+	}
+
+	log.Printf("PGP: Created key: %s", obj.Entity.PrivateKey.KeyIdShortString())
+	return obj, nil
+}
+
+// SaveKey writes the whole entity (including private key!) to a .gpg file.
+func (obj *PGP) SaveKey(path string) error {
+	f, err := os.Create(path)
+	if err != nil {
+		return errwrap.Wrapf(err, "can't create file from given path")
+	}
+
+	w := bufio.NewWriter(f)
+	if err != nil {
+		return errwrap.Wrapf(err, "can't create writer")
+	}
+
+	if err := obj.Entity.SerializePrivate(w, &CONFIG); err != nil {
+		return errwrap.Wrapf(err, "can't serialize private key")
+	}
+
+	for _, ident := range obj.Entity.Identities {
+		for _, sig := range ident.Signatures {
+			if err := sig.Serialize(w); err != nil {
+				return errwrap.Wrapf(err, "can't serialize signature")
+			}
+		}
+	}
+
+	if err := w.Flush(); err != nil {
+		return errwrap.Wrapf(err, "enable to flush writer")
+	}
+
+	return nil
+}
+
+// WriteFile from given buffer in specified path.
+func (obj *PGP) WriteFile(path string, buff *bytes.Buffer) error {
+	w, err := createWriter(path)
+	if err != nil {
+		return errwrap.Wrapf(err, "can't create writer")
+	}
+	buff.WriteTo(w)
+
+	if err := w.Flush(); err != nil {
+		return errwrap.Wrapf(err, "can't flush buffered data")
+	}
+	return nil
+}
+
+// CreateWriter remove duplicate function.
+func createWriter(path string) (*bufio.Writer, error) {
+	f, err := os.Create(path)
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "can't create file from given path")
+	}
+	return bufio.NewWriter(f), nil
+}
+
+// Encrypt message for specified entity.
+func (obj *PGP) Encrypt(to *openpgp.Entity, msg string) (string, error) {
+	buf, err := obj.EncryptMsg(to, msg)
+	if err != nil {
+		return "", errwrap.Wrapf(err, "can't encrypt message")
+	}
+
+	// encode to base64
+	bytes, err := ioutil.ReadAll(buf)
+	if err != nil {
+		return "", errwrap.Wrapf(err, "can't read unverified body")
+	}
+	return base64.StdEncoding.EncodeToString(bytes), nil
+}
+
+// EncryptMsg encrypts the message.
+func (obj *PGP) EncryptMsg(to *openpgp.Entity, msg string) (*bytes.Buffer, error) {
+	ents := []*openpgp.Entity{to}
+
+	buf := new(bytes.Buffer)
+	w, err := openpgp.Encrypt(buf, ents, obj.Entity, nil, nil)
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "can't encrypt message")
+	}
+
+	_, err = w.Write([]byte(msg))
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "can't write to buffer")
+	}
+
+	if err = w.Close(); err != nil {
+		return nil, errwrap.Wrapf(err, "can't close writer")
+	}
+	return buf, nil
+}
+
+// Decrypt an encrypted msg.
+func (obj *PGP) Decrypt(encString string) (string, error) {
+	entityList := openpgp.EntityList{obj.Entity}
+
+	// decode the base64 string
+	dec, err := base64.StdEncoding.DecodeString(encString)
+	if err != nil {
+		return "", errwrap.Wrapf(err, "fail at decoding encrypted string")
+	}
+
+	// decrypt it with the contents of the private key
+	md, err := openpgp.ReadMessage(bytes.NewBuffer(dec), entityList, nil, nil)
+	if err != nil {
+		return "", errwrap.Wrapf(err, "can't read message")
+	}
+
+	bytes, err := ioutil.ReadAll(md.UnverifiedBody)
+	if err != nil {
+		return "", errwrap.Wrapf(err, "can't read unverified body")
+	}
+	return string(bytes), nil
+}
+
+// GetIdentities return the first identities from current object.
+func (obj *PGP) GetIdentities() (string, error) {
+	identities := []*openpgp.Identity{}
+
+	for _, v := range obj.Entity.Identities {
+		identities = append(identities, v)
+	}
+	return identities[0].Name, nil
+}
+
+// ParseIdentity parses an identity into name, comment and email components.
+func ParseIdentity(identity string) (name, comment, email string, err error) {
+	// get name
+	n := strings.Split(identity, " <")
+	if len(n) != 2 {
+		return "", "", "", errwrap.Wrap(err, "user string mal formated")
+	}
+
+	// get email and comment
+	ec := strings.Split(n[1], "> ")
+	if len(ec) != 2 {
+		return "", "", "", errwrap.Wrap(err, "user string mal formated")
+	}
+
+	return n[0], ec[1], ec[0], nil
+}

pgraph/actions.go

@@ -0,0 +1,535 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgraph
+
+import (
+	"fmt"
+	"log"
+	"math"
+	"sync"
+	"time"
+
+	"github.com/purpleidea/mgmt/event"
+	"github.com/purpleidea/mgmt/resources"
+
+	errwrap "github.com/pkg/errors"
+)
+
+// GetTimestamp returns the timestamp of a vertex
+func (v *Vertex) GetTimestamp() int64 {
+	return v.timestamp
+}
+
+// UpdateTimestamp updates the timestamp on a vertex and returns the new value
+func (v *Vertex) UpdateTimestamp() int64 {
+	v.timestamp = time.Now().UnixNano() // update
+	return v.timestamp
+}
+
+// OKTimestamp returns true if this element can run right now?
+func (g *Graph) OKTimestamp(v *Vertex) bool {
+	// these are all the vertices pointing TO v, eg: ??? -> v
+	for _, n := range g.IncomingGraphVertices(v) {
+		// if the vertex has a greater timestamp than any pre-req (n)
+		// then we can't run right now...
+		// if they're equal (eg: on init of 0) then we also can't run
+		// b/c we should let our pre-req's go first...
+		x, y := v.GetTimestamp(), n.GetTimestamp()
+		if g.Flags.Debug {
+			log.Printf("%s[%s]: OKTimestamp: (%v) >= %s[%s](%v): !%v", v.Kind(), v.GetName(), x, n.Kind(), n.GetName(), y, x >= y)
+		}
+		if x >= y {
+			return false
+		}
+	}
+	return true
+}
+
+// Poke notifies nodes after me in the dependency graph that they need refreshing...
+// NOTE: this assumes that this can never fail or need to be rescheduled
+func (g *Graph) Poke(v *Vertex, activity bool) error {
+	var wg sync.WaitGroup
+	// these are all the vertices pointing AWAY FROM v, eg: v -> ???
+	for _, n := range g.OutgoingGraphVertices(v) {
+		// XXX: if we're in state event and haven't been cancelled by
+		// apply, then we can cancel a poke to a child, right? XXX
+		// XXX: if n.Res.getState() != resources.ResStateEvent || activity { // is this correct?
+		if true || activity { // XXX: ???
+			if g.Flags.Debug {
+				log.Printf("%s[%s]: Poke: %s[%s]", v.Kind(), v.GetName(), n.Kind(), n.GetName())
+			}
+			wg.Add(1)
+			go func(nn *Vertex) error {
+				defer wg.Done()
+				edge := g.Adjacency[v][nn] // lookup
+				notify := edge.Notify && edge.Refresh()
+
+				// FIXME: is it okay that this is sync?
+				nn.SendEvent(event.EventPoke, true, notify)
+				// TODO: check return value?
+				return nil // never error for now...
+			}(n)
+
+		} else {
+			if g.Flags.Debug {
+				log.Printf("%s[%s]: Poke: %s[%s]: Skipped!", v.Kind(), v.GetName(), n.Kind(), n.GetName())
+			}
+		}
+	}
+	wg.Wait() // wait for all the pokes to complete
+	return nil
+}
+
+// BackPoke pokes the pre-requisites that are stale and need to run before I can run.
+func (g *Graph) BackPoke(v *Vertex) {
+	// these are all the vertices pointing TO v, eg: ??? -> v
+	for _, n := range g.IncomingGraphVertices(v) {
+		x, y, s := v.GetTimestamp(), n.GetTimestamp(), n.Res.GetState()
+		// if the parent timestamp needs poking AND it's not in state
+		// ResStateEvent, then poke it. If the parent is in ResStateEvent it
+		// means that an event is pending, so we'll be expecting a poke
+		// back soon, so we can safely discard the extra parent poke...
+		// TODO: implement a stateLT (less than) to tell if something
+		// happens earlier in the state cycle and that doesn't wrap nil
+		if x >= y && (s != resources.ResStateEvent && s != resources.ResStateCheckApply) {
+			if g.Flags.Debug {
+				log.Printf("%s[%s]: BackPoke: %s[%s]", v.Kind(), v.GetName(), n.Kind(), n.GetName())
+			}
+			// FIXME: is it okay that this is sync?
+			n.SendEvent(event.EventBackPoke, true, false)
+		} else {
+			if g.Flags.Debug {
+				log.Printf("%s[%s]: BackPoke: %s[%s]: Skipped!", v.Kind(), v.GetName(), n.Kind(), n.GetName())
+			}
+		}
+	}
+}
+
+// RefreshPending determines if any previous nodes have a refresh pending here.
+// If this is true, it means I am expected to apply a refresh when I next run.
+func (g *Graph) RefreshPending(v *Vertex) bool {
+	var refresh bool
+	for _, edge := range g.IncomingGraphEdges(v) {
+		// if we asked for a notify *and* if one is pending!
+		if edge.Notify && edge.Refresh() {
+			refresh = true
+			break
+		}
+	}
+	return refresh
+}
+
+// SetUpstreamRefresh sets the refresh value to any upstream vertices.
+func (g *Graph) SetUpstreamRefresh(v *Vertex, b bool) {
+	for _, edge := range g.IncomingGraphEdges(v) {
+		if edge.Notify {
+			edge.SetRefresh(b)
+		}
+	}
+}
+
+// SetDownstreamRefresh sets the refresh value to any downstream vertices.
+func (g *Graph) SetDownstreamRefresh(v *Vertex, b bool) {
+	for _, edge := range g.OutgoingGraphEdges(v) {
+		// if we asked for a notify *and* if one is pending!
+		if edge.Notify {
+			edge.SetRefresh(b)
+		}
+	}
+}
+
+// Process is the primary function to execute for a particular vertex in the graph.
+func (g *Graph) Process(v *Vertex) error {
+	obj := v.Res
+	if g.Flags.Debug {
+		log.Printf("%s[%s]: Process()", obj.Kind(), obj.GetName())
+	}
+	obj.SetState(resources.ResStateEvent)
+	var ok = true
+	var applied = false // did we run an apply?
+	// is it okay to run dependency wise right now?
+	// if not, that's okay because when the dependency runs, it will poke
+	// us back and we will run if needed then!
+	if g.OKTimestamp(v) {
+		if g.Flags.Debug {
+			log.Printf("%s[%s]: OKTimestamp(%v)", obj.Kind(), obj.GetName(), v.GetTimestamp())
+		}
+
+		obj.SetState(resources.ResStateCheckApply)
+
+		// connect any senders to receivers and detect if values changed
+		if updated, err := obj.SendRecv(obj); err != nil {
+			return errwrap.Wrapf(err, "could not SendRecv in Process")
+		} else if len(updated) > 0 {
+			for _, changed := range updated {
+				if changed { // at least one was updated
+					obj.StateOK(false) // invalidate cache, mark as dirty
+					break
+				}
+			}
+		}
+
+		var noop = obj.Meta().Noop // lookup the noop value
+		var refresh bool
+		var checkOK bool
+		var err error
+
+		if g.Flags.Debug {
+			log.Printf("%s[%s]: CheckApply(%t)", obj.Kind(), obj.GetName(), !noop)
+		}
+
+		// lookup the refresh (notification) variable
+		refresh = g.RefreshPending(v) // do i need to perform a refresh?
+		obj.SetRefresh(refresh)       // tell the resource
+
+		// check cached state, to skip CheckApply; can't skip if refreshing
+		if !refresh && obj.IsStateOK() {
+			checkOK, err = true, nil
+
+			// NOTE: technically this block is wrong because we don't know
+			// if the resource implements refresh! If it doesn't, we could
+			// skip this, but it doesn't make a big difference under noop!
+		} else if noop && refresh { // had a refresh to do w/ noop!
+			checkOK, err = false, nil // therefore the state is wrong
+
+			// run the CheckApply!
+		} else {
+			// if this fails, don't UpdateTimestamp()
+			checkOK, err = obj.CheckApply(!noop)
+		}
+
+		if checkOK && err != nil { // should never return this way
+			log.Fatalf("%s[%s]: CheckApply(): %t, %+v", obj.Kind(), obj.GetName(), checkOK, err)
+		}
+		if g.Flags.Debug {
+			log.Printf("%s[%s]: CheckApply(): %t, %v", obj.Kind(), obj.GetName(), checkOK, err)
+		}
+
+		// if CheckApply ran without noop and without error, state should be good
+		if !noop && err == nil { // aka !noop || checkOK
+			obj.StateOK(true) // reset
+			if refresh {
+				g.SetUpstreamRefresh(v, false) // refresh happened, clear the request
+				obj.SetRefresh(false)
+			}
+		}
+
+		if !checkOK { // if state *was* not ok, we had to have apply'ed
+			if err != nil { // error during check or apply
+				ok = false
+			} else {
+				applied = true
+			}
+		}
+
+		// when noop is true we always want to update timestamp
+		if noop && err == nil {
+			ok = true
+		}
+
+		if ok {
+			// did we actually do work?
+			activity := applied
+			if noop {
+				activity = false // no we didn't do work...
+			}
+
+			if activity { // add refresh flag to downstream edges...
+				g.SetDownstreamRefresh(v, true)
+			}
+
+			// update this timestamp *before* we poke or the poked
+			// nodes might fail due to having a too old timestamp!
+			v.UpdateTimestamp()                    // this was touched...
+			obj.SetState(resources.ResStatePoking) // can't cancel parent poke
+			if err := g.Poke(v, activity); err != nil {
+				return errwrap.Wrapf(err, "the Poke() failed")
+			}
+		}
+		// poke at our pre-req's instead since they need to refresh/run...
+		return errwrap.Wrapf(err, "could not Process() successfully")
+	}
+	// else... only poke at the pre-req's that need to run
+	go g.BackPoke(v)
+	return nil
+}
+
+// SentinelErr is a sentinal as an error type that wraps an arbitrary error.
+type SentinelErr struct {
+	err error
+}
+
+// Error is the required method to fulfill the error type.
+func (obj *SentinelErr) Error() string {
+	return obj.err.Error()
+}
+
+// Worker is the common run frontend of the vertex. It handles all of the retry
+// and retry delay common code, and ultimately returns the final status of this
+// vertex execution.
+func (g *Graph) Worker(v *Vertex) error {
+	// listen for chan events from Watch() and run
+	// the Process() function when they're received
+	// this avoids us having to pass the data into
+	// the Watch() function about which graph it is
+	// running on, which isolates things nicely...
+	obj := v.Res
+	// TODO: is there a better system for the `Watching` flag?
+	obj.SetWatching(true)
+	defer obj.SetWatching(false)
+	processChan := make(chan event.Event)
+	go func() {
+		running := false
+		var timer = time.NewTimer(time.Duration(math.MaxInt64)) // longest duration
+		if !timer.Stop() {
+			<-timer.C // unnecessary, shouldn't happen
+		}
+		var delay = time.Duration(v.Meta().Delay) * time.Millisecond
+		var retry = v.Meta().Retry // number of tries left, -1 for infinite
+		var saved event.Event
+	Loop:
+		for {
+			// this has to be synchronous, because otherwise the Res
+			// event loop will keep running and change state,
+			// causing the converged timeout to fire!
+			select {
+			case event, ok := <-processChan: // must use like this
+				if running && ok {
+					// we got an event that wasn't a close,
+					// while we were waiting for the timer!
+					// if this happens, it might be a bug:(
+					log.Fatalf("%s[%s]: Worker: Unexpected event: %+v", v.Kind(), v.GetName(), event)
+				}
+				if !ok { // processChan closed, let's exit
+					break Loop // no event, so no ack!
+				}
+
+				// the above mentioned synchronous part, is the
+				// running of this function, paired with an ack.
+				if e := g.Process(v); e != nil {
+					saved = event
+					log.Printf("%s[%s]: CheckApply errored: %v", v.Kind(), v.GetName(), e)
+					if retry == 0 {
+						// wrap the error in the sentinel
+						event.ACKNACK(&SentinelErr{e}) // fail the Watch()
+						break Loop
+					}
+					if retry > 0 { // don't decrement the -1
+						retry--
+					}
+					log.Printf("%s[%s]: CheckApply: Retrying after %.4f seconds (%d left)", v.Kind(), v.GetName(), delay.Seconds(), retry)
+					// start the timer...
+					timer.Reset(delay)
+					running = true
+					continue
+				}
+				retry = v.Meta().Retry // reset on success
+				event.ACK()            // sync
+
+			case <-timer.C:
+				if !timer.Stop() {
+					//<-timer.C // blocks, docs are wrong!
+				}
+				running = false
+				log.Printf("%s[%s]: CheckApply delay expired!", v.Kind(), v.GetName())
+				// re-send this failed event, to trigger a CheckApply()
+				go func() { processChan <- saved }()
+				// TODO: should we send a fake event instead?
+				//saved = nil
+			}
+		}
+	}()
+	var err error // propagate the error up (this is a permanent BAD error!)
+	// the watch delay runs inside of the Watch resource loop, so that it
+	// can still process signals and exit if needed. It shouldn't run any
+	// resource specific code since this is supposed to be a retry delay.
+	// NOTE: we're using the same retry and delay metaparams that CheckApply
+	// uses. This is for practicality. We can separate them later if needed!
+	var watchDelay time.Duration
+	var watchRetry = v.Meta().Retry // number of tries left, -1 for infinite
+	// watch blocks until it ends, & errors to retry
+	for {
+		// TODO: do we have to stop the converged-timeout when in this block (perhaps we're in the delay block!)
+		// TODO: should we setup/manage some of the converged timeout stuff in here anyways?
+
+		// if a retry-delay was requested, wait, but don't block our events!
+		if watchDelay > 0 {
+			//var pendingSendEvent bool
+			timer := time.NewTimer(watchDelay)
+		Loop:
+			for {
+				select {
+				case <-timer.C: // the wait is over
+					break Loop // critical
+
+				// TODO: resources could have a separate exit channel to avoid this complexity!?
+				case event := <-obj.Events():
+					// NOTE: this code should match the similar Res code!
+					//cuid.SetConverged(false) // TODO: ?
+					if exit, send := obj.ReadEvent(&event); exit {
+						return nil // exit
+					} else if send {
+						// if we dive down this rabbit hole, our
+						// timer.C won't get seen until we get out!
+						// in this situation, the Watch() is blocked
+						// from performing until CheckApply returns
+						// successfully, or errors out. This isn't
+						// so bad, but we should document it. Is it
+						// possible that some resource *needs* Watch
+						// to run to be able to execute a CheckApply?
+						// That situation shouldn't be common, and
+						// should probably not be allowed. Can we
+						// avoid it though?
+						//if exit, err := doSend(); exit || err != nil {
+						//	return err // we exit or bubble up a NACK...
+						//}
+						// Instead of doing the above, we can
+						// add events to a pending list, and
+						// when we finish the delay, we can run
+						// them.
+						//pendingSendEvent = true // all events are identical for now...
+					}
+				}
+			}
+			timer.Stop() // it's nice to cleanup
+			log.Printf("%s[%s]: Watch delay expired!", v.Kind(), v.GetName())
+			// NOTE: we can avoid the send if running Watch guarantees
+			// one CheckApply event on startup!
+			//if pendingSendEvent { // TODO: should this become a list in the future?
+			//	if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
+			//		return err // we exit or bubble up a NACK...
+			//	}
+			//}
+		}
+
+		// TODO: reset the watch retry count after some amount of success
+		v.Res.RegisterConverger()
+		e := v.Res.Watch(processChan)
+		v.Res.UnregisterConverger()
+		if e == nil { // exit signal
+			err = nil // clean exit
+			break
+		}
+		if sentinelErr, ok := e.(*SentinelErr); ok { // unwrap the sentinel
+			err = sentinelErr.err
+			break // sentinel means, perma-exit
+		}
+		log.Printf("%s[%s]: Watch errored: %v", v.Kind(), v.GetName(), e)
+		if watchRetry == 0 {
+			err = fmt.Errorf("Permanent watch error: %v", e)
+			break
+		}
+		if watchRetry > 0 { // don't decrement the -1
+			watchRetry--
+		}
+		watchDelay = time.Duration(v.Meta().Delay) * time.Millisecond
+		log.Printf("%s[%s]: Watch: Retrying after %.4f seconds (%d left)", v.Kind(), v.GetName(), watchDelay.Seconds(), watchRetry)
+		// We need to trigger a CheckApply after Watch restarts, so that
+		// we catch any lost events that happened while down. We do this
+		// by getting the Watch resource to send one event once it's up!
+		//v.SendEvent(eventPoke, false, false)
+	}
+	close(processChan)
+	return err
+}
+
+// Start is a main kick to start the graph. It goes through in reverse topological
+// sort order so that events can't hit un-started vertices.
+func (g *Graph) Start(first bool) { // start or continue
+	log.Printf("State: %v -> %v", g.setState(graphStateStarting), g.getState())
+	defer log.Printf("State: %v -> %v", g.setState(graphStateStarted), g.getState())
+	var wg sync.WaitGroup
+	t, _ := g.TopologicalSort()
+	// TODO: only calculate indegree if `first` is true to save resources
+	indegree := g.InDegree() // compute all of the indegree's
+	for _, v := range Reverse(t) {
+		// selective poke: here we reduce the number of initial pokes
+		// to the minimum required to activate every vertex in the
+		// graph, either by direct action, or by getting poked by a
+		// vertex that was previously activated. if we poke each vertex
+		// that has no incoming edges, then we can be sure to reach the
+		// whole graph. Please note: this may mask certain optimization
+		// failures, such as any poke limiting code in Poke() or
+		// BackPoke(). You might want to disable this selective start
+		// when experimenting with and testing those elements.
+		// if we are unpausing (since it's not the first run of this
+		// function) we need to poke to *unpause* every graph vertex,
+		// and not just selectively the subset with no indegree.
+		if (!first) || indegree[v] == 0 {
+			v.Res.Starter(true) // let the startup code know to poke
+		}
+
+		if !v.Res.IsWatching() { // if Watch() is not running...
+			g.wg.Add(1)
+			// must pass in value to avoid races...
+			// see: https://ttboj.wordpress.com/2015/07/27/golang-parallelism-issues-causing-too-many-open-files-error/
+			go func(vv *Vertex) {
+				defer g.wg.Done()
+				// TODO: if a sufficient number of workers error,
+				// should something be done? Will these restart
+				// after perma-failure if we have a graph change?
+				if err := g.Worker(vv); err != nil { // contains the Watch and CheckApply loops
+					log.Printf("%s[%s]: Exited with failure: %v", vv.Kind(), vv.GetName(), err)
+					return
+				}
+				log.Printf("%s[%s]: Exited", vv.Kind(), vv.GetName())
+			}(v)
+		}
+
+		// let the vertices run their startup code in parallel
+		wg.Add(1)
+		go func(vv *Vertex) {
+			defer wg.Done()
+			vv.Res.Started() // block until started
+		}(v)
+
+		if !first { // unpause!
+			v.Res.SendEvent(event.EventStart, true, false) // sync!
+		}
+	}
+
+	wg.Wait() // wait for everyone
+}
+
+// Pause sends pause events to the graph in a topological sort order.
+func (g *Graph) Pause() {
+	log.Printf("State: %v -> %v", g.setState(graphStatePausing), g.getState())
+	defer log.Printf("State: %v -> %v", g.setState(graphStatePaused), g.getState())
+	t, _ := g.TopologicalSort()
+	for _, v := range t { // squeeze out the events...
+		v.SendEvent(event.EventPause, true, false)
+	}
+}
+
+// Exit sends exit events to the graph in a topological sort order.
+func (g *Graph) Exit() {
+	if g == nil {
+		return
+	} // empty graph that wasn't populated yet
+	t, _ := g.TopologicalSort()
+	for _, v := range t { // squeeze out the events...
+		// turn off the taps...
+		// XXX: consider instead doing this by closing the Res.events channel instead?
+		// XXX: do this by sending an exit signal, and then returning
+		// when we hit the 'default' in the select statement!
+		// XXX: we can do this to quiesce, but it's not necessary now
+
+		v.SendEvent(event.EventExit, true, false)
+	}
+	g.wg.Wait() // for now, this doesn't need to be a separate Wait() method
+}

pgraph/autoedge.go

@@ -22,33 +22,32 @@ import (
 	"fmt"
 	"log"
 
-	"github.com/purpleidea/mgmt/global"
 	"github.com/purpleidea/mgmt/resources"
 )
 
-// add edges to the vertex in a graph based on if it matches a uuid list
+// add edges to the vertex in a graph based on if it matches a uid list
-func (g *Graph) addEdgesByMatchingUUIDS(v *Vertex, uuids []resources.ResUUID) []bool {
+func (g *Graph) addEdgesByMatchingUIDS(v *Vertex, uids []resources.ResUID) []bool {
 	// search for edges and see what matches!
 	var result []bool
 
-	// loop through each uuid, and see if it matches any vertex
+	// loop through each uid, and see if it matches any vertex
-	for _, uuid := range uuids {
+	for _, uid := range uids {
 		var found = false
-		// uuid is a ResUUID object
+		// uid is a ResUID object
 		for _, vv := range g.GetVertices() { // search
 			if v == vv { // skip self
 				continue
 			}
-			if global.DEBUG {
+			if g.Flags.Debug {
-				log.Printf("Compile: AutoEdge: Match: %v[%v] with UUID: %v[%v]", vv.Kind(), vv.GetName(), uuid.Kind(), uuid.GetName())
+				log.Printf("Compile: AutoEdge: Match: %v[%v] with UID: %v[%v]", vv.Kind(), vv.GetName(), uid.Kind(), uid.GetName())
 			}
-			// we must match to an effective UUID for the resource,
+			// we must match to an effective UID for the resource,
 			// that is to say, the name value of a res is a helpful
 			// handle, but it is not necessarily a unique identity!
-			// remember, resources can return multiple UUID's each!
+			// remember, resources can return multiple UID's each!
-			if resources.UUIDExistsInUUIDs(uuid, vv.GetUUIDs()) {
+			if resources.UIDExistsInUIDs(uid, vv.GetUIDs()) {
 				// add edge from: vv -> v
-				if uuid.Reversed() {
+				if uid.Reversed() {
 					txt := fmt.Sprintf("AutoEdge: %v[%v] -> %v[%v]", vv.Kind(), vv.GetName(), v.Kind(), v.GetName())
 					log.Printf("Compile: Adding %v", txt)
 					g.AddEdge(vv, v, NewEdge(txt))
@@ -79,21 +78,21 @@ func (g *Graph) AutoEdges() {
 			continue // next vertex
 		}
 
-		for { // while the autoEdgeObj has more uuids to add...
+		for { // while the autoEdgeObj has more uids to add...
-			uuids := autoEdgeObj.Next() // get some!
+			uids := autoEdgeObj.Next() // get some!
-			if uuids == nil {
+			if uids == nil {
 				log.Printf("%v[%v]: Config: The auto edge list is empty!", v.Kind(), v.GetName())
 				break // inner loop
 			}
-			if global.DEBUG {
+			if g.Flags.Debug {
-				log.Println("Compile: AutoEdge: UUIDS:")
+				log.Println("Compile: AutoEdge: UIDS:")
-				for i, u := range uuids {
+				for i, u := range uids {
-					log.Printf("Compile: AutoEdge: UUID%d: %v", i, u)
+					log.Printf("Compile: AutoEdge: UID%d: %v", i, u)
 				}
 			}
 
 			// match and add edges
-			result := g.addEdgesByMatchingUUIDS(v, uuids)
+			result := g.addEdgesByMatchingUIDS(v, uids)
 
 			// report back, and find out if we should continue
 			if !autoEdgeObj.Test(result) {

pgraph/autogroup.go

@@ -21,7 +21,7 @@ import (
 	"fmt"
 	"log"
 
-	"github.com/purpleidea/mgmt/global"
+	errwrap "github.com/pkg/errors"
 )
 
 // AutoGrouper is the required interface to implement for an autogroup algorithm
@@ -219,7 +219,7 @@ func (g *Graph) VertexMerge(v1, v2 *Vertex, vertexMergeFn func(*Vertex, *Vertex)
 	}
 
 	// 2) edges that point towards v2 from X now point to v1 from X (no dupes)
-	for _, x := range g.IncomingGraphEdges(v2) { // all to vertex v (??? -> v)
+	for _, x := range g.IncomingGraphVertices(v2) { // all to vertex v (??? -> v)
 		e := g.Adjacency[x][v2] // previous edge
 		r := g.Reachability(x, v1)
 		// merge e with ex := g.Adjacency[x][v1] if it exists!
@@ -246,7 +246,7 @@ func (g *Graph) VertexMerge(v1, v2 *Vertex, vertexMergeFn func(*Vertex, *Vertex)
 	}
 
 	// 3) edges that point from v2 to X now point from v1 to X (no dupes)
-	for _, x := range g.OutgoingGraphEdges(v2) { // all from vertex v (v -> ???)
+	for _, x := range g.OutgoingGraphVertices(v2) { // all from vertex v (v -> ???)
 		e := g.Adjacency[v2][x] // previous edge
 		r := g.Reachability(v1, x)
 		// merge e with ex := g.Adjacency[v1][x] if it exists!
@@ -277,14 +277,14 @@ func (g *Graph) VertexMerge(v1, v2 *Vertex, vertexMergeFn func(*Vertex, *Vertex)
 		if v, err := vertexMergeFn(v1, v2); err != nil {
 			return err
 		} else if v != nil { // replace v1 with the "merged" version...
-			v1 = v // XXX: will this replace v1 the way we want?
+			*v1 = *v // TODO: is this safe? (replacing mutexes is undefined!)
 		}
 	}
 	g.DeleteVertex(v2) // remove grouped vertex
 
 	// 5) creation of a cyclic graph should throw an error
-	if _, dag := g.TopologicalSort(); !dag { // am i a dag or not?
+	if _, err := g.TopologicalSort(); err != nil { // am i a dag or not?
-		return fmt.Errorf("Graph is not a dag!")
+		return errwrap.Wrapf(err, "TopologicalSort failed") // not a dag
 	}
 	return nil // success
 }
@@ -310,7 +310,7 @@ func (g *Graph) autoGroup(ag AutoGrouper) chan string {
 			wStr := fmt.Sprintf("%s", w)
 
 			if err := ag.vertexCmp(v, w); err != nil { // cmp ?
-				if global.DEBUG {
+				if g.Flags.Debug {
 					strch <- fmt.Sprintf("Compile: Grouping: !GroupCmp for: %s into %s", wStr, vStr)
 				}
 

pgraph/graphviz.go

@@ -0,0 +1,110 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgraph
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"strconv"
+	"syscall"
+)
+
+// Graphviz outputs the graph in graphviz format.
+// https://en.wikipedia.org/wiki/DOT_%28graph_description_language%29
+func (g *Graph) Graphviz() (out string) {
+	//digraph g {
+	//	label="hello world";
+	//	node [shape=box];
+	//	A [label="A"];
+	//	B [label="B"];
+	//	C [label="C"];
+	//	D [label="D"];
+	//	E [label="E"];
+	//	A -> B [label=f];
+	//	B -> C [label=g];
+	//	D -> E [label=h];
+	//}
+	out += fmt.Sprintf("digraph %s {\n", g.GetName())
+	out += fmt.Sprintf("\tlabel=\"%s\";\n", g.GetName())
+	//out += "\tnode [shape=box];\n"
+	str := ""
+	for i := range g.Adjacency { // reverse paths
+		out += fmt.Sprintf("\t%s [label=\"%s[%s]\"];\n", i.GetName(), i.Kind(), i.GetName())
+		for j := range g.Adjacency[i] {
+			k := g.Adjacency[i][j]
+			// use str for clearer output ordering
+			str += fmt.Sprintf("\t%s -> %s [label=%s];\n", i.GetName(), j.GetName(), k.Name)
+		}
+	}
+	out += str
+	out += "}\n"
+	return
+}
+
+// ExecGraphviz writes out the graphviz data and runs the correct graphviz
+// filter command.
+func (g *Graph) ExecGraphviz(program, filename string) error {
+
+	switch program {
+	case "dot", "neato", "twopi", "circo", "fdp":
+	default:
+		return fmt.Errorf("Invalid graphviz program selected!")
+	}
+
+	if filename == "" {
+		return fmt.Errorf("No filename given!")
+	}
+
+	// run as a normal user if possible when run with sudo
+	uid, err1 := strconv.Atoi(os.Getenv("SUDO_UID"))
+	gid, err2 := strconv.Atoi(os.Getenv("SUDO_GID"))
+
+	err := ioutil.WriteFile(filename, []byte(g.Graphviz()), 0644)
+	if err != nil {
+		return fmt.Errorf("Error writing to filename!")
+	}
+
+	if err1 == nil && err2 == nil {
+		if err := os.Chown(filename, uid, gid); err != nil {
+			return fmt.Errorf("Error changing file owner!")
+		}
+	}
+
+	path, err := exec.LookPath(program)
+	if err != nil {
+		return fmt.Errorf("Graphviz is missing!")
+	}
+
+	out := fmt.Sprintf("%s.png", filename)
+	cmd := exec.Command(path, "-Tpng", fmt.Sprintf("-o%s", out), filename)
+
+	if err1 == nil && err2 == nil {
+		cmd.SysProcAttr = &syscall.SysProcAttr{}
+		cmd.SysProcAttr.Credential = &syscall.Credential{
+			Uid: uint32(uid),
+			Gid: uint32(gid),
+		}
+	}
+	_, err = cmd.Output()
+	if err != nil {
+		return fmt.Errorf("Error writing to image!")
+	}
+	return nil
+}

pgraph/pgraph.go

@@ -19,23 +19,14 @@
 package pgraph
 
 import (
-	"errors"
 	"fmt"
-	"io/ioutil"
-	"log"
-	"math"
-	"os"
-	"os/exec"
 	"sort"
-	"strconv"
 	"sync"
-	"syscall"
-	"time"
 
-	"github.com/purpleidea/mgmt/converger"
 	"github.com/purpleidea/mgmt/event"
-	"github.com/purpleidea/mgmt/global"
 	"github.com/purpleidea/mgmt/resources"
+
+	errwrap "github.com/pkg/errors"
 )
 
 //go:generate stringer -type=graphState -output=graphstate_stringer.go
@@ -49,6 +40,11 @@ const (
 	graphStatePaused
 )
 
+// Flags contains specific constants used by the graph.
+type Flags struct {
+	Debug bool
+}
+
 // Graph is the graph structure in this library.
 // The graph abstract data type (ADT) is defined as follows:
 // * the directed graph arrows point from left to right ( -> )
@@ -58,8 +54,10 @@ const (
 type Graph struct {
 	Name      string
 	Adjacency map[*Vertex]map[*Vertex]*Edge // *Vertex -> *Vertex (edge)
+	Flags     Flags
 	state     graphState
-	mutex     sync.Mutex // used when modifying graph State variable
+	mutex     *sync.Mutex // used when modifying graph State variable
+	wg        *sync.WaitGroup
 }
 
 // Vertex is the primary vertex struct in this library.
@@ -71,6 +69,9 @@ type Vertex struct {
 // Edge is the primary edge struct in this library.
 type Edge struct {
 	Name   string
+	Notify bool // should we send a refresh notification along this edge?
+
+	refresh bool // is there a notify pending for the dest vertex ?
 }
 
 // NewGraph builds a new graph.
@@ -79,6 +80,9 @@ func NewGraph(name string) *Graph {
 		Name:      name,
 		Adjacency: make(map[*Vertex]map[*Vertex]*Edge),
 		state:     graphStateNil,
+		// ptr b/c: Mutex/WaitGroup must not be copied after first use
+		mutex: &sync.Mutex{},
+		wg:    &sync.WaitGroup{},
 	}
 }
 
@@ -96,12 +100,25 @@ func NewEdge(name string) *Edge {
 	}
 }
 
+// Refresh returns the pending refresh status of this edge.
+func (obj *Edge) Refresh() bool {
+	return obj.refresh
+}
+
+// SetRefresh sets the pending refresh status of this edge.
+func (obj *Edge) SetRefresh(b bool) {
+	obj.refresh = b
+}
+
 // Copy makes a copy of the graph struct
 func (g *Graph) Copy() *Graph {
 	newGraph := &Graph{
 		Name:      g.Name,
 		Adjacency: make(map[*Vertex]map[*Vertex]*Edge, len(g.Adjacency)),
+		Flags:     g.Flags,
 		state:     g.state,
+		mutex:     g.mutex,
+		wg:        g.wg,
 	}
 	for k, v := range g.Adjacency {
 		newGraph.Adjacency[k] = v // copy
@@ -163,6 +180,18 @@ func (g *Graph) AddEdge(v1, v2 *Vertex, e *Edge) {
 	g.Adjacency[v1][v2] = e
 }
 
+// DeleteEdge deletes a particular edge from the graph.
+// FIXME: add test cases
+func (g *Graph) DeleteEdge(e *Edge) {
+	for v1 := range g.Adjacency {
+		for v2, edge := range g.Adjacency[v1] {
+			if e == edge {
+				delete(g.Adjacency[v1], v2)
+			}
+		}
+	}
+}
+
 // GetVertexMatch searches for an equivalent resource in the graph and returns
 // the vertex it is found in, or nil if not found.
 func (g *Graph) GetVertexMatch(obj resources.Res) *Vertex {
@@ -246,92 +275,9 @@ func (v *Vertex) String() string {
 	return fmt.Sprintf("%s[%s]", v.Res.Kind(), v.Res.GetName())
 }
 
-// Graphviz outputs the graph in graphviz format.
+// IncomingGraphVertices returns an array (slice) of all directed vertices to
-// https://en.wikipedia.org/wiki/DOT_%28graph_description_language%29
-func (g *Graph) Graphviz() (out string) {
-	//digraph g {
-	//	label="hello world";
-	//	node [shape=box];
-	//	A [label="A"];
-	//	B [label="B"];
-	//	C [label="C"];
-	//	D [label="D"];
-	//	E [label="E"];
-	//	A -> B [label=f];
-	//	B -> C [label=g];
-	//	D -> E [label=h];
-	//}
-	out += fmt.Sprintf("digraph %v {\n", g.GetName())
-	out += fmt.Sprintf("\tlabel=\"%v\";\n", g.GetName())
-	//out += "\tnode [shape=box];\n"
-	str := ""
-	for i := range g.Adjacency { // reverse paths
-		out += fmt.Sprintf("\t%v [label=\"%v[%v]\"];\n", i.GetName(), i.Kind(), i.GetName())
-		for j := range g.Adjacency[i] {
-			k := g.Adjacency[i][j]
-			// use str for clearer output ordering
-			str += fmt.Sprintf("\t%v -> %v [label=%v];\n", i.GetName(), j.GetName(), k.Name)
-		}
-	}
-	out += str
-	out += "}\n"
-	return
-}
-
-// ExecGraphviz writes out the graphviz data and runs the correct graphviz
-// filter command.
-func (g *Graph) ExecGraphviz(program, filename string) error {
-
-	switch program {
-	case "dot", "neato", "twopi", "circo", "fdp":
-	default:
-		return errors.New("Invalid graphviz program selected!")
-	}
-
-	if filename == "" {
-		return errors.New("No filename given!")
-	}
-
-	// run as a normal user if possible when run with sudo
-	uid, err1 := strconv.Atoi(os.Getenv("SUDO_UID"))
-	gid, err2 := strconv.Atoi(os.Getenv("SUDO_GID"))
-
-	err := ioutil.WriteFile(filename, []byte(g.Graphviz()), 0644)
-	if err != nil {
-		return errors.New("Error writing to filename!")
-	}
-
-	if err1 == nil && err2 == nil {
-		if err := os.Chown(filename, uid, gid); err != nil {
-			return errors.New("Error changing file owner!")
-		}
-	}
-
-	path, err := exec.LookPath(program)
-	if err != nil {
-		return errors.New("Graphviz is missing!")
-	}
-
-	out := fmt.Sprintf("%v.png", filename)
-	cmd := exec.Command(path, "-Tpng", fmt.Sprintf("-o%v", out), filename)
-
-	if err1 == nil && err2 == nil {
-		cmd.SysProcAttr = &syscall.SysProcAttr{}
-		cmd.SysProcAttr.Credential = &syscall.Credential{
-			Uid: uint32(uid),
-			Gid: uint32(gid),
-		}
-	}
-	_, err = cmd.Output()
-	if err != nil {
-		return errors.New("Error writing to image!")
-	}
-	return nil
-}
-
-// IncomingGraphEdges returns an array (slice) of all directed vertices to
 // vertex v (??? -> v). OKTimestamp should probably use this.
-func (g *Graph) IncomingGraphEdges(v *Vertex) []*Vertex {
+func (g *Graph) IncomingGraphVertices(v *Vertex) []*Vertex {
 	// TODO: we might be able to implement this differently by reversing
 	// the Adjacency graph and then looping through it again...
 	var s []*Vertex
@@ -345,9 +291,9 @@ func (g *Graph) IncomingGraphEdges(v *Vertex) []*Vertex {
 	return s
 }
 
-// OutgoingGraphEdges returns an array (slice) of all vertices that vertex v
+// OutgoingGraphVertices returns an array (slice) of all vertices that vertex v
 // points to (v -> ???). Poke should probably use this.
-func (g *Graph) OutgoingGraphEdges(v *Vertex) []*Vertex {
+func (g *Graph) OutgoingGraphVertices(v *Vertex) []*Vertex {
 	var s []*Vertex
 	for k := range g.Adjacency[v] { // forward paths
 		s = append(s, k)
@@ -355,15 +301,46 @@ func (g *Graph) OutgoingGraphEdges(v *Vertex) []*Vertex {
 	return s
 }
 
-// GraphEdges returns an array (slice) of all vertices that connect to vertex v.
+// GraphVertices returns an array (slice) of all vertices that connect to vertex v.
-// This is the union of IncomingGraphEdges and OutgoingGraphEdges.
+// This is the union of IncomingGraphVertices and OutgoingGraphVertices.
-func (g *Graph) GraphEdges(v *Vertex) []*Vertex {
+func (g *Graph) GraphVertices(v *Vertex) []*Vertex {
 	var s []*Vertex
-	s = append(s, g.IncomingGraphEdges(v)...)
+	s = append(s, g.IncomingGraphVertices(v)...)
-	s = append(s, g.OutgoingGraphEdges(v)...)
+	s = append(s, g.OutgoingGraphVertices(v)...)
 	return s
 }
 
+// IncomingGraphEdges returns all of the edges that point to vertex v (??? -> v).
+func (g *Graph) IncomingGraphEdges(v *Vertex) []*Edge {
+	var edges []*Edge
+	for v1 := range g.Adjacency { // reverse paths
+		for v2, e := range g.Adjacency[v1] {
+			if v2 == v {
+				edges = append(edges, e)
+			}
+		}
+	}
+	return edges
+}
+
+// OutgoingGraphEdges returns all of the edges that point from vertex v (v -> ???).
+func (g *Graph) OutgoingGraphEdges(v *Vertex) []*Edge {
+	var edges []*Edge
+	for _, e := range g.Adjacency[v] { // forward paths
+		edges = append(edges, e)
+	}
+	return edges
+}
+
+// GraphEdges returns an array (slice) of all edges that connect to vertex v.
+// This is the union of IncomingGraphEdges and OutgoingGraphEdges.
+func (g *Graph) GraphEdges(v *Vertex) []*Edge {
+	var edges []*Edge
+	edges = append(edges, g.IncomingGraphEdges(v)...)
+	edges = append(edges, g.OutgoingGraphEdges(v)...)
+	return edges
+}
+
 // DFS returns a depth first search for the graph, starting at the input vertex.
 func (g *Graph) DFS(start *Vertex) []*Vertex {
 	var d []*Vertex // discovered
@@ -379,7 +356,7 @@ func (g *Graph) DFS(start *Vertex) []*Vertex {
 		if !VertexContains(v, d) { // if not discovered
 			d = append(d, v) // label as discovered
 
-			for _, w := range g.GraphEdges(v) {
+			for _, w := range g.GraphVertices(v) {
 				s = append(s, w)
 			}
 		}
@@ -392,7 +369,7 @@ func (g *Graph) FilterGraph(name string, vertices []*Vertex) *Graph {
 	newgraph := NewGraph(name)
 	for k1, x := range g.Adjacency {
 		for k2, e := range x {
-			//log.Printf("Filter: %v -> %v # %v", k1.Name, k2.Name, e.Name)
+			//log.Printf("Filter: %s -> %s # %s", k1.Name, k2.Name, e.Name)
 			if VertexContains(k1, vertices) || VertexContains(k2, vertices) {
 				newgraph.AddEdge(k1, k2, e)
 			}
@@ -468,7 +445,7 @@ func (g *Graph) OutDegree() map[*Vertex]int {
 // TopologicalSort returns the sort of graph vertices in that order.
 // based on descriptions and code from wikipedia and rosetta code
 // TODO: add memoization, and cache invalidation to speed this up :)
-func (g *Graph) TopologicalSort() (result []*Vertex, ok bool) { // kahn's algorithm
+func (g *Graph) TopologicalSort() ([]*Vertex, error) { // kahn's algorithm
 	var L []*Vertex                    // empty list that will contain the sorted elements
 	var S []*Vertex                    // set of all nodes with no incoming edges
 	remaining := make(map[*Vertex]int) // amount of edges remaining
@@ -505,13 +482,13 @@ func (g *Graph) TopologicalSort() (result []*Vertex, ok bool) { // kahn's algori
 		if in > 0 {
 			for n := range g.Adjacency[c] {
 				if remaining[n] > 0 {
-					return nil, false // not a dag!
+					return nil, fmt.Errorf("Not a dag!")
 				}
 			}
 		}
 	}
 
-	return L, true
+	return L, nil
 }
 
 // Reachability finds the shortest path in a DAG from a to b, and returns the
@@ -526,7 +503,7 @@ func (g *Graph) Reachability(a, b *Vertex) []*Vertex {
 	if a == nil || b == nil {
 		return nil
 	}
-	vertices := g.OutgoingGraphEdges(a) // what points away from a ?
+	vertices := g.OutgoingGraphVertices(a) // what points away from a ?
 	if len(vertices) == 0 {
 		return []*Vertex{} // nope
 	}
@@ -554,395 +531,98 @@ func (g *Graph) Reachability(a, b *Vertex) []*Vertex {
 	return result
 }
 
-// GetTimestamp returns the timestamp of a vertex
+// GraphSync updates the oldGraph so that it matches the newGraph receiver. It
-func (v *Vertex) GetTimestamp() int64 {
+// leaves identical elements alone so that they don't need to be refreshed.
-	return v.timestamp
+// FIXME: add test cases
+func (g *Graph) GraphSync(oldGraph *Graph) (*Graph, error) {
+
+	if oldGraph == nil {
+		oldGraph = NewGraph(g.GetName()) // copy over the name
+	}
+	oldGraph.SetName(g.GetName()) // overwrite the name
+
+	var lookup = make(map[*Vertex]*Vertex)
+	var vertexKeep []*Vertex // list of vertices which are the same in new graph
+	var edgeKeep []*Edge     // list of vertices which are the same in new graph
+
+	for v := range g.Adjacency { // loop through the vertices (resources)
+		res := v.Res // resource
+
+		vertex := oldGraph.GetVertexMatch(res)
+		if vertex == nil { // no match found
+			if err := res.Init(); err != nil {
+				return nil, errwrap.Wrapf(err, "could not Init() resource")
+			}
+			vertex = NewVertex(res)
+			oldGraph.AddVertex(vertex) // call standalone in case not part of an edge
+		}
+		lookup[v] = vertex                      // used for constructing edges
+		vertexKeep = append(vertexKeep, vertex) // append
 	}
 
-// UpdateTimestamp updates the timestamp on a vertex and returns the new value
+	// get rid of any vertices we shouldn't keep (that aren't in new graph)
-func (v *Vertex) UpdateTimestamp() int64 {
+	for v := range oldGraph.Adjacency {
-	v.timestamp = time.Now().UnixNano() // update
+		if !VertexContains(v, vertexKeep) {
-	return v.timestamp
+			// wait for exit before starting new graph!
-}
-
-// OKTimestamp returns true if this element can run right now?
-func (g *Graph) OKTimestamp(v *Vertex) bool {
-	// these are all the vertices pointing TO v, eg: ??? -> v
-	for _, n := range g.IncomingGraphEdges(v) {
-		// if the vertex has a greater timestamp than any pre-req (n)
-		// then we can't run right now...
-		// if they're equal (eg: on init of 0) then we also can't run
-		// b/c we should let our pre-req's go first...
-		x, y := v.GetTimestamp(), n.GetTimestamp()
-		if global.DEBUG {
-			log.Printf("%v[%v]: OKTimestamp: (%v) >= %v[%v](%v): !%v", v.Kind(), v.GetName(), x, n.Kind(), n.GetName(), y, x >= y)
-		}
-		if x >= y {
-			return false
-		}
-	}
-	return true
-}
-
-// Poke notifies nodes after me in the dependency graph that they need refreshing...
-// NOTE: this assumes that this can never fail or need to be rescheduled
-func (g *Graph) Poke(v *Vertex, activity bool) {
-	// these are all the vertices pointing AWAY FROM v, eg: v -> ???
-	for _, n := range g.OutgoingGraphEdges(v) {
-		// XXX: if we're in state event and haven't been cancelled by
-		// apply, then we can cancel a poke to a child, right? XXX
-		// XXX: if n.Res.getState() != resources.ResStateEvent { // is this correct?
-		if true { // XXX
-			if global.DEBUG {
-				log.Printf("%v[%v]: Poke: %v[%v]", v.Kind(), v.GetName(), n.Kind(), n.GetName())
-			}
-			n.SendEvent(event.EventPoke, false, activity) // XXX: can this be switched to sync?
-		} else {
-			if global.DEBUG {
-				log.Printf("%v[%v]: Poke: %v[%v]: Skipped!", v.Kind(), v.GetName(), n.Kind(), n.GetName())
-			}
-		}
-	}
-}
-
-// BackPoke pokes the pre-requisites that are stale and need to run before I can run.
-func (g *Graph) BackPoke(v *Vertex) {
-	// these are all the vertices pointing TO v, eg: ??? -> v
-	for _, n := range g.IncomingGraphEdges(v) {
-		x, y, s := v.GetTimestamp(), n.GetTimestamp(), n.Res.GetState()
-		// if the parent timestamp needs poking AND it's not in state
-		// ResStateEvent, then poke it. If the parent is in ResStateEvent it
-		// means that an event is pending, so we'll be expecting a poke
-		// back soon, so we can safely discard the extra parent poke...
-		// TODO: implement a stateLT (less than) to tell if something
-		// happens earlier in the state cycle and that doesn't wrap nil
-		if x >= y && (s != resources.ResStateEvent && s != resources.ResStateCheckApply) {
-			if global.DEBUG {
-				log.Printf("%v[%v]: BackPoke: %v[%v]", v.Kind(), v.GetName(), n.Kind(), n.GetName())
-			}
-			n.SendEvent(event.EventBackPoke, false, false) // XXX: can this be switched to sync?
-		} else {
-			if global.DEBUG {
-				log.Printf("%v[%v]: BackPoke: %v[%v]: Skipped!", v.Kind(), v.GetName(), n.Kind(), n.GetName())
-			}
-		}
-	}
-}
-
-// Process is the primary function to execute for a particular vertex in the graph.
-func (g *Graph) Process(v *Vertex) error {
-	obj := v.Res
-	if global.DEBUG {
-		log.Printf("%v[%v]: Process()", obj.Kind(), obj.GetName())
-	}
-	obj.SetState(resources.ResStateEvent)
-	var ok = true
-	var apply = false // did we run an apply?
-	// is it okay to run dependency wise right now?
-	// if not, that's okay because when the dependency runs, it will poke
-	// us back and we will run if needed then!
-	if g.OKTimestamp(v) {
-		if global.DEBUG {
-			log.Printf("%v[%v]: OKTimestamp(%v)", obj.Kind(), obj.GetName(), v.GetTimestamp())
-		}
-
-		obj.SetState(resources.ResStateCheckApply)
-		// if this fails, don't UpdateTimestamp()
-		checkok, err := obj.CheckApply(!obj.Meta().Noop)
-		if checkok && err != nil { // should never return this way
-			log.Fatalf("%v[%v]: CheckApply(): %t, %+v", obj.Kind(), obj.GetName(), checkok, err)
-		}
-		if global.DEBUG {
-			log.Printf("%v[%v]: CheckApply(): %t, %v", obj.Kind(), obj.GetName(), checkok, err)
-		}
-
-		if !checkok { // if state *was* not ok, we had to have apply'ed
-			if err != nil { // error during check or apply
-				ok = false
-			} else {
-				apply = true
-			}
-		}
-
-		// when noop is true we always want to update timestamp
-		if obj.Meta().Noop && err == nil {
-			ok = true
-		}
-
-		if ok {
-			// update this timestamp *before* we poke or the poked
-			// nodes might fail due to having a too old timestamp!
-			v.UpdateTimestamp()                    // this was touched...
-			obj.SetState(resources.ResStatePoking) // can't cancel parent poke
-			g.Poke(v, apply)
-		}
-		// poke at our pre-req's instead since they need to refresh/run...
-		return err
-	}
-	// else... only poke at the pre-req's that need to run
-	go g.BackPoke(v)
-	return nil
-}
-
-// SentinelErr is a sentinal as an error type that wraps an arbitrary error.
-type SentinelErr struct {
-	err error
-}
-
-// Error is the required method to fulfill the error type.
-func (obj *SentinelErr) Error() string {
-	return obj.err.Error()
-}
-
-// Worker is the common run frontend of the vertex. It handles all of the retry
-// and retry delay common code, and ultimately returns the final status of this
-// vertex execution.
-func (g *Graph) Worker(v *Vertex) error {
-	// listen for chan events from Watch() and run
-	// the Process() function when they're received
-	// this avoids us having to pass the data into
-	// the Watch() function about which graph it is
-	// running on, which isolates things nicely...
-	obj := v.Res
-	chanProcess := make(chan event.Event)
-	go func() {
-		running := false
-		var timer = time.NewTimer(time.Duration(math.MaxInt64)) // longest duration
-		if !timer.Stop() {
-			<-timer.C // unnecessary, shouldn't happen
-		}
-		var delay = time.Duration(v.Meta().Delay) * time.Millisecond
-		var retry = v.Meta().Retry // number of tries left, -1 for infinite
-		var saved event.Event
-	Loop:
-		for {
-			// this has to be synchronous, because otherwise the Res
-			// event loop will keep running and change state,
-			// causing the converged timeout to fire!
-			select {
-			case event, ok := <-chanProcess: // must use like this
-				if running && ok {
-					// we got an event that wasn't a close,
-					// while we were waiting for the timer!
-					// if this happens, it might be a bug:(
-					log.Fatalf("%v[%v]: Worker: Unexpected event: %+v", v.Kind(), v.GetName(), event)
-				}
-				if !ok { // chanProcess closed, let's exit
-					break Loop // no event, so no ack!
-				}
-
-				// the above mentioned synchronous part, is the
-				// running of this function, paired with an ack.
-				if e := g.Process(v); e != nil {
-					saved = event
-					log.Printf("%v[%v]: CheckApply errored: %v", v.Kind(), v.GetName(), e)
-					if retry == 0 {
-						// wrap the error in the sentinel
-						event.ACKNACK(&SentinelErr{e}) // fail the Watch()
-						break Loop
-					}
-					if retry > 0 { // don't decrement the -1
-						retry--
-					}
-					log.Printf("%v[%v]: CheckApply: Retrying after %.4f seconds (%d left)", v.Kind(), v.GetName(), delay.Seconds(), retry)
-					// start the timer...
-					timer.Reset(delay)
-					running = true
-					continue
-				}
-				retry = v.Meta().Retry // reset on success
-				event.ACK()            // sync
-
-			case <-timer.C:
-				if !timer.Stop() {
-					//<-timer.C // blocks, docs are wrong!
-				}
-				running = false
-				log.Printf("%s[%s]: CheckApply delay expired!", v.Kind(), v.GetName())
-				// re-send this failed event, to trigger a CheckApply()
-				go func() { chanProcess <- saved }()
-				// TODO: should we send a fake event instead?
-				//saved = nil
-			}
-		}
-	}()
-	var err error // propagate the error up (this is a permanent BAD error!)
-	// the watch delay runs inside of the Watch resource loop, so that it
-	// can still process signals and exit if needed. It shouldn't run any
-	// resource specific code since this is supposed to be a retry delay.
-	// NOTE: we're using the same retry and delay metaparams that CheckApply
-	// uses. This is for practicality. We can separate them later if needed!
-	var watchDelay time.Duration
-	var watchRetry = v.Meta().Retry // number of tries left, -1 for infinite
-	// watch blocks until it ends, & errors to retry
-	for {
-		// TODO: do we have to stop the converged-timeout when in this block (perhaps we're in the delay block!)
-		// TODO: should we setup/manage some of the converged timeout stuff in here anyways?
-
-		// if a retry-delay was requested, wait, but don't block our events!
-		if watchDelay > 0 {
-			//var pendingSendEvent bool
-			timer := time.NewTimer(watchDelay)
-		Loop:
-			for {
-				select {
-				case <-timer.C: // the wait is over
-					break Loop // critical
-
-				// TODO: resources could have a separate exit channel to avoid this complexity!?
-				case event := <-obj.Events():
-					// NOTE: this code should match the similar Res code!
-					//cuuid.SetConverged(false) // TODO ?
-					if exit, send := obj.ReadEvent(&event); exit {
-						return nil // exit
-					} else if send {
-						// if we dive down this rabbit hole, our
-						// timer.C won't get seen until we get out!
-						// in this situation, the Watch() is blocked
-						// from performing until CheckApply returns
-						// successfully, or errors out. This isn't
-						// so bad, but we should document it. Is it
-						// possible that some resource *needs* Watch
-						// to run to be able to execute a CheckApply?
-						// That situation shouldn't be common, and
-						// should probably not be allowed. Can we
-						// avoid it though?
-						//if exit, err := doSend(); exit || err != nil {
-						//	return err // we exit or bubble up a NACK...
-						//}
-						// Instead of doing the above, we can
-						// add events to a pending list, and
-						// when we finish the delay, we can run
-						// them.
-						//pendingSendEvent = true // all events are identical for now...
-					}
-				}
-			}
-			timer.Stop() // it's nice to cleanup
-			log.Printf("%s[%s]: Watch delay expired!", v.Kind(), v.GetName())
-			// NOTE: we can avoid the send if running Watch guarantees
-			// one CheckApply event on startup!
-			//if pendingSendEvent { // TODO: should this become a list in the future?
-			//	if exit, err := obj.DoSend(chanProcess, ""); exit || err != nil {
-			//		return err // we exit or bubble up a NACK...
-			//	}
-			//}
-		}
-
-		// TODO: reset the watch retry count after some amount of success
-		e := v.Res.Watch(chanProcess)
-		if e == nil { // exit signal
-			err = nil // clean exit
-			break
-		}
-		if sentinelErr, ok := e.(*SentinelErr); ok { // unwrap the sentinel
-			err = sentinelErr.err
-			break // sentinel means, perma-exit
-		}
-		log.Printf("%v[%v]: Watch errored: %v", v.Kind(), v.GetName(), e)
-		if watchRetry == 0 {
-			err = fmt.Errorf("Permanent watch error: %v", e)
-			break
-		}
-		if watchRetry > 0 { // don't decrement the -1
-			watchRetry--
-		}
-		watchDelay = time.Duration(v.Meta().Delay) * time.Millisecond
-		log.Printf("%v[%v]: Watch: Retrying after %.4f seconds (%d left)", v.Kind(), v.GetName(), watchDelay.Seconds(), watchRetry)
-		// We need to trigger a CheckApply after Watch restarts, so that
-		// we catch any lost events that happened while down. We do this
-		// by getting the Watch resource to send one event once it's up!
-		//v.SendEvent(eventPoke, false, false)
-	}
-	close(chanProcess)
-	return err
-}
-
-// Start is a main kick to start the graph. It goes through in reverse topological
-// sort order so that events can't hit un-started vertices.
-func (g *Graph) Start(wg *sync.WaitGroup, first bool) { // start or continue
-	log.Printf("State: %v -> %v", g.setState(graphStateStarting), g.getState())
-	defer log.Printf("State: %v -> %v", g.setState(graphStateStarted), g.getState())
-	t, _ := g.TopologicalSort()
-	// TODO: only calculate indegree if `first` is true to save resources
-	indegree := g.InDegree() // compute all of the indegree's
-	for _, v := range Reverse(t) {
-
-		if !v.Res.IsWatching() { // if Watch() is not running...
-			wg.Add(1)
-			// must pass in value to avoid races...
-			// see: https://ttboj.wordpress.com/2015/07/27/golang-parallelism-issues-causing-too-many-open-files-error/
-			go func(vv *Vertex) {
-				defer wg.Done()
-				// TODO: if a sufficient number of workers error,
-				// should something be done? Will these restart
-				// after perma-failure if we have a graph change?
-				if err := g.Worker(vv); err != nil { // contains the Watch and CheckApply loops
-					log.Printf("%s[%s]: Exited with failure: %v", vv.Kind(), vv.GetName(), err)
-					return
-				}
-				log.Printf("%v[%v]: Exited", vv.Kind(), vv.GetName())
-			}(v)
-		}
-
-		// selective poke: here we reduce the number of initial pokes
-		// to the minimum required to activate every vertex in the
-		// graph, either by direct action, or by getting poked by a
-		// vertex that was previously activated. if we poke each vertex
-		// that has no incoming edges, then we can be sure to reach the
-		// whole graph. Please note: this may mask certain optimization
-		// failures, such as any poke limiting code in Poke() or
-		// BackPoke(). You might want to disable this selective start
-		// when experimenting with and testing those elements.
-		// if we are unpausing (since it's not the first run of this
-		// function) we need to poke to *unpause* every graph vertex,
-		// and not just selectively the subset with no indegree.
-		if (!first) || indegree[v] == 0 {
-			// ensure state is started before continuing on to next vertex
-			for !v.SendEvent(event.EventStart, true, false) {
-				if global.DEBUG {
-					// if SendEvent fails, we aren't up yet
-					log.Printf("%v[%v]: Retrying SendEvent(Start)", v.Kind(), v.GetName())
-					// sleep here briefly or otherwise cause
-					// a different goroutine to be scheduled
-					time.Sleep(1 * time.Millisecond)
-				}
-			}
-		}
-	}
-}
-
-// Pause sends pause events to the graph in a topological sort order.
-func (g *Graph) Pause() {
-	log.Printf("State: %v -> %v", g.setState(graphStatePausing), g.getState())
-	defer log.Printf("State: %v -> %v", g.setState(graphStatePaused), g.getState())
-	t, _ := g.TopologicalSort()
-	for _, v := range t { // squeeze out the events...
-		v.SendEvent(event.EventPause, true, false)
-	}
-}
-
-// Exit sends exit events to the graph in a topological sort order.
-func (g *Graph) Exit() {
-	if g == nil {
-		return
-	} // empty graph that wasn't populated yet
-	t, _ := g.TopologicalSort()
-	for _, v := range t { // squeeze out the events...
-		// turn off the taps...
-		// XXX: consider instead doing this by closing the Res.events channel instead?
-		// XXX: do this by sending an exit signal, and then returning
-		// when we hit the 'default' in the select statement!
-		// XXX: we can do this to quiesce, but it's not necessary now
-
 			v.SendEvent(event.EventExit, true, false)
+			oldGraph.DeleteVertex(v)
 		}
 	}
 
-// AssociateData associates some data with the object in the graph in question
+	// compare edges
-func (g *Graph) AssociateData(converger converger.Converger) {
+	for v1 := range g.Adjacency { // loop through the vertices (resources)
-	for v := range g.GetVerticesChan() {
+		for v2, e := range g.Adjacency[v1] {
-		v.Res.AssociateData(converger)
+			// we have an edge!
+
+			// lookup vertices (these should exist now)
+			//res1 := v1.Res // resource
+			//res2 := v2.Res
+			//vertex1 := oldGraph.GetVertexMatch(res1)
+			//vertex2 := oldGraph.GetVertexMatch(res2)
+			vertex1, exists1 := lookup[v1]
+			vertex2, exists2 := lookup[v2]
+			if !exists1 || !exists2 { // no match found, bug?
+				//if vertex1 == nil || vertex2 == nil { // no match found
+				return nil, fmt.Errorf("New vertices weren't found!") // programming error
+			}
+
+			edge, exists := oldGraph.Adjacency[vertex1][vertex2]
+			if !exists || edge.Name != e.Name { // TODO: edgeCmp
+				edge = e // use or overwrite edge
+			}
+			oldGraph.Adjacency[vertex1][vertex2] = edge // store it (AddEdge)
+			edgeKeep = append(edgeKeep, edge)           // mark as saved
+		}
+	}
+
+	// delete unused edges
+	for v1 := range oldGraph.Adjacency {
+		for _, e := range oldGraph.Adjacency[v1] {
+			// we have an edge!
+			if !EdgeContains(e, edgeKeep) {
+				oldGraph.DeleteEdge(e)
+			}
+		}
+	}
+
+	return oldGraph, nil
+}
+
+// GraphMetas returns a list of pointers to each of the resource MetaParams.
+func (g *Graph) GraphMetas() []*resources.MetaParams {
+	metas := []*resources.MetaParams{}
+	for v := range g.Adjacency { // loop through the vertices (resources))
+		res := v.Res // resource
+		meta := res.Meta()
+		metas = append(metas, meta)
+	}
+	return metas
+}
+
+// AssociateData associates some data with the object in the graph in question.
+func (g *Graph) AssociateData(data *resources.Data) {
+	for k := range g.Adjacency {
+		k.Res.AssociateData(data)
 	}
 }
 
@@ -956,6 +636,16 @@ func VertexContains(needle *Vertex, haystack []*Vertex) bool {
 	return false
 }
 
+// EdgeContains is an "in array" function to test for an edge in a slice of edges.
+func EdgeContains(needle *Edge, haystack []*Edge) bool {
+	for _, v := range haystack {
+		if needle == v {
+			return true
+		}
+	}
+	return false
+}
+
 // Reverse reverses a list of vertices.
 func Reverse(vs []*Vertex) []*Vertex {
 	//var out []*Vertex       // XXX: golint suggests, but it fails testing

pgraph/pgraph_test.go

@@ -26,6 +26,15 @@ import (
 	"time"
 )
 
+// NV is a helper function to make testing easier. It creates a new noop vertex.
+func NV(s string) *Vertex {
+	obj, err := NewNoopRes(s)
+	if err != nil {
+		panic(err) // unlikely test failure!
+	}
+	return NewVertex(obj)
+}
+
 func TestPgraphT1(t *testing.T) {
 
 	G := NewGraph("g1")
@@ -38,8 +47,8 @@ func TestPgraphT1(t *testing.T) {
 		t.Errorf("Should have 0 edges instead of: %d.", i)
 	}
 
-	v1 := NewVertex(NewNoopRes("v1"))
+	v1 := NV("v1")
-	v2 := NewVertex(NewNoopRes("v2"))
+	v2 := NV("v2")
 	e1 := NewEdge("e1")
 	G.AddEdge(v1, v2, e1)
 
@@ -55,12 +64,12 @@ func TestPgraphT1(t *testing.T) {
 func TestPgraphT2(t *testing.T) {
 
 	G := NewGraph("g2")
-	v1 := NewVertex(NewNoopRes("v1"))
+	v1 := NV("v1")
-	v2 := NewVertex(NewNoopRes("v2"))
+	v2 := NV("v2")
-	v3 := NewVertex(NewNoopRes("v3"))
+	v3 := NV("v3")
-	v4 := NewVertex(NewNoopRes("v4"))
+	v4 := NV("v4")
-	v5 := NewVertex(NewNoopRes("v5"))
+	v5 := NV("v5")
-	v6 := NewVertex(NewNoopRes("v6"))
+	v6 := NV("v6")
 	e1 := NewEdge("e1")
 	e2 := NewEdge("e2")
 	e3 := NewEdge("e3")
@@ -82,12 +91,12 @@ func TestPgraphT2(t *testing.T) {
 func TestPgraphT3(t *testing.T) {
 
 	G := NewGraph("g3")
-	v1 := NewVertex(NewNoopRes("v1"))
+	v1 := NV("v1")
-	v2 := NewVertex(NewNoopRes("v2"))
+	v2 := NV("v2")
-	v3 := NewVertex(NewNoopRes("v3"))
+	v3 := NV("v3")
-	v4 := NewVertex(NewNoopRes("v4"))
+	v4 := NV("v4")
-	v5 := NewVertex(NewNoopRes("v5"))
+	v5 := NV("v5")
-	v6 := NewVertex(NewNoopRes("v6"))
+	v6 := NV("v6")
 	e1 := NewEdge("e1")
 	e2 := NewEdge("e2")
 	e3 := NewEdge("e3")
@@ -123,9 +132,9 @@ func TestPgraphT3(t *testing.T) {
 func TestPgraphT4(t *testing.T) {
 
 	G := NewGraph("g4")
-	v1 := NewVertex(NewNoopRes("v1"))
+	v1 := NV("v1")
-	v2 := NewVertex(NewNoopRes("v2"))
+	v2 := NV("v2")
-	v3 := NewVertex(NewNoopRes("v3"))
+	v3 := NV("v3")
 	e1 := NewEdge("e1")
 	e2 := NewEdge("e2")
 	e3 := NewEdge("e3")
@@ -145,12 +154,12 @@ func TestPgraphT4(t *testing.T) {
 
 func TestPgraphT5(t *testing.T) {
 	G := NewGraph("g5")
-	v1 := NewVertex(NewNoopRes("v1"))
+	v1 := NV("v1")
-	v2 := NewVertex(NewNoopRes("v2"))
+	v2 := NV("v2")
-	v3 := NewVertex(NewNoopRes("v3"))
+	v3 := NV("v3")
-	v4 := NewVertex(NewNoopRes("v4"))
+	v4 := NV("v4")
-	v5 := NewVertex(NewNoopRes("v5"))
+	v5 := NV("v5")
-	v6 := NewVertex(NewNoopRes("v6"))
+	v6 := NV("v6")
 	e1 := NewEdge("e1")
 	e2 := NewEdge("e2")
 	e3 := NewEdge("e3")
@@ -174,12 +183,12 @@ func TestPgraphT5(t *testing.T) {
 
 func TestPgraphT6(t *testing.T) {
 	G := NewGraph("g6")
-	v1 := NewVertex(NewNoopRes("v1"))
+	v1 := NV("v1")
-	v2 := NewVertex(NewNoopRes("v2"))
+	v2 := NV("v2")
-	v3 := NewVertex(NewNoopRes("v3"))
+	v3 := NV("v3")
-	v4 := NewVertex(NewNoopRes("v4"))
+	v4 := NV("v4")
-	v5 := NewVertex(NewNoopRes("v5"))
+	v5 := NV("v5")
-	v6 := NewVertex(NewNoopRes("v6"))
+	v6 := NV("v6")
 	e1 := NewEdge("e1")
 	e2 := NewEdge("e2")
 	e3 := NewEdge("e3")
@@ -212,9 +221,9 @@ func TestPgraphT6(t *testing.T) {
 func TestPgraphT7(t *testing.T) {
 
 	G := NewGraph("g7")
-	v1 := NewVertex(NewNoopRes("v1"))
+	v1 := NV("v1")
-	v2 := NewVertex(NewNoopRes("v2"))
+	v2 := NV("v2")
-	v3 := NewVertex(NewNoopRes("v3"))
+	v3 := NV("v3")
 	e1 := NewEdge("e1")
 	e2 := NewEdge("e2")
 	e3 := NewEdge("e3")
@@ -253,28 +262,28 @@ func TestPgraphT7(t *testing.T) {
 
 func TestPgraphT8(t *testing.T) {
 
-	v1 := NewVertex(NewNoopRes("v1"))
+	v1 := NV("v1")
-	v2 := NewVertex(NewNoopRes("v2"))
+	v2 := NV("v2")
-	v3 := NewVertex(NewNoopRes("v3"))
+	v3 := NV("v3")
 	if VertexContains(v1, []*Vertex{v1, v2, v3}) != true {
 		t.Errorf("Should be true instead of false.")
 	}
 
-	v4 := NewVertex(NewNoopRes("v4"))
+	v4 := NV("v4")
-	v5 := NewVertex(NewNoopRes("v5"))
+	v5 := NV("v5")
-	v6 := NewVertex(NewNoopRes("v6"))
+	v6 := NV("v6")
 	if VertexContains(v4, []*Vertex{v5, v6}) != false {
 		t.Errorf("Should be false instead of true.")
 	}
 
-	v7 := NewVertex(NewNoopRes("v7"))
+	v7 := NV("v7")
-	v8 := NewVertex(NewNoopRes("v8"))
+	v8 := NV("v8")
-	v9 := NewVertex(NewNoopRes("v9"))
+	v9 := NV("v9")
 	if VertexContains(v8, []*Vertex{v7, v8, v9}) != true {
 		t.Errorf("Should be true instead of false.")
 	}
 
-	v1b := NewVertex(NewNoopRes("v1")) // same value, different objects
+	v1b := NV("v1") // same value, different objects
 	if VertexContains(v1b, []*Vertex{v1, v2, v3}) != false {
 		t.Errorf("Should be false instead of true.")
 	}
@@ -283,12 +292,12 @@ func TestPgraphT8(t *testing.T) {
 func TestPgraphT9(t *testing.T) {
 
 	G := NewGraph("g9")
-	v1 := NewVertex(NewNoopRes("v1"))
+	v1 := NV("v1")
-	v2 := NewVertex(NewNoopRes("v2"))
+	v2 := NV("v2")
-	v3 := NewVertex(NewNoopRes("v3"))
+	v3 := NV("v3")
-	v4 := NewVertex(NewNoopRes("v4"))
+	v4 := NV("v4")
-	v5 := NewVertex(NewNoopRes("v5"))
+	v5 := NV("v5")
-	v6 := NewVertex(NewNoopRes("v6"))
+	v6 := NV("v6")
 	e1 := NewEdge("e1")
 	e2 := NewEdge("e2")
 	e3 := NewEdge("e3")
@@ -343,11 +352,11 @@ func TestPgraphT9(t *testing.T) {
 		t.Errorf("Outdegree of v6 should be 0 instead of: %d.", i)
 	}
 
-	s, ok := G.TopologicalSort()
+	s, err := G.TopologicalSort()
 	// either possibility is a valid toposort
 	match := reflect.DeepEqual(s, []*Vertex{v1, v2, v3, v4, v5, v6}) || reflect.DeepEqual(s, []*Vertex{v1, v3, v2, v4, v5, v6})
-	if !ok || !match {
+	if err != nil || !match {
-		t.Errorf("Topological sort failed, status: %v.", ok)
+		t.Errorf("Topological sort failed, error: %v.", err)
 		str := "Found:"
 		for _, v := range s {
 			str += " " + v.Res.GetName()
@@ -359,12 +368,12 @@ func TestPgraphT9(t *testing.T) {
 func TestPgraphT10(t *testing.T) {
 
 	G := NewGraph("g10")
-	v1 := NewVertex(NewNoopRes("v1"))
+	v1 := NV("v1")
-	v2 := NewVertex(NewNoopRes("v2"))
+	v2 := NV("v2")
-	v3 := NewVertex(NewNoopRes("v3"))
+	v3 := NV("v3")
-	v4 := NewVertex(NewNoopRes("v4"))
+	v4 := NV("v4")
-	v5 := NewVertex(NewNoopRes("v5"))
+	v5 := NV("v5")
-	v6 := NewVertex(NewNoopRes("v6"))
+	v6 := NV("v6")
 	e1 := NewEdge("e1")
 	e2 := NewEdge("e2")
 	e3 := NewEdge("e3")
@@ -378,8 +387,8 @@ func TestPgraphT10(t *testing.T) {
 	G.AddEdge(v5, v6, e5)
 	G.AddEdge(v4, v2, e6) // cycle
 
-	if _, ok := G.TopologicalSort(); ok {
+	if _, err := G.TopologicalSort(); err == nil {
-		t.Errorf("Topological sort passed, but graph is cyclic.")
+		t.Errorf("Topological sort passed, but graph is cyclic!")
 	}
 }
 
@@ -399,8 +408,8 @@ func TestPgraphReachability0(t *testing.T) {
 	}
 	{
 		G := NewGraph("g")
-		v1 := NewVertex(NewNoopRes("v1"))
+		v1 := NV("v1")
-		v6 := NewVertex(NewNoopRes("v6"))
+		v6 := NV("v6")
 
 		result := G.Reachability(v1, v6)
 		expected := []*Vertex{}
@@ -416,12 +425,12 @@ func TestPgraphReachability0(t *testing.T) {
 	}
 	{
 		G := NewGraph("g")
-		v1 := NewVertex(NewNoopRes("v1"))
+		v1 := NV("v1")
-		v2 := NewVertex(NewNoopRes("v2"))
+		v2 := NV("v2")
-		v3 := NewVertex(NewNoopRes("v3"))
+		v3 := NV("v3")
-		v4 := NewVertex(NewNoopRes("v4"))
+		v4 := NV("v4")
-		v5 := NewVertex(NewNoopRes("v5"))
+		v5 := NV("v5")
-		v6 := NewVertex(NewNoopRes("v6"))
+		v6 := NV("v6")
 		e1 := NewEdge("e1")
 		e2 := NewEdge("e2")
 		e3 := NewEdge("e3")
@@ -450,12 +459,12 @@ func TestPgraphReachability0(t *testing.T) {
 // simple linear path
 func TestPgraphReachability1(t *testing.T) {
 	G := NewGraph("g")
-	v1 := NewVertex(NewNoopRes("v1"))
+	v1 := NV("v1")
-	v2 := NewVertex(NewNoopRes("v2"))
+	v2 := NV("v2")
-	v3 := NewVertex(NewNoopRes("v3"))
+	v3 := NV("v3")
-	v4 := NewVertex(NewNoopRes("v4"))
+	v4 := NV("v4")
-	v5 := NewVertex(NewNoopRes("v5"))
+	v5 := NV("v5")
-	v6 := NewVertex(NewNoopRes("v6"))
+	v6 := NV("v6")
 	e1 := NewEdge("e1")
 	e2 := NewEdge("e2")
 	e3 := NewEdge("e3")
@@ -484,12 +493,12 @@ func TestPgraphReachability1(t *testing.T) {
 // pick one of two correct paths
 func TestPgraphReachability2(t *testing.T) {
 	G := NewGraph("g")
-	v1 := NewVertex(NewNoopRes("v1"))
+	v1 := NV("v1")
-	v2 := NewVertex(NewNoopRes("v2"))
+	v2 := NV("v2")
-	v3 := NewVertex(NewNoopRes("v3"))
+	v3 := NV("v3")
-	v4 := NewVertex(NewNoopRes("v4"))
+	v4 := NV("v4")
-	v5 := NewVertex(NewNoopRes("v5"))
+	v5 := NV("v5")
-	v6 := NewVertex(NewNoopRes("v6"))
+	v6 := NV("v6")
 	e1 := NewEdge("e1")
 	e2 := NewEdge("e2")
 	e3 := NewEdge("e3")
@@ -521,12 +530,12 @@ func TestPgraphReachability2(t *testing.T) {
 // pick shortest path
 func TestPgraphReachability3(t *testing.T) {
 	G := NewGraph("g")
-	v1 := NewVertex(NewNoopRes("v1"))
+	v1 := NV("v1")
-	v2 := NewVertex(NewNoopRes("v2"))
+	v2 := NV("v2")
-	v3 := NewVertex(NewNoopRes("v3"))
+	v3 := NV("v3")
-	v4 := NewVertex(NewNoopRes("v4"))
+	v4 := NV("v4")
-	v5 := NewVertex(NewNoopRes("v5"))
+	v5 := NV("v5")
-	v6 := NewVertex(NewNoopRes("v6"))
+	v6 := NV("v6")
 	e1 := NewEdge("e1")
 	e2 := NewEdge("e2")
 	e3 := NewEdge("e3")
@@ -556,12 +565,12 @@ func TestPgraphReachability3(t *testing.T) {
 // direct path
 func TestPgraphReachability4(t *testing.T) {
 	G := NewGraph("g")
-	v1 := NewVertex(NewNoopRes("v1"))
+	v1 := NV("v1")
-	v2 := NewVertex(NewNoopRes("v2"))
+	v2 := NV("v2")
-	v3 := NewVertex(NewNoopRes("v3"))
+	v3 := NV("v3")
-	v4 := NewVertex(NewNoopRes("v4"))
+	v4 := NV("v4")
-	v5 := NewVertex(NewNoopRes("v5"))
+	v5 := NV("v5")
-	v6 := NewVertex(NewNoopRes("v6"))
+	v6 := NV("v6")
 	e1 := NewEdge("e1")
 	e2 := NewEdge("e2")
 	e3 := NewEdge("e3")
@@ -589,12 +598,12 @@ func TestPgraphReachability4(t *testing.T) {
 }
 
 func TestPgraphT11(t *testing.T) {
-	v1 := NewVertex(NewNoopRes("v1"))
+	v1 := NV("v1")
-	v2 := NewVertex(NewNoopRes("v2"))
+	v2 := NV("v2")
-	v3 := NewVertex(NewNoopRes("v3"))
+	v3 := NV("v3")
-	v4 := NewVertex(NewNoopRes("v4"))
+	v4 := NV("v4")
-	v5 := NewVertex(NewNoopRes("v5"))
+	v5 := NV("v5")
-	v6 := NewVertex(NewNoopRes("v6"))
+	v6 := NV("v6")
 
 	if rev := Reverse([]*Vertex{}); !reflect.DeepEqual(rev, []*Vertex{}) {
 		t.Errorf("Reverse of vertex slice failed.")

puppet/gapi.go

@@ -0,0 +1,121 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package puppet
+
+import (
+	"fmt"
+	"log"
+	"sync"
+	"time"
+
+	"github.com/purpleidea/mgmt/gapi"
+	"github.com/purpleidea/mgmt/pgraph"
+)
+
+// GAPI implements the main puppet GAPI interface.
+type GAPI struct {
+	PuppetParam *string // puppet mode to run; nil if undefined
+	PuppetConf  string  // the path to an alternate puppet.conf file
+
+	data        gapi.Data
+	initialized bool
+	closeChan   chan struct{}
+	wg          sync.WaitGroup // sync group for tunnel go routines
+}
+
+// NewGAPI creates a new puppet GAPI struct and calls Init().
+func NewGAPI(data gapi.Data, puppetParam *string, puppetConf string) (*GAPI, error) {
+	obj := &GAPI{
+		PuppetParam: puppetParam,
+		PuppetConf:  puppetConf,
+	}
+	return obj, obj.Init(data)
+}
+
+// Init initializes the puppet GAPI struct.
+func (obj *GAPI) Init(data gapi.Data) error {
+	if obj.initialized {
+		return fmt.Errorf("Already initialized!")
+	}
+	if obj.PuppetParam == nil {
+		return fmt.Errorf("The PuppetParam param must be specified!")
+	}
+	obj.data = data // store for later
+	obj.closeChan = make(chan struct{})
+	obj.initialized = true
+	return nil
+}
+
+// Graph returns a current Graph.
+func (obj *GAPI) Graph() (*pgraph.Graph, error) {
+	if !obj.initialized {
+		return nil, fmt.Errorf("Puppet: GAPI is not initialized!")
+	}
+	config := ParseConfigFromPuppet(*obj.PuppetParam, obj.PuppetConf)
+	if config == nil {
+		return nil, fmt.Errorf("Puppet: ParseConfigFromPuppet returned nil!")
+	}
+	g, err := config.NewGraphFromConfig(obj.data.Hostname, obj.data.World, obj.data.Noop)
+	return g, err
+}
+
+// Next returns nil errors every time there could be a new graph.
+func (obj *GAPI) Next() chan error {
+	if obj.data.NoWatch {
+		return nil
+	}
+	puppetChan := func() <-chan time.Time { // helper function
+		return time.Tick(time.Duration(PuppetInterval(obj.PuppetConf)) * time.Second)
+	}
+	ch := make(chan error)
+	obj.wg.Add(1)
+	go func() {
+		defer obj.wg.Done()
+		defer close(ch) // this will run before the obj.wg.Done()
+		if !obj.initialized {
+			ch <- fmt.Errorf("Puppet: GAPI is not initialized!")
+			return
+		}
+		pChan := puppetChan()
+		for {
+			select {
+			case _, ok := <-pChan:
+				if !ok { // the channel closed!
+					return
+				}
+				log.Printf("Puppet: Generating new graph...")
+				pChan = puppetChan() // TODO: okay to update interval in case it changed?
+				ch <- nil            // trigger a run
+			case <-obj.closeChan:
+				return
+			}
+		}
+	}()
+	return ch
+}
+
+// Close shuts down the Puppet GAPI.
+func (obj *GAPI) Close() error {
+	if !obj.initialized {
+		return fmt.Errorf("Puppet: GAPI is not initialized!")
+	}
+	close(obj.closeChan)
+	obj.wg.Wait()
+	obj.initialized = false // closed = true
+	return nil
+}

puppet/puppet.go

@@ -26,17 +26,18 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/purpleidea/mgmt/gconfig"
+	"github.com/purpleidea/mgmt/yamlgraph"
-	"github.com/purpleidea/mgmt/global"
 )
 
 const (
 	// PuppetYAMLBufferSize is the maximum buffer size for the yaml input data
 	PuppetYAMLBufferSize = 65535
+	// Debug is a local debug constant used in this module
+	Debug = false // FIXME: integrate with global debug flag
 )
 
 func runPuppetCommand(cmd *exec.Cmd) ([]byte, error) {
-	if global.DEBUG {
+	if Debug {
 		log.Printf("Puppet: running command: %v", cmd)
 	}
 
@@ -71,7 +72,7 @@ func runPuppetCommand(cmd *exec.Cmd) ([]byte, error) {
 		// will choke on an oversized slice. http://stackoverflow.com/a/33726617/3356612
 		result = append(result, data[0:count]...)
 	}
-	if global.DEBUG {
+	if Debug {
 		log.Printf("Puppet: read %v bytes of data from puppet", len(result))
 	}
 	for scanner := bufio.NewScanner(stderr); scanner.Scan(); {
@@ -87,7 +88,7 @@ func runPuppetCommand(cmd *exec.Cmd) ([]byte, error) {
 
 // ParseConfigFromPuppet takes a special puppet param string and config and
 // returns the graph configuration structure.
-func ParseConfigFromPuppet(puppetParam, puppetConf string) *gconfig.GraphConfig {
+func ParseConfigFromPuppet(puppetParam, puppetConf string) *yamlgraph.GraphConfig {
 	var puppetConfArg string
 	if puppetConf != "" {
 		puppetConfArg = "--config=" + puppetConf
@@ -104,7 +105,7 @@ func ParseConfigFromPuppet(puppetParam, puppetConf string) *gconfig.GraphConfig
 
 	log.Println("Puppet: launching translator")
 
-	var config gconfig.GraphConfig
+	var config yamlgraph.GraphConfig
 	if data, err := runPuppetCommand(cmd); err != nil {
 		return nil
 	} else if err := config.Parse(data); err != nil {
@@ -117,7 +118,7 @@ func ParseConfigFromPuppet(puppetParam, puppetConf string) *gconfig.GraphConfig
 
 // PuppetInterval returns the graph refresh interval from the puppet configuration.
 func PuppetInterval(puppetConf string) int {
-	if global.DEBUG {
+	if Debug {
 		log.Printf("Puppet: determining graph refresh interval")
 	}
 	var cmd *exec.Cmd

recwatch/configwatch.go

@@ -20,12 +20,12 @@ package recwatch
 import (
 	"log"
 	"sync"
-
-	"github.com/purpleidea/mgmt/global"
 )
 
 // ConfigWatcher returns events on a channel anytime one of its files events.
 type ConfigWatcher struct {
+	Flags Flags
+
 	ch        chan string
 	wg        sync.WaitGroup
 	closechan chan struct{}
@@ -56,7 +56,7 @@ func (obj *ConfigWatcher) Add(file ...string) {
 	obj.wg.Add(1)
 	go func() {
 		defer obj.wg.Done()
-		ch := ConfigWatch(file[0])
+		ch := obj.ConfigWatch(file[0])
 		for {
 			select {
 			case e := <-ch:
@@ -100,7 +100,7 @@ func (obj *ConfigWatcher) Close() {
 }
 
 // ConfigWatch writes on the channel every time an event is seen for the path.
-func ConfigWatch(file string) chan error {
+func (obj *ConfigWatcher) ConfigWatch(file string) chan error {
 	ch := make(chan error)
 	go func() {
 		recWatcher, err := NewRecWatcher(file, false)
@@ -109,9 +109,10 @@ func ConfigWatch(file string) chan error {
 			close(ch)
 			return
 		}
+		recWatcher.Flags = obj.Flags
 		defer recWatcher.Close()
 		for {
-			if global.DEBUG {
+			if obj.Flags.Debug {
 				log.Printf("Watching: %v", file)
 			}
 			select {

recwatch/flags.go

@@ -15,12 +15,9 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package main
+package recwatch
 
-import (
+// Flags contains all the constant flags that recwatch needs.
-//"testing"
+type Flags struct {
-)
+	Debug bool
-
+}
-//func TestT1(t *testing.T) {
-
-//}

recwatch/recwatch.go

@@ -29,7 +29,6 @@ import (
 	"sync"
 	"syscall"
 
-	"github.com/purpleidea/mgmt/global" // XXX: package mgmtmain instead?
 	"github.com/purpleidea/mgmt/util"
 
 	"gopkg.in/fsnotify.v1"
@@ -46,6 +45,7 @@ type Event struct {
 type RecWatcher struct {
 	Path     string // computed path
 	Recurse  bool   // should we watch recursively?
+	Flags    Flags
 	isDir    bool   // computed isDir
 	safename string // safe path
 	watcher  *fsnotify.Watcher
@@ -96,11 +96,11 @@ func (obj *RecWatcher) Init() error {
 	return nil
 }
 
-//func (obj *RecWatcher) Add(path string) error { // XXX implement me or not?
+//func (obj *RecWatcher) Add(path string) error { // XXX: implement me or not?
 //
 //}
 //
-//func (obj *RecWatcher) Remove(path string) error { // XXX implement me or not?
+//func (obj *RecWatcher) Remove(path string) error { // XXX: implement me or not?
 //
 //}
 
@@ -150,12 +150,12 @@ func (obj *RecWatcher) Watch() error {
 		if current == "" { // the empty string top is the root dir ("/")
 			current = "/"
 		}
-		if global.DEBUG {
+		if obj.Flags.Debug {
 			log.Printf("Watching: %s", current) // attempting to watch...
 		}
 		// initialize in the loop so that we can reset on rm-ed handles
 		if err := obj.watcher.Add(current); err != nil {
-			if global.DEBUG {
+			if obj.Flags.Debug {
 				log.Printf("watcher.Add(%s): Error: %v", current, err)
 			}
 
@@ -178,7 +178,7 @@ func (obj *RecWatcher) Watch() error {
 
 		select {
 		case event := <-obj.watcher.Events:
-			if global.DEBUG {
+			if obj.Flags.Debug {
 				log.Printf("Watch(%s), Event(%s): %v", current, event.Name, event.Op)
 			}
 			// the deeper you go, the bigger the deltaDepth is...
@@ -209,7 +209,7 @@ func (obj *RecWatcher) Watch() error {
 				}
 
 			} else {
-				// TODO different watchers get each others events!
+				// TODO: different watchers get each others events!
 				// https://github.com/go-fsnotify/fsnotify/issues/95
 				// this happened with two values such as:
 				// event.Name: /tmp/mgmt/f3 and current: /tmp/mgmt/f2
@@ -291,7 +291,7 @@ func (obj *RecWatcher) addSubFolders(p string) error {
 	}
 	// look at all subfolders...
 	walkFn := func(path string, info os.FileInfo, err error) error {
-		if global.DEBUG {
+		if obj.Flags.Debug {
 			log.Printf("Walk: %s (%v): %v", path, info, err)
 		}
 		if err != nil {

remote/remote.go

@@ -62,12 +62,13 @@ import (
 	"time"
 
 	cv "github.com/purpleidea/mgmt/converger"
-	"github.com/purpleidea/mgmt/gconfig"
-	"github.com/purpleidea/mgmt/global"
 	"github.com/purpleidea/mgmt/util"
+	"github.com/purpleidea/mgmt/yamlgraph"
 
+	multierr "github.com/hashicorp/go-multierror"
 	"github.com/howeyc/gopass"
 	"github.com/kardianos/osext"
+	errwrap "github.com/pkg/errors"
 	"github.com/pkg/sftp"
 	"golang.org/x/crypto/ssh"
 )
@@ -83,6 +84,12 @@ const (
 	nonInteractivePasswordTimeout        = 5 * 2                                  // five minutes
 )
 
+// Flags are constants required by the remote lib.
+type Flags struct {
+	Program string
+	Debug   bool
+}
+
 // The SSH struct is the unit building block for a single remote SSH connection.
 type SSH struct {
 	hostname string // uuid of the host, as used by the --hostname argument
@@ -114,7 +121,7 @@ type SSH struct {
 	lock    sync.Mutex     // mutex to avoid exit races
 	exiting bool           // flag to let us know if we're exiting
 
-	program  string // name of the binary
+	flags    Flags  // constant runtime values
 	remotewd string // path to remote working directory
 	execpath string // path to remote mgmt binary
 	filepath string // path to remote file config
@@ -149,11 +156,11 @@ func (obj *SSH) Sftp() error {
 	var err error
 
 	if obj.client == nil {
-		return fmt.Errorf("Not dialed!")
+		return fmt.Errorf("not dialed")
 	}
 	// this check is needed because the golang path.Base function is weird!
 	if strings.HasSuffix(obj.file, "/") {
-		return fmt.Errorf("File must not be a directory.")
+		return fmt.Errorf("file must not be a directory")
 	}
 
 	// we run local operations first so that remote clean up is easier...
@@ -171,7 +178,7 @@ func (obj *SSH) Sftp() error {
 
 	// TODO: make the path configurable to deal with /tmp/ mounted noexec?
 	tmpdir := func() string {
-		return fmt.Sprintf(formatPattern, fmtUUID(10)) // eg: /tmp/mgmt.abcdefghij/
+		return fmt.Sprintf(formatPattern, fmtUID(10)) // eg: /tmp/mgmt.abcdefghij/
 	}
 	var ready bool
 	obj.remotewd = ""
@@ -189,7 +196,7 @@ func (obj *SSH) Sftp() error {
 	}
 
 	for i := 0; true; {
-		// NOTE: since fmtUUID is deterministic, if we don't clean up
+		// NOTE: since fmtUID is deterministic, if we don't clean up
 		// previous runs, we may get the same paths generated, and here
 		// they will conflict.
 		if err := obj.sftp.Mkdir(obj.remotewd); err != nil {
@@ -222,7 +229,7 @@ func (obj *SSH) Sftp() error {
 		break
 	}
 
-	obj.execpath = path.Join(obj.remotewd, obj.program) // program is a compile time string
+	obj.execpath = path.Join(obj.remotewd, obj.flags.Program) // program is a compile time string
 	log.Printf("Remote: Remote path is: %s", obj.execpath)
 
 	var same bool
@@ -247,7 +254,7 @@ func (obj *SSH) Sftp() error {
 	// make file executable; don't cache this in case it didn't ever happen
 	// TODO: do we want the group or other bits set?
 	if err := obj.sftp.Chmod(obj.execpath, 0770); err != nil {
-		return fmt.Errorf("Can't set file mode bits!")
+		return fmt.Errorf("can't set file mode bits")
 	}
 
 	// copy graph file
@@ -266,7 +273,7 @@ func (obj *SSH) Sftp() error {
 // SftpGraphCopy is a helper function used for re-copying the graph definition.
 func (obj *SSH) SftpGraphCopy() (int64, error) {
 	if obj.filepath == "" {
-		return -1, fmt.Errorf("Sftp session isn't ready yet!")
+		return -1, fmt.Errorf("sftp session isn't ready yet")
 	}
 	return obj.SftpCopy(obj.file, obj.filepath)
 }
@@ -274,7 +281,7 @@ func (obj *SSH) SftpGraphCopy() (int64, error) {
 // SftpCopy is a simple helper function that runs a local -> remote sftp copy.
 func (obj *SSH) SftpCopy(src, dst string) (int64, error) {
 	if obj.sftp == nil {
-		return -1, fmt.Errorf("Sftp session is not active!")
+		return -1, fmt.Errorf("sftp session is not active")
 	}
 	var err error
 	// TODO: add a check to make sure we don't run two copies of this
@@ -306,7 +313,7 @@ func (obj *SSH) SftpCopy(src, dst string) (int64, error) {
 		return n, fmt.Errorf("Can't copy to remote path: %v", err)
 	}
 	if n <= 0 {
-		return n, fmt.Errorf("Zero bytes copied!")
+		return n, fmt.Errorf("zero bytes copied")
 	}
 	return n, nil
 }
@@ -384,10 +391,10 @@ func (obj *SSH) Tunnel() error {
 	var err error
 
 	if len(obj.clientURLs) < 1 {
-		return fmt.Errorf("Need at least one client URL to tunnel!")
+		return fmt.Errorf("need at least one client URL to tunnel")
 	}
 	if len(obj.remoteURLs) < 1 {
-		return fmt.Errorf("Need at least one remote URL to tunnel!")
+		return fmt.Errorf("need at least one remote URL to tunnel")
 	}
 
 	// TODO: do something less arbitrary about which one we pick?
@@ -446,7 +453,7 @@ func (obj *SSH) forward(remoteConn net.Conn) net.Conn {
 			log.Printf("Remote: io.Copy error: %s", err)
 			// FIXME: what should we do here???
 		}
-		if global.DEBUG {
+		if obj.flags.Debug {
 			log.Printf("Remote: io.Copy finished: %d", n)
 		}
 	}
@@ -470,10 +477,10 @@ func (obj *SSH) TunnelClose() error {
 // Exec runs the binary on the remote server.
 func (obj *SSH) Exec() error {
 	if obj.execpath == "" {
-		return fmt.Errorf("Must have a binary path to execute!")
+		return fmt.Errorf("must have a binary path to execute")
 	}
 	if obj.filepath == "" {
-		return fmt.Errorf("Must have a graph definition to run!")
+		return fmt.Errorf("must have a graph definition to run")
 	}
 
 	var err error
@@ -491,7 +498,7 @@ func (obj *SSH) Exec() error {
 	// TODO: do something less arbitrary about which one we pick?
 	url := cleanURL(obj.remoteURLs[0])                           // arbitrarily pick the first one
 	seeds := fmt.Sprintf("--no-server --seeds 'http://%s'", url) // XXX: escape untrusted input? (or check if url is valid)
-	file := fmt.Sprintf("--file '%s'", obj.filepath)             // XXX: escape untrusted input! (or check if file path exists)
+	file := fmt.Sprintf("--yaml '%s'", obj.filepath)             // XXX: escape untrusted input! (or check if file path exists)
 	depth := fmt.Sprintf("--depth %d", obj.depth+1)              // child is +1 distance
 	args := []string{hostname, seeds, file, depth}
 	if obj.noop {
@@ -561,7 +568,7 @@ func (obj *SSH) ExecExit() error {
 	}
 
 	// FIXME: workaround: force a signal!
-	if _, err := obj.simpleRun(fmt.Sprintf("killall -SIGINT %s", obj.program)); err != nil { // FIXME: low specificity
+	if _, err := obj.simpleRun(fmt.Sprintf("killall -SIGINT %s", obj.flags.Program)); err != nil { // FIXME: low specificity
 		log.Printf("Remote: Failed to send SIGINT: %s", err.Error())
 	}
 
@@ -570,12 +577,12 @@ func (obj *SSH) ExecExit() error {
 		// try killing the process more violently
 		time.Sleep(10 * time.Second)
 		//obj.session.Signal(ssh.SIGKILL)
-		cmd := fmt.Sprintf("killall -SIGKILL %s", obj.program) // FIXME: low specificity
+		cmd := fmt.Sprintf("killall -SIGKILL %s", obj.flags.Program) // FIXME: low specificity
 		obj.simpleRun(cmd)
 	}()
 
 	// FIXME: workaround: wait (spin lock) until process quits cleanly...
-	cmd := fmt.Sprintf("while killall -0 %s 2> /dev/null; do sleep 1s; done", obj.program) // FIXME: low specificity
+	cmd := fmt.Sprintf("while killall -0 %s 2> /dev/null; do sleep 1s; done", obj.flags.Program) // FIXME: low specificity
 	if _, err := obj.simpleRun(cmd); err != nil {
 		return fmt.Errorf("Error waiting: %s", err)
 	}
@@ -698,15 +705,15 @@ type Remotes struct {
 	exitChan           chan struct{}              // closes when we should exit
 	semaphore          Semaphore                  // counting semaphore to limit concurrent connections
 	hostnames          []string                   // list of hostnames we've seen so far
-	cuuid              cv.ConvergerUUID            // convergerUUID for the remote itself
+	cuid               cv.ConvergerUID            // convergerUID for the remote itself
-	cuuids             map[string]cv.ConvergerUUID // map to each SSH struct with the remote as the key
+	cuids              map[string]cv.ConvergerUID // map to each SSH struct with the remote as the key
 	callbackCancelFunc func()                     // stored callback function cancel function
 
-	program string // name of the program
+	flags Flags // constant runtime values
 }
 
 // NewRemotes builds a Remotes struct.
-func NewRemotes(clientURLs, remoteURLs []string, noop bool, remotes []string, fileWatch chan string, cConns uint16, interactive bool, sshPrivIdRsa string, caching bool, depth uint16, prefix string, converger cv.Converger, convergerCb func(func(map[string]bool) error) (func(), error), program string) *Remotes {
+func NewRemotes(clientURLs, remoteURLs []string, noop bool, remotes []string, fileWatch chan string, cConns uint16, interactive bool, sshPrivIdRsa string, caching bool, depth uint16, prefix string, converger cv.Converger, convergerCb func(func(map[string]bool) error) (func(), error), flags Flags) *Remotes {
 	return &Remotes{
 		clientURLs:   clientURLs,
 		remoteURLs:   remoteURLs,
@@ -725,8 +732,8 @@ func NewRemotes(clientURLs, remoteURLs []string, noop bool, remotes []string, fi
 		exitChan:     make(chan struct{}),
 		semaphore:    NewSemaphore(int(cConns)),
 		hostnames:    make([]string, len(remotes)),
-		cuuids:       make(map[string]cv.ConvergerUUID),
+		cuids:        make(map[string]cv.ConvergerUID),
-		program:      program,
+		flags:        flags,
 	}
 }
 
@@ -734,7 +741,7 @@ func NewRemotes(clientURLs, remoteURLs []string, noop bool, remotes []string, fi
 // It takes as input the path to a graph definition file.
 func (obj *Remotes) NewSSH(file string) (*SSH, error) {
 	// first do the parsing...
-	config := gconfig.ParseConfigFromFile(file)
+	config := yamlgraph.ParseConfigFromFile(file) // FIXME: GAPI-ify somehow?
 	if config == nil {
 		return nil, fmt.Errorf("Remote: Error parsing remote graph: %s", file)
 	}
@@ -765,7 +772,7 @@ func (obj *Remotes) NewSSH(file string) (*SSH, error) {
 	}
 	host = x[0]
 	if host == "" {
-		return nil, fmt.Errorf("Empty hostname!")
+		return nil, fmt.Errorf("empty hostname")
 	}
 
 	user := defaultUser // default
@@ -788,15 +795,16 @@ func (obj *Remotes) NewSSH(file string) (*SSH, error) {
 	}
 
 	if len(auth) == 0 {
-		return nil, fmt.Errorf("No authentication methods available!")
+		return nil, fmt.Errorf("no authentication methods available")
 	}
 
-	hostname := config.Hostname
+	//hostname := config.Hostname // TODO: optionally specify local hostname somehow
+	hostname := ""
 	if hostname == "" {
 		hostname = host // default to above
 	}
 	if util.StrInList(hostname, obj.hostnames) {
-		return nil, fmt.Errorf("Remote: Hostname `%s` already exists!", hostname)
+		return nil, fmt.Errorf("Remote: Hostname `%s` already exists", hostname)
 	}
 	obj.hostnames = append(obj.hostnames, hostname)
 
@@ -815,14 +823,14 @@ func (obj *Remotes) NewSSH(file string) (*SSH, error) {
 		caching:    obj.caching,
 		converger:  obj.converger,
 		prefix:     obj.prefix,
-		program:    obj.program,
+		flags:      obj.flags,
 	}, nil
 }
 
 // sshKeyAuth is a helper function to get the ssh key auth struct needed
 func (obj *Remotes) sshKeyAuth() (ssh.AuthMethod, error) {
 	if obj.sshPrivIdRsa == "" {
-		return nil, fmt.Errorf("Empty path specified!")
+		return nil, fmt.Errorf("empty path specified")
 	}
 	p := ""
 	// TODO: this doesn't match strings of the form: ~james/.ssh/id_rsa
@@ -835,7 +843,7 @@ func (obj *Remotes) sshKeyAuth() (ssh.AuthMethod, error) {
 		p = path.Join(usr.HomeDir, obj.sshPrivIdRsa[len("~/"):])
 	}
 	if p == "" {
-		return nil, fmt.Errorf("Empty path specified!")
+		return nil, fmt.Errorf("empty path specified")
 	}
 	// A public key may be used to authenticate against the server by using
 	// an unencrypted PEM-encoded private key file. If you have an encrypted
@@ -884,7 +892,7 @@ func (obj *Remotes) passwordCallback(user, host string) func() (string, error) {
 		case e := <-failchan:
 			return "", e
 		case <-util.TimeAfterOrBlock(timeout):
-			return "", fmt.Errorf("Interactive timeout reached!")
+			return "", fmt.Errorf("interactive timeout reached")
 		}
 	}
 	return cb
@@ -894,12 +902,12 @@ func (obj *Remotes) passwordCallback(user, host string) func() (string, error) {
 func (obj *Remotes) Run() {
 	// TODO: we can disable a lot of this if we're not using --converged-timeout
 	// link in all the converged timeout checking and callbacks...
-	obj.cuuid = obj.converger.Register() // one for me!
+	obj.cuid = obj.converger.Register() // one for me!
-	obj.cuuid.SetName("Remote: Run")
+	obj.cuid.SetName("Remote: Run")
 	for _, f := range obj.remotes { // one for each remote...
-		obj.cuuids[f] = obj.converger.Register() // save a reference
+		obj.cuids[f] = obj.converger.Register() // save a reference
-		obj.cuuids[f].SetName(fmt.Sprintf("Remote: %s", f))
+		obj.cuids[f].SetName(fmt.Sprintf("Remote: %s", f))
-		//obj.cuuids[f].SetConverged(false) // everyone starts off false
+		//obj.cuids[f].SetConverged(false) // everyone starts off false
 	}
 
 	// watch for converged state in the group of remotes...
@@ -921,12 +929,12 @@ func (obj *Remotes) Run() {
 			if !ok { // no status on hostname means unconverged!
 				continue
 			}
-			if global.DEBUG {
+			if obj.flags.Debug {
 				log.Printf("Remote: Converged: Status: %+v", obj.converger.Status())
 			}
 			// if exiting, don't update, it will be unregistered...
 			if !sshobj.exiting { // this is actually racy, but safe
-				obj.cuuids[f].SetConverged(b) // ignore errors!
+				obj.cuids[f].SetConverged(b) // ignore errors!
 			}
 		}
 
@@ -953,10 +961,10 @@ func (obj *Remotes) Run() {
 					if !more {
 						return
 					}
-					obj.cuuid.SetConverged(false) // activity!
+					obj.cuid.SetConverged(false) // activity!
 
-				case <-obj.cuuid.ConvergedTimer():
+				case <-obj.cuid.ConvergedTimer():
-					obj.cuuid.SetConverged(true) // converged!
+					obj.cuid.SetConverged(true) // converged!
 					continue
 				}
 				obj.lock.Lock()
@@ -975,7 +983,7 @@ func (obj *Remotes) Run() {
 			}
 		}()
 	} else {
-		obj.cuuid.SetConverged(true) // if no watches, we're converged!
+		obj.cuid.SetConverged(true) // if no watches, we're converged!
 	}
 
 	// the semaphore provides the max simultaneous connection limit
@@ -993,7 +1001,7 @@ func (obj *Remotes) Run() {
 			if obj.cConns != 0 {
 				obj.semaphore.V(1) // don't lock the loop
 			}
-			obj.cuuids[f].Unregister() // don't stall the converge!
+			obj.cuids[f].Unregister() // don't stall the converge!
 			continue
 		}
 		obj.sshmap[f] = sshobj // save a reference
@@ -1004,7 +1012,7 @@ func (obj *Remotes) Run() {
 				defer obj.semaphore.V(1)
 			}
 			defer obj.wg.Done()
-			defer obj.cuuids[f].Unregister()
+			defer obj.cuids[f].Unregister()
 
 			if err := sshobj.Go(); err != nil {
 				log.Printf("Remote: Error: %s", err)
@@ -1017,11 +1025,12 @@ func (obj *Remotes) Run() {
 
 // Exit causes as much of the Remotes struct to shutdown as quickly and as
 // cleanly as possible. It only returns once everything is shutdown.
-func (obj *Remotes) Exit() {
+func (obj *Remotes) Exit() error {
 	obj.lock.Lock()
 	obj.exiting = true // don't spawn new ones once this flag is set!
 	obj.lock.Unlock()
 	close(obj.exitChan)
+	var reterr error
 	for _, f := range obj.remotes {
 		sshobj, exists := obj.sshmap[f]
 		if !exists || sshobj == nil {
@@ -1030,7 +1039,8 @@ func (obj *Remotes) Exit() {
 
 		// TODO: should we run these as go routines?
 		if err := sshobj.Stop(); err != nil {
-			log.Printf("Remote: Error stopping: %s", err)
+			err = errwrap.Wrapf(err, "Remote: Error stopping!")
+			reterr = multierr.Append(reterr, err) // list of errors
 		}
 	}
 
@@ -1038,14 +1048,15 @@ func (obj *Remotes) Exit() {
 		obj.callbackCancelFunc() // cancel our callback
 	}
 
-	defer obj.cuuid.Unregister()
+	defer obj.cuid.Unregister()
 	obj.wg.Wait() // wait for everyone to exit
+	return reterr
 }
 
-// fmtUUID makes a random string of length n, it is not cryptographically safe.
+// fmtUID makes a random string of length n, it is not cryptographically safe.
 // This function actually usually generates the same sequence of random strings
 // each time the program is run, which makes repeatability of this code easier.
-func fmtUUID(n int) string {
+func fmtUID(n int) string {
 	b := make([]byte, n)
 	for i := range b {
 		b[i] = formatChars[rand.Intn(len(formatChars))]

resources/exec.go

@@ -21,15 +21,15 @@ import (
 	"bufio"
 	"bytes"
 	"encoding/gob"
-	"errors"
 	"fmt"
 	"log"
 	"os/exec"
 	"strings"
-	"time"
 
 	"github.com/purpleidea/mgmt/event"
 	"github.com/purpleidea/mgmt/util"
+
+	errwrap "github.com/pkg/errors"
 )
 
 func init() {
@@ -51,7 +51,7 @@ type ExecRes struct {
 }
 
 // NewExecRes is a constructor for this resource. It also calls Init() for you.
-func NewExecRes(name, cmd, shell string, timeout int, watchcmd, watchshell, ifcmd, ifshell string, pollint int, state string) *ExecRes {
+func NewExecRes(name, cmd, shell string, timeout int, watchcmd, watchshell, ifcmd, ifshell string, pollint int, state string) (*ExecRes, error) {
 	obj := &ExecRes{
 		BaseRes: BaseRes{
 			Name: name,
@@ -66,8 +66,7 @@ func NewExecRes(name, cmd, shell string, timeout int, watchcmd, watchshell, ifcm
 		PollInt:    pollint,
 		State:      state,
 	}
-	obj.Init()
+	return obj, obj.Init()
-	return obj
 }
 
 // Init runs some startup code for this resource.
@@ -99,7 +98,7 @@ func (obj *ExecRes) BufioChanScanner(scanner *bufio.Scanner) (chan string, chan
 			ch <- scanner.Text() // blocks here ?
 			if e := scanner.Err(); e != nil {
 				errch <- e // send any misc errors we encounter
-				//break // TODO ?
+				//break // TODO: ?
 			}
 		}
 		close(ch)
@@ -111,22 +110,7 @@ func (obj *ExecRes) BufioChanScanner(scanner *bufio.Scanner) (chan string, chan
 
 // Watch is the primary listener for this resource and it outputs events.
 func (obj *ExecRes) Watch(processChan chan event.Event) error {
-	if obj.IsWatching() {
+	cuid := obj.Converger() // get the converger uid used to report status
-		return nil
-	}
-	obj.SetWatching(true)
-	defer obj.SetWatching(false)
-	cuuid := obj.converger.Register()
-	defer cuuid.Unregister()
-
-	var startup bool
-	Startup := func(block bool) <-chan time.Time {
-		if block {
-			return nil // blocks forever
-			//return make(chan time.Time) // blocks forever
-		}
-		return time.After(time.Duration(500) * time.Millisecond) // 1/2 the resolution of converged timeout
-	}
 
 	var send = false // send event?
 	var exit = false
@@ -152,7 +136,7 @@ func (obj *ExecRes) Watch(processChan chan event.Event) error {
 
 		cmdReader, err := cmd.StdoutPipe()
 		if err != nil {
-			return fmt.Errorf("%s[%s]: Error creating StdoutPipe for Cmd: %v", obj.Kind(), obj.GetName(), err)
+			return errwrap.Wrapf(err, "Error creating StdoutPipe for Cmd")
 		}
 		scanner := bufio.NewScanner(cmdReader)
 
@@ -163,54 +147,54 @@ func (obj *ExecRes) Watch(processChan chan event.Event) error {
 			cmd.Process.Kill() // TODO: is this necessary?
 		}()
 		if err := cmd.Start(); err != nil {
-			return fmt.Errorf("%s[%s]: Error starting Cmd: %v", obj.Kind(), obj.GetName(), err)
+			return errwrap.Wrapf(err, "Error starting Cmd")
 		}
 
 		bufioch, errch = obj.BufioChanScanner(scanner)
 	}
 
+	// notify engine that we're running
+	if err := obj.Running(processChan); err != nil {
+		return err // bubble up a NACK...
+	}
+
 	for {
 		obj.SetState(ResStateWatching) // reset
 		select {
 		case text := <-bufioch:
-			cuuid.SetConverged(false)
+			cuid.SetConverged(false)
 			// each time we get a line of output, we loop!
-			log.Printf("%v[%v]: Watch output: %s", obj.Kind(), obj.GetName(), text)
+			log.Printf("%s[%s]: Watch output: %s", obj.Kind(), obj.GetName(), text)
 			if text != "" {
 				send = true
 			}
 
 		case err := <-errch:
-			cuuid.SetConverged(false)
+			cuid.SetConverged(false)
 			if err == nil { // EOF
 				// FIXME: add an "if watch command ends/crashes"
 				// restart or generate error option
-				return fmt.Errorf("%s[%s]: Reached EOF", obj.Kind(), obj.GetName())
+				return fmt.Errorf("Reached EOF")
 			}
 			// error reading input?
-			return fmt.Errorf("Unknown %s[%s] error: %v", obj.Kind(), obj.GetName(), err)
+			return errwrap.Wrapf(err, "Unknown error")
 
-		case event := <-obj.events:
+		case event := <-obj.Events():
-			cuuid.SetConverged(false)
+			cuid.SetConverged(false)
 			if exit, send = obj.ReadEvent(&event); exit {
 				return nil // exit
 			}
 
-		case <-cuuid.ConvergedTimer():
+		case <-cuid.ConvergedTimer():
-			cuuid.SetConverged(true) // converged!
+			cuid.SetConverged(true) // converged!
 			continue
-
-		case <-Startup(startup):
-			cuuid.SetConverged(false)
-			send = true
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
-			startup = true // startup finished
 			send = false
 			// it is okay to invalidate the clean state on poke too
-			obj.isStateOK = false // something made state dirty
+			obj.StateOK(false) // something made state dirty
 			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
 				return err // we exit or bubble up a NACK...
 			}
@@ -221,12 +205,11 @@ func (obj *ExecRes) Watch(processChan chan event.Event) error {
 // CheckApply checks the resource state and applies the resource if the bool
 // input is true. It returns error info and if the state check passed or not.
 // TODO: expand the IfCmd to be a list of commands
-func (obj *ExecRes) CheckApply(apply bool) (checkok bool, err error) {
+func (obj *ExecRes) CheckApply(apply bool) (checkOK bool, err error) {
-	log.Printf("%v[%v]: CheckApply(%t)", obj.Kind(), obj.GetName(), apply)
 
 	// if there is a watch command, but no if command, run based on state
 	if obj.WatchCmd != "" && obj.IfCmd == "" {
-		if obj.isStateOK {
+		if obj.IsStateOK() { // FIXME: this is done by engine now...
 			return true, nil
 		}
 
@@ -236,8 +219,8 @@ func (obj *ExecRes) CheckApply(apply bool) (checkok bool, err error) {
 		//} else if obj.IfCmd != "" && obj.WatchCmd != "" {
 
 		if obj.PollInt > 0 { // && obj.WatchCmd == ""
-			// XXX have the Watch() command output onlyif poll events...
+			// XXX: have the Watch() command output onlyif poll events...
-			// XXX we can optimize by saving those results for returning here
+			// XXX: we can optimize by saving those results for returning here
 			// return XXX
 		}
 
@@ -264,7 +247,7 @@ func (obj *ExecRes) CheckApply(apply bool) (checkok bool, err error) {
 		// if there is no watcher and no onlyif check, assume we should run
 	} else { // if obj.WatchCmd == "" && obj.IfCmd == "" {
 		// just run if state is dirty
-		if obj.isStateOK {
+		if obj.IsStateOK() { // FIXME: this is done by engine now...
 			return true, nil
 		}
 	}
@@ -275,7 +258,7 @@ func (obj *ExecRes) CheckApply(apply bool) (checkok bool, err error) {
 	}
 
 	// apply portion
-	log.Printf("%v[%v]: Apply", obj.Kind(), obj.GetName())
+	log.Printf("%s[%s]: Apply", obj.Kind(), obj.GetName())
 	var cmdName string
 	var cmdArgs []string
 	if obj.Shell == "" {
@@ -296,9 +279,8 @@ func (obj *ExecRes) CheckApply(apply bool) (checkok bool, err error) {
 	var out bytes.Buffer
 	cmd.Stdout = &out
 
-	if err = cmd.Start(); err != nil {
+	if err := cmd.Start(); err != nil {
-		log.Printf("%v[%v]: Error starting Cmd: %v", obj.Kind(), obj.GetName(), err)
+		return false, errwrap.Wrapf(err, "Error starting Cmd")
-		return false, err
 	}
 
 	timeout := obj.Timeout
@@ -309,48 +291,48 @@ func (obj *ExecRes) CheckApply(apply bool) (checkok bool, err error) {
 	go func() { done <- cmd.Wait() }()
 
 	select {
-	case err = <-done:
+	case err := <-done:
 		if err != nil {
-			log.Printf("%v[%v]: Error waiting for Cmd: %v", obj.Kind(), obj.GetName(), err)
+			e := errwrap.Wrapf(err, "Error waiting for Cmd")
-			return false, err
+			return false, e
 		}
 
 	case <-util.TimeAfterOrBlock(timeout):
-		log.Printf("%v[%v]: Timeout waiting for Cmd", obj.Kind(), obj.GetName())
 		//cmd.Process.Kill() // TODO: is this necessary?
-		return false, errors.New("Timeout waiting for Cmd!")
+		return false, fmt.Errorf("Timeout waiting for Cmd!")
 	}
 
 	// TODO: if we printed the stdout while the command is running, this
 	// would be nice, but it would require terminal log output that doesn't
 	// interleave all the parallel parts which would mix it all up...
 	if s := out.String(); s == "" {
-		log.Printf("Exec[%v]: Command output is empty!", obj.Name)
+		log.Printf("%s[%s]: Command output is empty!", obj.Kind(), obj.GetName())
+
 	} else {
-		log.Printf("Exec[%v]: Command output is:", obj.Name)
+		log.Printf("%s[%s]: Command output is:", obj.Kind(), obj.GetName())
 		log.Printf(out.String())
 	}
 	// XXX: return based on exit value!!
 
-	// the state tracking is for exec resources that can't "detect" their
+	// The state tracking is for exec resources that can't "detect" their
 	// state, and assume it's invalid when the Watch() function triggers.
-	// if we apply state successfully, we should reset it here so that we
+	// If we apply state successfully, we should reset it here so that we
 	// know that we have applied since the state was set not ok by event!
-	obj.isStateOK = true // reset
+	// This now happens automatically after the engine runs CheckApply().
 	return false, nil // success
 }
 
-// ExecUUID is the UUID struct for ExecRes.
+// ExecUID is the UID struct for ExecRes.
-type ExecUUID struct {
+type ExecUID struct {
-	BaseUUID
+	BaseUID
 	Cmd   string
 	IfCmd string
 	// TODO: add more elements here
 }
 
 // IFF aka if and only if they are equivalent, return true. If not, false.
-func (obj *ExecUUID) IFF(uuid ResUUID) bool {
+func (obj *ExecUID) IFF(uid ResUID) bool {
-	res, ok := uuid.(*ExecUUID)
+	res, ok := uid.(*ExecUID)
 	if !ok {
 		return false
 	}
@@ -389,16 +371,16 @@ func (obj *ExecRes) AutoEdges() AutoEdge {
 	return nil
 }
 
-// GetUUIDs includes all params to make a unique identification of this object.
+// GetUIDs includes all params to make a unique identification of this object.
 // Most resources only return one, although some resources can return multiple.
-func (obj *ExecRes) GetUUIDs() []ResUUID {
+func (obj *ExecRes) GetUIDs() []ResUID {
-	x := &ExecUUID{
+	x := &ExecUID{
-		BaseUUID: BaseUUID{name: obj.GetName(), kind: obj.Kind()},
+		BaseUID: BaseUID{name: obj.GetName(), kind: obj.Kind()},
 		Cmd:     obj.Cmd,
 		IfCmd:   obj.IfCmd,
 		// TODO: add more params here
 	}
-	return []ResUUID{x}
+	return []ResUID{x}
 }
 
 // GroupCmp returns whether two resources can be grouped together or not.

resources/file.go

@@ -30,12 +30,12 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/purpleidea/mgmt/event"
-	"github.com/purpleidea/mgmt/global" // XXX: package mgmtmain instead?
 	"github.com/purpleidea/mgmt/recwatch"
 	"github.com/purpleidea/mgmt/util"
+
+	errwrap "github.com/pkg/errors"
 )
 
 func init() {
@@ -48,7 +48,7 @@ type FileRes struct {
 	Path       string  `yaml:"path"` // path variable (should default to name)
 	Dirname    string  `yaml:"dirname"`
 	Basename   string  `yaml:"basename"`
-	Content    string `yaml:"content"` // FIXME: how do you describe: "leave content alone" - state = "create" ?
+	Content    *string `yaml:"content"` // nil to mark as undefined
 	Source     string  `yaml:"source"`  // file path for source content
 	State      string  `yaml:"state"`   // state: exists/present?, absent, (undefined?)
 	Recurse    bool    `yaml:"recurse"`
@@ -60,7 +60,7 @@ type FileRes struct {
 }
 
 // NewFileRes is a constructor for this resource. It also calls Init() for you.
-func NewFileRes(name, path, dirname, basename, content, source, state string, recurse, force bool) *FileRes {
+func NewFileRes(name, path, dirname, basename string, content *string, source, state string, recurse, force bool) (*FileRes, error) {
 	obj := &FileRes{
 		BaseRes: BaseRes{
 			Name: name,
@@ -74,8 +74,7 @@ func NewFileRes(name, path, dirname, basename, content, source, state string, re
 		Recurse:  recurse,
 		Force:    force,
 	}
-	obj.Init()
+	return obj, obj.Init()
-	return obj
 }
 
 // Init runs some startup code for this resource.
@@ -119,11 +118,11 @@ func (obj *FileRes) Validate() error {
 		return fmt.Errorf("Basename must not start with a slash.")
 	}
 
-	if obj.Content != "" && obj.Source != "" {
+	if obj.Content != nil && obj.Source != "" {
 		return fmt.Errorf("Can't specify both Content and Source.")
 	}
 
-	if obj.isDir && obj.Content != "" { // makes no sense
+	if obj.isDir && obj.Content != nil { // makes no sense
 		return fmt.Errorf("Can't specify Content when creating a Dir.")
 	}
 
@@ -142,22 +141,7 @@ func (obj *FileRes) Validate() error {
 // must be restarted. On a clean exit it returns nil.
 // FIXME: Also watch the source directory when using obj.Source !!!
 func (obj *FileRes) Watch(processChan chan event.Event) error {
-	if obj.IsWatching() {
+	cuid := obj.Converger() // get the converger uid used to report status
-		return nil // TODO: should this be an error?
-	}
-	obj.SetWatching(true)
-	defer obj.SetWatching(false)
-	cuuid := obj.converger.Register()
-	defer cuuid.Unregister()
-
-	var startup bool
-	Startup := func(block bool) <-chan time.Time {
-		if block {
-			return nil // blocks forever
-			//return make(chan time.Time) // blocks forever
-		}
-		return time.After(time.Duration(500) * time.Millisecond) // 1/2 the resolution of converged timeout
-	}
 
 	var err error
 	obj.recWatcher, err = recwatch.NewRecWatcher(obj.Path, obj.Recurse)
@@ -166,12 +150,16 @@ func (obj *FileRes) Watch(processChan chan event.Event) error {
 	}
 	defer obj.recWatcher.Close()
 
+	// notify engine that we're running
+	if err := obj.Running(processChan); err != nil {
+		return err // bubble up a NACK...
+	}
+
 	var send = false // send event?
 	var exit = false
-	var dirty = false
 
 	for {
-		if global.DEBUG {
+		if obj.debug {
 			log.Printf("%s[%s]: Watching: %s", obj.Kind(), obj.GetName(), obj.Path) // attempting to watch...
 		}
 
@@ -181,42 +169,31 @@ func (obj *FileRes) Watch(processChan chan event.Event) error {
 			if !ok { // channel shutdown
 				return nil
 			}
-			cuuid.SetConverged(false)
+			cuid.SetConverged(false)
 			if err := event.Error; err != nil {
-				return fmt.Errorf("Unknown %s[%s] watcher error: %v", obj.Kind(), obj.GetName(), err)
+				return errwrap.Wrapf(err, "Unknown %s[%s] watcher error", obj.Kind(), obj.GetName())
 			}
-			if global.DEBUG { // don't access event.Body if event.Error isn't nil
+			if obj.debug { // don't access event.Body if event.Error isn't nil
 				log.Printf("%s[%s]: Event(%s): %v", obj.Kind(), obj.GetName(), event.Body.Name, event.Body.Op)
 			}
 			send = true
-			dirty = true
+			obj.StateOK(false) // dirty
 
-		case event := <-obj.events:
+		case event := <-obj.Events():
-			cuuid.SetConverged(false)
+			cuid.SetConverged(false)
 			if exit, send = obj.ReadEvent(&event); exit {
 				return nil // exit
 			}
-			//dirty = false // these events don't invalidate state
+			//obj.StateOK(false) // dirty // these events don't invalidate state
 
-		case <-cuuid.ConvergedTimer():
+		case <-cuid.ConvergedTimer():
-			cuuid.SetConverged(true) // converged!
+			cuid.SetConverged(true) // converged!
 			continue
-
-		case <-Startup(startup):
-			cuuid.SetConverged(false)
-			send = true
-			dirty = true
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
-			startup = true // startup finished
 			send = false
-			// only invalid state on certain types of events
-			if dirty {
-				dirty = false
-				obj.isStateOK = false // something made state dirty
-			}
 			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
 				return err // we exit or bubble up a NACK...
 			}
@@ -259,7 +236,7 @@ func ReadDir(path string) ([]FileInfo, error) {
 		abs := path + smartPath(fi)
 		rel, err := filepath.Rel(path, abs) // NOTE: calls Clean()
 		if err != nil {                     // shouldn't happen
-			return nil, fmt.Errorf("ReadDir: Unhandled error: %v", err)
+			return nil, errwrap.Wrapf(err, "ReadDir: Unhandled error")
 		}
 		if fi.IsDir() {
 			rel += "/" // add a trailing slash for dirs
@@ -294,7 +271,7 @@ func mapPaths(fileInfos []FileInfo) map[string]FileInfo {
 func (obj *FileRes) fileCheckApply(apply bool, src io.ReadSeeker, dst string, sha256sum string) (string, bool, error) {
 	// TODO: does it make sense to switch dst to an io.Writer ?
 	// TODO: use obj.Force when dealing with symlinks and other file types!
-	if global.DEBUG {
+	if obj.debug {
 		log.Printf("fileCheckApply: %s -> %s", src, dst)
 	}
 
@@ -365,7 +342,7 @@ func (obj *FileRes) fileCheckApply(apply bool, src io.ReadSeeker, dst string, sh
 		// hash comparison (efficient because we can cache hash of content str)
 		if sha256sum == "" { // cache is invalid
 			hash := sha256.New()
-			// TODO file existence test?
+			// TODO: file existence test?
 			if _, err := io.Copy(hash, src); err != nil {
 				return "", false, err
 			}
@@ -391,7 +368,7 @@ func (obj *FileRes) fileCheckApply(apply bool, src io.ReadSeeker, dst string, sh
 	if !apply {
 		return sha256sum, false, nil
 	}
-	if global.DEBUG {
+	if obj.debug {
 		log.Printf("fileCheckApply: Apply: %s -> %s", src, dst)
 	}
 
@@ -412,12 +389,12 @@ func (obj *FileRes) fileCheckApply(apply bool, src io.ReadSeeker, dst string, sh
 	// syscall.Splice(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int64, err error)
 
 	// TODO: should we offer a way to cancel the copy on ^C ?
-	if global.DEBUG {
+	if obj.debug {
 		log.Printf("fileCheckApply: Copy: %s -> %s", src, dst)
 	}
 	if n, err := io.Copy(dstFile, src); err != nil {
 		return sha256sum, false, err
-	} else if global.DEBUG {
+	} else if obj.debug {
 		log.Printf("fileCheckApply: Copied: %v", n)
 	}
 	return sha256sum, false, dstFile.Sync()
@@ -427,7 +404,7 @@ func (obj *FileRes) fileCheckApply(apply bool, src io.ReadSeeker, dst string, sh
 // It is recursive and can create directories directly, and files via the usual
 // fileCheckApply method. It returns checkOK and error as is normally expected.
 func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
-	if global.DEBUG {
+	if obj.debug {
 		log.Printf("syncCheckApply: %s -> %s", src, dst)
 	}
 	if src == "" || dst == "" {
@@ -445,12 +422,12 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 	}
 
 	if !srcIsDir && !dstIsDir {
-		if global.DEBUG {
+		if obj.debug {
 			log.Printf("syncCheckApply: %s -> %s", src, dst)
 		}
 		fin, err := os.Open(src)
 		if err != nil {
-			if global.DEBUG && os.IsNotExist(err) { // if we get passed an empty src
+			if obj.debug && os.IsNotExist(err) { // if we get passed an empty src
 				log.Printf("syncCheckApply: Missing src: %s", src)
 			}
 			return false, err
@@ -506,7 +483,7 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 					delete(smartDst, relPathFile) // rm from purge list
 				}
 
-				if global.DEBUG {
+				if obj.debug {
 					log.Printf("syncCheckApply: mkdir -m %s '%s'", fileInfo.Mode(), absDst)
 				}
 				if err := os.Mkdir(absDst, fileInfo.Mode()); err != nil {
@@ -517,12 +494,12 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 			// if we're a regular file, the recurse will create it
 		}
 
-		if global.DEBUG {
+		if obj.debug {
 			log.Printf("syncCheckApply: Recurse: %s -> %s", absSrc, absDst)
 		}
 		if obj.Recurse {
 			if c, err := obj.syncCheckApply(apply, absSrc, absDst); err != nil { // recurse
-				return false, fmt.Errorf("syncCheckApply: Recurse failed: %v", err)
+				return false, errwrap.Wrapf(err, "syncCheckApply: Recurse failed")
 			} else if !c { // don't let subsequent passes make this true
 				checkOK = false
 			}
@@ -563,7 +540,7 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 		_ = absSrc
 		//log.Printf("syncCheckApply: Recurse rm: %s -> %s", absSrc, absDst)
 		//if c, err := obj.syncCheckApply(apply, absSrc, absDst); err != nil {
-		//	return false, fmt.Errorf("syncCheckApply: Recurse rm failed: %v", err)
+		//	return false, errwrap.Wrapf(err, "syncCheckApply: Recurse rm failed")
 		//} else if !c { // don't let subsequent passes make this true
 		//	checkOK = false
 		//}
@@ -581,7 +558,7 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 
 // contentCheckApply performs a CheckApply for the file existence and content.
 func (obj *FileRes) contentCheckApply(apply bool) (checkOK bool, _ error) {
-	log.Printf("%v[%v]: contentCheckApply(%t)", obj.Kind(), obj.GetName(), apply)
+	log.Printf("%s[%s]: contentCheckApply(%t)", obj.Kind(), obj.GetName(), apply)
 
 	if obj.State == "absent" {
 		if _, err := os.Stat(obj.path); os.IsNotExist(err) {
@@ -609,12 +586,17 @@ func (obj *FileRes) contentCheckApply(apply bool) (checkOK bool, _ error) {
 		return false, err             // either nil or not
 	}
 
+	// content is not defined, leave it alone...
+	if obj.Content == nil {
+		return true, nil
+	}
+
 	if obj.Source == "" { // do the obj.Content checks first...
 		if obj.isDir { // TODO: should we create an empty dir this way?
 			log.Fatal("XXX: Not implemented!") // XXX
 		}
 
-		bufferSrc := bytes.NewReader([]byte(obj.Content))
+		bufferSrc := bytes.NewReader([]byte(*obj.Content))
 		sha256sum, checkOK, err := obj.fileCheckApply(apply, bufferSrc, obj.path, obj.sha256sum)
 		if sha256sum != "" { // empty values mean errored or didn't hash
 			// this can be valid even when the whole function errors
@@ -639,10 +621,17 @@ func (obj *FileRes) contentCheckApply(apply bool) (checkOK bool, _ error) {
 // CheckApply checks the resource state and applies the resource if the bool
 // input is true. It returns error info and if the state check passed or not.
 func (obj *FileRes) CheckApply(apply bool) (checkOK bool, _ error) {
-	log.Printf("%v[%v]: CheckApply(%t)", obj.Kind(), obj.GetName(), apply)
 
-	if obj.isStateOK { // cache the state
+	// NOTE: all send/recv change notifications *must* be processed before
-		return true, nil
+	// there is a possibility of failure in CheckApply. This is because if
+	// we fail (and possibly run again) the subsequent send->recv transfer
+	// might not have a new value to copy, and therefore we won't see this
+	// notification of change. Therefore, it is important to process these
+	// promptly, if they must not be lost, such as for cache invalidation.
+	if val, exists := obj.Recv["Content"]; exists && val.Changed {
+		// if we received on Content, and it changed, invalidate the cache!
+		log.Printf("contentCheckApply: Invalidating sha256sum of `Content`")
+		obj.sha256sum = "" // invalidate!!
 	}
 
 	checkOK = true
@@ -667,22 +656,18 @@ func (obj *FileRes) CheckApply(apply bool) (checkOK bool, _ error) {
 	//	checkOK = false
 	//}
 
-	// if we did work successfully, or are in a good state, then state is ok
-	if apply || checkOK {
-		obj.isStateOK = true
-	}
 	return checkOK, nil // w00t
 }
 
-// FileUUID is the UUID struct for FileRes.
+// FileUID is the UID struct for FileRes.
-type FileUUID struct {
+type FileUID struct {
-	BaseUUID
+	BaseUID
 	path string
 }
 
 // IFF aka if and only if they are equivalent, return true. If not, false.
-func (obj *FileUUID) IFF(uuid ResUUID) bool {
+func (obj *FileUID) IFF(uid ResUID) bool {
-	res, ok := uuid.(*FileUUID)
+	res, ok := uid.(*FileUID)
 	if !ok {
 		return false
 	}
@@ -691,13 +676,13 @@ func (obj *FileUUID) IFF(uuid ResUUID) bool {
 
 // FileResAutoEdges holds the state of the auto edge generator.
 type FileResAutoEdges struct {
-	data    []ResUUID
+	data    []ResUID
 	pointer int
 	found   bool
 }
 
 // Next returns the next automatic edge.
-func (obj *FileResAutoEdges) Next() []ResUUID {
+func (obj *FileResAutoEdges) Next() []ResUID {
 	if obj.found {
 		log.Fatal("Shouldn't be called anymore!")
 	}
@@ -706,7 +691,7 @@ func (obj *FileResAutoEdges) Next() []ResUUID {
 	}
 	value := obj.data[obj.pointer]
 	obj.pointer++
-	return []ResUUID{value} // we return one, even though api supports N
+	return []ResUID{value} // we return one, even though api supports N
 }
 
 // Test gets results of the earlier Next() call, & returns if we should continue!
@@ -731,13 +716,13 @@ func (obj *FileResAutoEdges) Test(input []bool) bool {
 // AutoEdges generates a simple linear sequence of each parent directory from
 // the bottom up!
 func (obj *FileRes) AutoEdges() AutoEdge {
-	var data []ResUUID                             // store linear result chain here...
+	var data []ResUID                              // store linear result chain here...
 	values := util.PathSplitFullReversed(obj.path) // build it
 	_, values = values[0], values[1:]              // get rid of first value which is me!
 	for _, x := range values {
 		var reversed = true // cheat by passing a pointer
-		data = append(data, &FileUUID{
+		data = append(data, &FileUID{
-			BaseUUID: BaseUUID{
+			BaseUID: BaseUID{
 				name:     obj.GetName(),
 				kind:     obj.Kind(),
 				reversed: &reversed,
@@ -752,14 +737,14 @@ func (obj *FileRes) AutoEdges() AutoEdge {
 	}
 }
 
-// GetUUIDs includes all params to make a unique identification of this object.
+// GetUIDs includes all params to make a unique identification of this object.
 // Most resources only return one, although some resources can return multiple.
-func (obj *FileRes) GetUUIDs() []ResUUID {
+func (obj *FileRes) GetUIDs() []ResUID {
-	x := &FileUUID{
+	x := &FileUID{
-		BaseUUID: BaseUUID{name: obj.GetName(), kind: obj.Kind()},
+		BaseUID: BaseUID{name: obj.GetName(), kind: obj.Kind()},
 		path:    obj.path,
 	}
-	return []ResUUID{x}
+	return []ResUID{x}
 }
 
 // GroupCmp returns whether two resources can be grouped together or not.
@@ -785,12 +770,17 @@ func (obj *FileRes) Compare(res Res) bool {
 		if obj.Name != res.Name {
 			return false
 		}
-		if obj.path != res.Path {
+		if obj.path != res.path {
 			return false
 		}
-		if obj.Content != res.Content {
+		if (obj.Content == nil) != (res.Content == nil) { // xor
 			return false
 		}
+		if obj.Content != nil && res.Content != nil {
+			if *obj.Content != *res.Content { // compare the strings
+				return false
+			}
+		}
 		if obj.Source != res.Source {
 			return false
 		}

resources/hostname.go

@@ -0,0 +1,295 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"encoding/gob"
+	"errors"
+	"fmt"
+	"log"
+
+	"github.com/purpleidea/mgmt/event"
+	"github.com/purpleidea/mgmt/util"
+
+	"github.com/godbus/dbus"
+	errwrap "github.com/pkg/errors"
+)
+
+// ErrResourceInsufficientParameters is returned when the configuration of the resource
+// is insufficient for the resource to do any useful work.
+var ErrResourceInsufficientParameters = errors.New(
+	"Insufficient parameters for this resource")
+
+func init() {
+	gob.Register(&HostnameRes{})
+}
+
+const (
+	hostname1Path  = "/org/freedesktop/hostname1"
+	hostname1Iface = "org.freedesktop.hostname1"
+	dbusAddMatch   = "org.freedesktop.DBus.AddMatch"
+)
+
+// HostnameRes is a resource that allows setting and watching the hostname.
+//
+// StaticHostname is the one configured in /etc/hostname or a similar file.
+// It is chosen by the local user. It is not always in sync with the current
+// host name as returned by the gethostname() system call.
+//
+// TransientHostname is the one configured via the kernel's sethostbyname().
+// It can be different from the static hostname in case DHCP or mDNS have been
+// configured to change the name based on network information.
+//
+// PrettyHostname is a free-form UTF8 host name for presentation to the user.
+//
+// Hostname is the fallback value for all 3 fields above, if only Hostname is
+// specified, it will set all 3 fields to this value.
+type HostnameRes struct {
+	BaseRes           `yaml:",inline"`
+	Hostname          string `yaml:"hostname"`
+	PrettyHostname    string `yaml:"pretty_hostname"`
+	StaticHostname    string `yaml:"static_hostname"`
+	TransientHostname string `yaml:"transient_hostname"`
+
+	conn *dbus.Conn
+}
+
+// NewHostnameRes is a constructor for this resource. It also calls Init() for you.
+func NewHostnameRes(name, staticHostname, transientHostname, prettyHostname string) (*HostnameRes, error) {
+	obj := &HostnameRes{
+		BaseRes: BaseRes{
+			Name: name,
+		},
+		PrettyHostname:    prettyHostname,
+		StaticHostname:    staticHostname,
+		TransientHostname: transientHostname,
+	}
+	return obj, obj.Init()
+}
+
+// Init runs some startup code for this resource.
+func (obj *HostnameRes) Init() error {
+	obj.BaseRes.kind = "Hostname"
+	if obj.PrettyHostname == "" {
+		obj.PrettyHostname = obj.Hostname
+	}
+	if obj.StaticHostname == "" {
+		obj.StaticHostname = obj.Hostname
+	}
+	if obj.TransientHostname == "" {
+		obj.TransientHostname = obj.Hostname
+	}
+	return obj.BaseRes.Init() // call base init, b/c we're overriding
+}
+
+// Validate if the params passed in are valid data.
+// FIXME: where should this get called ?
+func (obj *HostnameRes) Validate() error {
+	if obj.PrettyHostname == "" && obj.StaticHostname == "" && obj.TransientHostname == "" {
+		return ErrResourceInsufficientParameters
+	}
+	return nil
+}
+
+// Watch is the primary listener for this resource and it outputs events.
+func (obj *HostnameRes) Watch(processChan chan event.Event) error {
+	cuid := obj.Converger() // get the converger uid used to report status
+
+	// if we share the bus with others, we will get each others messages!!
+	bus, err := util.SystemBusPrivateUsable() // don't share the bus connection!
+	if err != nil {
+		return errwrap.Wrap(err, "Failed to connect to bus")
+	}
+	defer bus.Close()
+	callResult := bus.BusObject().Call(
+		"org.freedesktop.DBus.AddMatch", 0,
+		fmt.Sprintf("type='signal',path='%s',interface='org.freedesktop.DBus.Properties',member='PropertiesChanged'", hostname1Path))
+	if callResult.Err != nil {
+		return errwrap.Wrap(callResult.Err, "Failed to subscribe to DBus events for hostname1")
+	}
+
+	signals := make(chan *dbus.Signal, 10) // closed by dbus package
+	bus.Signal(signals)
+
+	// notify engine that we're running
+	if err := obj.Running(processChan); err != nil {
+		return err // bubble up a NACK...
+	}
+
+	var send = false // send event?
+
+	for {
+		obj.SetState(ResStateWatching) // reset
+		select {
+		case <-signals:
+			cuid.SetConverged(false)
+			send = true
+			obj.StateOK(false) // dirty
+
+		case event := <-obj.Events():
+			cuid.SetConverged(false)
+			// we avoid sending events on unpause
+			if exit, _ := obj.ReadEvent(&event); exit {
+				return nil // exit
+			}
+			send = true
+			obj.StateOK(false) // dirty
+
+		case <-cuid.ConvergedTimer():
+			cuid.SetConverged(true) // converged!
+			continue
+		}
+
+		// do all our event sending all together to avoid duplicate msgs
+		if send {
+			send = false
+
+			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
+				return err // we exit or bubble up a NACK...
+			}
+		}
+	}
+}
+
+func updateHostnameProperty(object dbus.BusObject, expectedValue, property, setterName string, apply bool) (checkOK bool, err error) {
+	propertyObject, err := object.GetProperty("org.freedesktop.hostname1." + property)
+	if err != nil {
+		return false, errwrap.Wrapf(err, "failed to get org.freedesktop.hostname1.%s", property)
+	}
+	if propertyObject.Value() == nil {
+		return false, errwrap.Errorf("Unexpected nil value received when reading property %s", property)
+	}
+
+	propertyValue, ok := propertyObject.Value().(string)
+	if !ok {
+		return false, fmt.Errorf("Received unexpected type as %s value, expected string got '%T'", property, propertyValue)
+	}
+
+	// expected value and actual value match => checkOk
+	if propertyValue == expectedValue {
+		return true, nil
+	}
+
+	// nothing to do anymore
+	if !apply {
+		return false, nil
+	}
+
+	// attempting to apply the changes
+	log.Printf("Changing %s: %s => %s", property, propertyValue, expectedValue)
+	if err := object.Call("org.freedesktop.hostname1."+setterName, 0, expectedValue, false).Err; err != nil {
+		return false, errwrap.Wrapf(err, "failed to call org.freedesktop.hostname1.%s", setterName)
+	}
+
+	// all good changes should now be applied again
+	return false, nil
+}
+
+// CheckApply method for Hostname resource.
+func (obj *HostnameRes) CheckApply(apply bool) (checkOK bool, err error) {
+	conn, err := util.SystemBusPrivateUsable()
+	if err != nil {
+		return false, errwrap.Wrap(err, "Failed to connect to the private system bus")
+	}
+	defer conn.Close()
+
+	hostnameObject := conn.Object(hostname1Iface, hostname1Path)
+
+	checkOK = true
+	if obj.PrettyHostname != "" {
+		propertyCheckOK, err := updateHostnameProperty(hostnameObject, obj.PrettyHostname, "PrettyHostname", "SetPrettyHostname", apply)
+		if err != nil {
+			return false, err
+		}
+		checkOK = checkOK && propertyCheckOK
+	}
+	if obj.StaticHostname != "" {
+		propertyCheckOK, err := updateHostnameProperty(hostnameObject, obj.StaticHostname, "StaticHostname", "SetStaticHostname", apply)
+		if err != nil {
+			return false, err
+		}
+		checkOK = checkOK && propertyCheckOK
+	}
+	if obj.TransientHostname != "" {
+		propertyCheckOK, err := updateHostnameProperty(hostnameObject, obj.TransientHostname, "Hostname", "SetHostname", apply)
+		if err != nil {
+			return false, err
+		}
+		checkOK = checkOK && propertyCheckOK
+	}
+
+	return checkOK, nil
+}
+
+// HostnameUID is the UID struct for HostnameRes.
+type HostnameUID struct {
+	BaseUID
+	name              string
+	prettyHostname    string
+	staticHostname    string
+	transientHostname string
+}
+
+// AutoEdges returns the AutoEdge interface. In this case no autoedges are used.
+func (obj *HostnameRes) AutoEdges() AutoEdge {
+	return nil
+}
+
+// GetUIDs includes all params to make a unique identification of this object.
+// Most resources only return one, although some resources can return multiple.
+func (obj *HostnameRes) GetUIDs() []ResUID {
+	x := &HostnameUID{
+		BaseUID:           BaseUID{name: obj.GetName(), kind: obj.Kind()},
+		name:              obj.Name,
+		prettyHostname:    obj.PrettyHostname,
+		staticHostname:    obj.StaticHostname,
+		transientHostname: obj.TransientHostname,
+	}
+	return []ResUID{x}
+}
+
+// GroupCmp returns whether two resources can be grouped together or not.
+func (obj *HostnameRes) GroupCmp(r Res) bool {
+	return false
+}
+
+// Compare two resources and return if they are equivalent.
+func (obj *HostnameRes) Compare(res Res) bool {
+	switch res := res.(type) {
+	// we can only compare HostnameRes to others of the same resource
+	case *HostnameRes:
+		if !obj.BaseRes.Compare(res) { // call base Compare
+			return false
+		}
+		if obj.Name != res.Name {
+			return false
+		}
+		if obj.PrettyHostname != res.PrettyHostname {
+			return false
+		}
+		if obj.StaticHostname != res.StaticHostname {
+			return false
+		}
+		if obj.TransientHostname != res.TransientHostname {
+			return false
+		}
+	default:
+		return false
+	}
+	return true
+}

resources/msg.go

@@ -23,7 +23,6 @@ import (
 	"log"
 	"regexp"
 	"strings"
-	"time"
 
 	"github.com/purpleidea/mgmt/event"
 
@@ -47,14 +46,14 @@ type MsgRes struct {
 	syslogStateOK  bool
 }
 
-// MsgUUID is a unique representation for a MsgRes object.
+// MsgUID is a unique representation for a MsgRes object.
-type MsgUUID struct {
+type MsgUID struct {
-	BaseUUID
+	BaseUID
 	body string
 }
 
 // NewMsgRes is a constructor for this resource.
-func NewMsgRes(name, body, priority string, journal, syslog bool, fields map[string]string) *MsgRes {
+func NewMsgRes(name, body, priority string, journal, syslog bool, fields map[string]string) (*MsgRes, error) {
 	message := name
 	if body != "" {
 		message = body
@@ -71,8 +70,7 @@ func NewMsgRes(name, body, priority string, journal, syslog bool, fields map[str
 		Syslog:   syslog,
 	}
 
-	obj.Init()
+	return obj, obj.Init()
-	return obj
 }
 
 // Init runs some startup code for this resource.
@@ -95,23 +93,53 @@ func (obj *MsgRes) Validate() error {
 	return nil
 }
 
+// isAllStateOK derives a compound state from all internal cache flags that apply to this resource.
+func (obj *MsgRes) isAllStateOK() bool {
+	if obj.Journal && !obj.journalStateOK {
+		return false
+	}
+	if obj.Syslog && !obj.syslogStateOK {
+		return false
+	}
+	return obj.logStateOK
+}
+
+// updateStateOK sets the global state so it can be read by the engine.
+func (obj *MsgRes) updateStateOK() {
+	obj.StateOK(obj.isAllStateOK())
+}
+
+// JournalPriority converts a string description to a numeric priority.
+// XXX: Have Validate() make sure it actually is one of these.
+func (obj *MsgRes) journalPriority() journal.Priority {
+	switch obj.Priority {
+	case "Emerg":
+		return journal.PriEmerg
+	case "Alert":
+		return journal.PriAlert
+	case "Crit":
+		return journal.PriCrit
+	case "Err":
+		return journal.PriErr
+	case "Warning":
+		return journal.PriWarning
+	case "Notice":
+		return journal.PriNotice
+	case "Info":
+		return journal.PriInfo
+	case "Debug":
+		return journal.PriDebug
+	}
+	return journal.PriNotice
+}
+
 // Watch is the primary listener for this resource and it outputs events.
 func (obj *MsgRes) Watch(processChan chan event.Event) error {
-	if obj.IsWatching() {
+	cuid := obj.Converger() // get the converger uid used to report status
-		return nil
-	}
-	obj.SetWatching(true)
-	defer obj.SetWatching(false)
-	cuuid := obj.converger.Register()
-	defer cuuid.Unregister()
 
-	var startup bool
+	// notify engine that we're running
-	Startup := func(block bool) <-chan time.Time {
+	if err := obj.Running(processChan); err != nil {
-		if block {
+		return err // bubble up a NACK...
-			return nil // blocks forever
-			//return make(chan time.Time) // blocks forever
-		}
-		return time.After(time.Duration(500) * time.Millisecond) // 1/2 the resolution of converged timeout
 	}
 
 	var send = false // send event?
@@ -119,40 +147,21 @@ func (obj *MsgRes) Watch(processChan chan event.Event) error {
 	for {
 		obj.SetState(ResStateWatching) // reset
 		select {
-		case event := <-obj.events:
+		case event := <-obj.Events():
-			cuuid.SetConverged(false)
+			cuid.SetConverged(false)
 			// we avoid sending events on unpause
 			if exit, send = obj.ReadEvent(&event); exit {
 				return nil // exit
 			}
 
-			/*
+		case <-cuid.ConvergedTimer():
-				// TODO: invalidate cached state on poke events
+			cuid.SetConverged(true) // converged!
-				obj.logStateOK = false
-				if obj.Journal {
-					obj.journalStateOK = false
-				}
-				if obj.Syslog {
-					obj.syslogStateOK = false
-				}
-			*/
-			send = true
-
-		case <-cuuid.ConvergedTimer():
-			cuuid.SetConverged(true) // converged!
 			continue
-
-		case <-Startup(startup):
-			cuuid.SetConverged(false)
-			send = true
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
-			startup = true // startup finished
 			send = false
-			// only do this on certain types of events
-			//obj.isStateOK = false // something made state dirty
 			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
 				return err // we exit or bubble up a NACK...
 			}
@@ -160,17 +169,62 @@ func (obj *MsgRes) Watch(processChan chan event.Event) error {
 	}
 }
 
-// GetUUIDs includes all params to make a unique identification of this object.
+// CheckApply method for Msg resource.
+// Every check leads to an apply, meaning that the message is flushed to the journal.
+func (obj *MsgRes) CheckApply(apply bool) (bool, error) {
+
+	// isStateOK() done by engine, so we updateStateOK() to pass in value
+	//if obj.isAllStateOK() {
+	//	return true, nil
+	//}
+
+	if obj.Refresh() { // if we were notified...
+		// invalidate cached state...
+		obj.logStateOK = false
+		if obj.Journal {
+			obj.journalStateOK = false
+		}
+		if obj.Syslog {
+			obj.syslogStateOK = false
+		}
+		obj.updateStateOK()
+	}
+
+	if !obj.logStateOK {
+		log.Printf("%s[%s]: Body: %s", obj.Kind(), obj.GetName(), obj.Body)
+		obj.logStateOK = true
+		obj.updateStateOK()
+	}
+
+	if !apply {
+		return false, nil
+	}
+	if obj.Journal && !obj.journalStateOK {
+		if err := journal.Send(obj.Body, obj.journalPriority(), obj.Fields); err != nil {
+			return false, err
+		}
+		obj.journalStateOK = true
+		obj.updateStateOK()
+	}
+	if obj.Syslog && !obj.syslogStateOK {
+		// TODO: implement syslog client
+		obj.syslogStateOK = true
+		obj.updateStateOK()
+	}
+	return false, nil
+}
+
+// GetUIDs includes all params to make a unique identification of this object.
 // Most resources only return one, although some resources can return multiple.
-func (obj *MsgRes) GetUUIDs() []ResUUID {
+func (obj *MsgRes) GetUIDs() []ResUID {
-	x := &MsgUUID{
+	x := &MsgUID{
-		BaseUUID: BaseUUID{
+		BaseUID: BaseUID{
 			name: obj.GetName(),
 			kind: obj.Kind(),
 		},
 		body: obj.Body,
 	}
-	return []ResUUID{x}
+	return []ResUID{x}
 }
 
 // AutoEdges returns the AutoEdges. In this case none are used.
@@ -205,68 +259,3 @@ func (obj *MsgRes) Compare(res Res) bool {
 	}
 	return true
 }
-
-// IsAllStateOK derives a compound state from all internal cache flags that apply to this resource.
-func (obj *MsgRes) isAllStateOK() bool {
-	if obj.Journal && !obj.journalStateOK {
-		return false
-	}
-	if obj.Syslog && !obj.syslogStateOK {
-		return false
-	}
-	return obj.logStateOK
-}
-
-// JournalPriority converts a string description to a numeric priority.
-// XXX Have Validate() make sure it actually is one of these.
-func (obj *MsgRes) journalPriority() journal.Priority {
-	switch obj.Priority {
-	case "Emerg":
-		return journal.PriEmerg
-	case "Alert":
-		return journal.PriAlert
-	case "Crit":
-		return journal.PriCrit
-	case "Err":
-		return journal.PriErr
-	case "Warning":
-		return journal.PriWarning
-	case "Notice":
-		return journal.PriNotice
-	case "Info":
-		return journal.PriInfo
-	case "Debug":
-		return journal.PriDebug
-	}
-	return journal.PriNotice
-}
-
-// CheckApply method for Msg resource.
-// Every check leads to an apply, meaning that the message is flushed to the journal.
-func (obj *MsgRes) CheckApply(apply bool) (bool, error) {
-	log.Printf("%s[%s]: CheckApply(%t)", obj.Kind(), obj.GetName(), apply)
-
-	if obj.isAllStateOK() {
-		return true, nil
-	}
-
-	if !obj.logStateOK {
-		log.Printf("%s[%s]: Body: %s", obj.Kind(), obj.GetName(), obj.Body)
-		obj.logStateOK = true
-	}
-
-	if !apply {
-		return false, nil
-	}
-	if obj.Journal && !obj.journalStateOK {
-		if err := journal.Send(obj.Body, obj.journalPriority(), obj.Fields); err != nil {
-			return false, err
-		}
-		obj.journalStateOK = true
-	}
-	if obj.Syslog && !obj.syslogStateOK {
-		// TODO: implement syslog client
-		obj.syslogStateOK = true
-	}
-	return false, nil
-}

resources/noop.go

@@ -20,7 +20,6 @@ package resources
 import (
 	"encoding/gob"
 	"log"
-	"time"
 
 	"github.com/purpleidea/mgmt/event"
 )
@@ -36,15 +35,14 @@ type NoopRes struct {
 }
 
 // NewNoopRes is a constructor for this resource. It also calls Init() for you.
-func NewNoopRes(name string) *NoopRes {
+func NewNoopRes(name string) (*NoopRes, error) {
 	obj := &NoopRes{
 		BaseRes: BaseRes{
 			Name: name,
 		},
 		Comment: "",
 	}
-	obj.Init()
+	return obj, obj.Init()
-	return obj
 }
 
 // Init runs some startup code for this resource.
@@ -61,21 +59,11 @@ func (obj *NoopRes) Validate() error {
 
 // Watch is the primary listener for this resource and it outputs events.
 func (obj *NoopRes) Watch(processChan chan event.Event) error {
-	if obj.IsWatching() {
+	cuid := obj.Converger() // get the converger uid used to report status
-		return nil // TODO: should this be an error?
-	}
-	obj.SetWatching(true)
-	defer obj.SetWatching(false)
-	cuuid := obj.converger.Register()
-	defer cuuid.Unregister()
 
-	var startup bool
+	// notify engine that we're running
-	Startup := func(block bool) <-chan time.Time {
+	if err := obj.Running(processChan); err != nil {
-		if block {
+		return err // bubble up a NACK...
-			return nil // blocks forever
-			//return make(chan time.Time) // blocks forever
-		}
-		return time.After(time.Duration(500) * time.Millisecond) // 1/2 the resolution of converged timeout
 	}
 
 	var send = false // send event?
@@ -83,28 +71,21 @@ func (obj *NoopRes) Watch(processChan chan event.Event) error {
 	for {
 		obj.SetState(ResStateWatching) // reset
 		select {
-		case event := <-obj.events:
+		case event := <-obj.Events():
-			cuuid.SetConverged(false)
+			cuid.SetConverged(false)
 			// we avoid sending events on unpause
 			if exit, send = obj.ReadEvent(&event); exit {
 				return nil // exit
 			}
 
-		case <-cuuid.ConvergedTimer():
+		case <-cuid.ConvergedTimer():
-			cuuid.SetConverged(true) // converged!
+			cuid.SetConverged(true) // converged!
 			continue
-
-		case <-Startup(startup):
-			cuuid.SetConverged(false)
-			send = true
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
-			startup = true // startup finished
 			send = false
-			// only do this on certain types of events
-			//obj.isStateOK = false // something made state dirty
 			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
 				return err // we exit or bubble up a NACK...
 			}
@@ -113,14 +94,16 @@ func (obj *NoopRes) Watch(processChan chan event.Event) error {
 }
 
 // CheckApply method for Noop resource. Does nothing, returns happy!
-func (obj *NoopRes) CheckApply(apply bool) (checkok bool, err error) {
+func (obj *NoopRes) CheckApply(apply bool) (checkOK bool, err error) {
-	log.Printf("%v[%v]: CheckApply(%t)", obj.Kind(), obj.GetName(), apply)
+	if obj.Refresh() {
+		log.Printf("%s[%s]: Received a notification!", obj.Kind(), obj.GetName())
+	}
 	return true, nil // state is always okay
 }
 
-// NoopUUID is the UUID struct for NoopRes.
+// NoopUID is the UID struct for NoopRes.
-type NoopUUID struct {
+type NoopUID struct {
-	BaseUUID
+	BaseUID
 	name string
 }
 
@@ -129,14 +112,14 @@ func (obj *NoopRes) AutoEdges() AutoEdge {
 	return nil
 }
 
-// GetUUIDs includes all params to make a unique identification of this object.
+// GetUIDs includes all params to make a unique identification of this object.
 // Most resources only return one, although some resources can return multiple.
-func (obj *NoopRes) GetUUIDs() []ResUUID {
+func (obj *NoopRes) GetUIDs() []ResUID {
-	x := &NoopUUID{
+	x := &NoopUID{
-		BaseUUID: BaseUUID{name: obj.GetName(), kind: obj.Kind()},
+		BaseUID: BaseUID{name: obj.GetName(), kind: obj.Kind()},
 		name:    obj.Name,
 	}
-	return []ResUUID{x}
+	return []ResUID{x}
 }
 
 // GroupCmp returns whether two resources can be grouped together or not.

resources/nspawn.go

@@ -0,0 +1,306 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"encoding/gob"
+	"errors"
+	"fmt"
+	"log"
+
+	"github.com/purpleidea/mgmt/event"
+	"github.com/purpleidea/mgmt/util"
+
+	systemdUtil "github.com/coreos/go-systemd/util"
+	"github.com/godbus/dbus"
+	errwrap "github.com/pkg/errors"
+	machined "github.com/purpleidea/go-systemd/machine1"
+)
+
+const (
+	running           = "running"
+	stopped           = "stopped"
+	dbusInterface     = "org.freedesktop.machine1.Manager"
+	machineNew        = "org.freedesktop.machine1.Manager.MachineNew"
+	machineRemoved    = "org.freedesktop.machine1.Manager.MachineRemoved"
+	nspawnServiceTmpl = "systemd-nspawn@%s"
+)
+
+func init() {
+	gob.Register(&NspawnRes{})
+}
+
+// NspawnRes is an nspawn container resource
+type NspawnRes struct {
+	BaseRes `yaml:",inline"`
+	State   string `yaml:"state"`
+	// we're using the svc resource to start the machine because that's
+	// what machinectl does. We're not using svc.Watch because then we
+	// would have two watches potentially racing each other and producing
+	// potentially unexpected results. We get everything we need to
+	// monitor the machine state changes from the org.freedesktop.machine1 object.
+	svc *SvcRes
+}
+
+// Init runs some startup code for this resource
+func (obj *NspawnRes) Init() error {
+	var serviceName = fmt.Sprintf(nspawnServiceTmpl, obj.GetName())
+	obj.svc = &SvcRes{}
+	obj.svc.Name = serviceName
+	obj.svc.State = obj.State
+	if err := obj.svc.Init(); err != nil {
+		return err
+	}
+	obj.BaseRes.kind = "Nspawn"
+	return obj.BaseRes.Init()
+}
+
+// NewNspawnRes is the constructor for this resource
+func NewNspawnRes(name string, state string) (*NspawnRes, error) {
+	obj := &NspawnRes{
+		BaseRes: BaseRes{
+			Name: name,
+		},
+		State: state,
+	}
+	return obj, obj.Init()
+}
+
+// Validate params
+func (obj *NspawnRes) Validate() error {
+	validStates := map[string]struct{}{
+		stopped: {},
+		running: {},
+	}
+	if _, exists := validStates[obj.State]; !exists {
+		return fmt.Errorf("Invalid State: %s", obj.State)
+	}
+	return obj.svc.Validate()
+}
+
+// Watch for state changes and sends a message to the bus if there is a change
+func (obj *NspawnRes) Watch(processChan chan event.Event) error {
+	cuid := obj.Converger() // get the converger uid used to report status
+
+	// this resource depends on systemd ensure that it's running
+	if !systemdUtil.IsRunningSystemd() {
+		return fmt.Errorf("Systemd is not running.")
+	}
+
+	// create a private message bus
+	bus, err := util.SystemBusPrivateUsable()
+	if err != nil {
+		return errwrap.Wrapf(err, "Failed to connect to bus")
+	}
+
+	// add a match rule to match messages going through the message bus
+	call := bus.BusObject().Call("org.freedesktop.DBus.AddMatch", 0,
+		fmt.Sprintf("type='signal',interface='%s',eavesdrop='true'",
+			dbusInterface))
+	// <-call.Done
+	if err := call.Err; err != nil {
+		return err
+	}
+	buschan := make(chan *dbus.Signal, 10)
+	bus.Signal(buschan)
+
+	// notify engine that we're running
+	if err := obj.Running(processChan); err != nil {
+		return err // bubble up a NACK...
+	}
+
+	var send = false
+	var exit = false
+
+	for {
+		obj.SetState(ResStateWatching)
+		select {
+		case event := <-buschan:
+			// process org.freedesktop.machine1 events for this resource's name
+			if event.Body[0] == obj.GetName() {
+				log.Printf("%s[%s]: Event received: %v", obj.Kind(), obj.GetName(), event.Name)
+				if event.Name == machineNew {
+					log.Printf("%s[%s]: Machine started", obj.Kind(), obj.GetName())
+				} else if event.Name == machineRemoved {
+					log.Printf("%s[%s]: Machine stopped", obj.Kind(), obj.GetName())
+				} else {
+					return fmt.Errorf("Unknown event: %s", event.Name)
+				}
+				send = true
+				obj.StateOK(false) // dirty
+			}
+
+		case event := <-obj.Events():
+			cuid.SetConverged(false)
+			if exit, send = obj.ReadEvent(&event); exit {
+				return nil // exit
+			}
+
+		case <-cuid.ConvergedTimer():
+			cuid.SetConverged(true) // converged!
+			continue
+		}
+
+		// do all our event sending all together to avoid duplicate msgs
+		if send {
+			send = false
+			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
+				return err // we exit or bubble up a NACK...
+			}
+		}
+	}
+}
+
+// CheckApply is run to check the state and, if apply is true, to apply the
+// necessary changes to reach the desired state. this is run before Watch and
+// again if watch finds a change occurring to the state
+func (obj *NspawnRes) CheckApply(apply bool) (checkOK bool, err error) {
+	// this resource depends on systemd ensure that it's running
+	if !systemdUtil.IsRunningSystemd() {
+		return false, errors.New("Systemd is not running.")
+	}
+
+	// connect to org.freedesktop.machine1.Manager
+	conn, err := machined.New()
+	if err != nil {
+		return false, errwrap.Wrapf(err, "Failed to connect to dbus")
+	}
+
+	// compare the current state with the desired state and perform the
+	// appropriate action
+	var exists = true
+	properties, err := conn.GetProperties(obj.GetName())
+	if err != nil {
+		if err, ok := err.(dbus.Error); ok && err.Name !=
+			"org.freedesktop.machine1.NoSuchMachine" {
+			return false, err
+		}
+		exists = false
+		// if we could not successfully get the properties because
+		// there's no such machine the machine is stopped
+		// error if we need the image ignore if we don't
+		if _, err = conn.GetImage(obj.GetName()); err != nil && obj.State != stopped {
+			return false, fmt.Errorf(
+				"No machine nor image named '%s'",
+				obj.GetName())
+		}
+	}
+	if obj.debug {
+		log.Printf("%s[%s]: properties: %v", obj.Kind(), obj.GetName(), properties)
+	}
+	// if the machine doesn't exist and is supposed to
+	// be stopped or the state matches we're done
+	if !exists && obj.State == stopped || properties["State"] == obj.State {
+		if obj.debug {
+			log.Printf("%s[%s]: CheckApply() in valid state", obj.Kind(), obj.GetName())
+		}
+		return true, nil
+	}
+
+	// end of state checking. if we're here, checkOK is false
+	if !apply {
+		return false, nil
+	}
+
+	if obj.debug {
+		log.Printf("%s[%s]: CheckApply() applying '%s' state", obj.Kind(), obj.GetName(), obj.State)
+	}
+
+	if obj.State == running {
+		// start the machine using svc resource
+		log.Printf("%s[%s]: Starting machine", obj.Kind(), obj.GetName())
+		// assume state had to be changed at this point, ignore checkOK
+		if _, err := obj.svc.CheckApply(apply); err != nil {
+			return false, errwrap.Wrapf(err, "Nested svc failed")
+		}
+	}
+	if obj.State == stopped {
+		// terminate the machine with
+		// org.freedesktop.machine1.Manager.KillMachine
+		log.Printf("%s[%s]: Stopping machine", obj.Kind(), obj.GetName())
+		if err := conn.TerminateMachine(obj.GetName()); err != nil {
+			return false, errwrap.Wrapf(err, "Failed to stop machine")
+		}
+	}
+
+	return false, nil
+}
+
+// NspawnUID is a unique resource identifier
+type NspawnUID struct {
+	// NOTE: there is also a name variable in the BaseUID struct, this is
+	// information about where this UID came from, and is unrelated to the
+	// information about the resource we're matching. That data which is
+	// used in the IFF function, is what you see in the struct fields here
+	BaseUID
+	name string // the machine name
+}
+
+// IFF aka if and only if they are equivalent, return true. If not, false
+func (obj *NspawnUID) IFF(uid ResUID) bool {
+	res, ok := uid.(*NspawnUID)
+	if !ok {
+		return false
+	}
+	return obj.name == res.name
+}
+
+// GetUIDs includes all params to make a unique identification of this object
+// most resources only return one although some resources can return multiple
+func (obj *NspawnRes) GetUIDs() []ResUID {
+	x := &NspawnUID{
+		BaseUID: BaseUID{name: obj.GetName(), kind: obj.Kind()},
+		name:    obj.Name, // svc name
+	}
+	return append([]ResUID{x}, obj.svc.GetUIDs()...)
+}
+
+// GroupCmp returns whether two resources can be grouped together or not
+func (obj *NspawnRes) GroupCmp(r Res) bool {
+	_, ok := r.(*NspawnRes)
+	if !ok {
+		return false
+	}
+	// TODO: this would be quite useful for this resource!
+	return false
+}
+
+// Compare two resources and return if they are equivalent
+func (obj *NspawnRes) Compare(res Res) bool {
+	switch res.(type) {
+	case *NspawnRes:
+		res := res.(*NspawnRes)
+		if !obj.BaseRes.Compare(res) {
+			return false
+		}
+		if obj.Name != res.Name {
+			return false
+		}
+		if !obj.svc.Compare(res.svc) {
+			return false
+		}
+	default:
+		return false
+	}
+	return true
+}
+
+// AutoEdges returns the AutoEdge interface in this case no autoedges are used
+func (obj *NspawnRes) AutoEdges() AutoEdge {
+	return nil
+}

resources/packagekit/packagekit.go

@@ -15,8 +15,9 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-// DOCS: https://www.freedesktop.org/software/PackageKit/gtk-doc/index.html
+// Package packagekit provides an interface to interact with packagekit.
-
+// See: https://www.freedesktop.org/software/PackageKit/gtk-doc/index.html for
+// more information.
 package packagekit
 
 import (

resources/password.go

@@ -0,0 +1,365 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"crypto/rand"
+	"encoding/gob"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"math/big"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/purpleidea/mgmt/event"
+	"github.com/purpleidea/mgmt/recwatch"
+
+	errwrap "github.com/pkg/errors"
+)
+
+func init() {
+	gob.Register(&PasswordRes{})
+}
+
+const (
+	alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+	newline  = "\n" // something not in alphabet that TrimSpace can trim
+)
+
+// PasswordRes is a no-op resource that returns a random password string.
+type PasswordRes struct {
+	BaseRes `yaml:",inline"`
+	// FIXME: is uint16 too big?
+	Length        uint16  `yaml:"length"` // number of characters to return
+	Saved         bool    // this caches the password in the clear locally
+	CheckRecovery bool    // recovery from integrity checks by re-generating
+	Password      *string // the generated password, read only, do not set!
+
+	path       string // the path to local storage
+	recWatcher *recwatch.RecWatcher
+}
+
+// NewPasswordRes is a constructor for this resource. It also calls Init() for you.
+func NewPasswordRes(name string, length uint16) (*PasswordRes, error) {
+	obj := &PasswordRes{
+		BaseRes: BaseRes{
+			Name: name,
+		},
+		Length: length,
+	}
+	return obj, obj.Init()
+}
+
+// Init generates a new password for this resource if one was not provided. It
+// will save this into a local file. It will load it back in from previous runs.
+func (obj *PasswordRes) Init() error {
+	obj.BaseRes.kind = "Password" // must be set before using VarDir
+
+	dir, err := obj.VarDir("")
+	if err != nil {
+		return errwrap.Wrapf(err, "could not get VarDir in Init()")
+	}
+	obj.path = path.Join(dir, "password") // return a unique file
+
+	return obj.BaseRes.Init() // call base init, b/c we're overriding
+}
+
+// Validate if the params passed in are valid data.
+// FIXME: where should this get called ?
+func (obj *PasswordRes) Validate() error {
+	return nil
+}
+
+func (obj *PasswordRes) read() (string, error) {
+	file, err := os.Open(obj.path) // open a handle to read the file
+	if err != nil {
+		return "", err
+	}
+	defer file.Close()
+	data, err := ioutil.ReadAll(file)
+	if err != nil {
+		return "", errwrap.Wrapf(err, "could not read from file")
+	}
+	return strings.TrimSpace(string(data)), nil
+}
+
+func (obj *PasswordRes) write(password string) (int, error) {
+	file, err := os.Create(obj.path) // open a handle to create the file
+	if err != nil {
+		return -1, errwrap.Wrapf(err, "can't create file")
+	}
+	defer file.Close()
+	var c int
+	if c, err = file.Write([]byte(password + newline)); err != nil {
+		return c, errwrap.Wrapf(err, "can't write file")
+	}
+	return c, file.Sync()
+}
+
+// generate generates a new password.
+func (obj *PasswordRes) generate() (string, error) {
+	max := len(alphabet) - 1 // last index
+	output := ""
+
+	// FIXME: have someone verify this is cryptographically secure & correct
+	for i := uint16(0); i < obj.Length; i++ {
+		big, err := rand.Int(rand.Reader, big.NewInt(int64(max)))
+		if err != nil {
+			return "", errwrap.Wrapf(err, "could not generate password")
+		}
+		ix := big.Int64()
+		output += string(alphabet[ix])
+	}
+
+	if output == "" { // safety against empty passwords
+		return "", fmt.Errorf("password is empty")
+	}
+
+	if uint16(len(output)) != obj.Length { // safety against weird bugs
+		return "", fmt.Errorf("password length is too short") // bug!
+	}
+
+	return output, nil
+}
+
+// check validates a stored password string
+func (obj *PasswordRes) check(value string) error {
+	length := uint16(len(value))
+
+	if !obj.Saved && length == 0 { // expecting an empty string
+		return nil
+	}
+	if !obj.Saved && length != 0 { // should have no stored password
+		return fmt.Errorf("Expected empty token only!")
+	}
+
+	if length != obj.Length {
+		return fmt.Errorf("String length is not %d", obj.Length)
+	}
+Loop:
+	for i := uint16(0); i < length; i++ {
+		for j := 0; j < len(alphabet); j++ {
+			if value[i] == alphabet[j] {
+				continue Loop
+			}
+		}
+		// we couldn't find that character, so error!
+		return fmt.Errorf("Invalid character `%s`", string(value[i]))
+	}
+	return nil
+}
+
+// Watch is the primary listener for this resource and it outputs events.
+func (obj *PasswordRes) Watch(processChan chan event.Event) error {
+	cuid := obj.Converger() // get the converger uid used to report status
+
+	var err error
+	obj.recWatcher, err = recwatch.NewRecWatcher(obj.path, false)
+	if err != nil {
+		return err
+	}
+	defer obj.recWatcher.Close()
+
+	// notify engine that we're running
+	if err := obj.Running(processChan); err != nil {
+		return err // bubble up a NACK...
+	}
+
+	var send = false // send event?
+	var exit = false
+	for {
+		obj.SetState(ResStateWatching) // reset
+		select {
+		// NOTE: this part is very similar to the file resource code
+		case event, ok := <-obj.recWatcher.Events():
+			if !ok { // channel shutdown
+				return nil
+			}
+			cuid.SetConverged(false)
+			if err := event.Error; err != nil {
+				return errwrap.Wrapf(err, "Unknown %s[%s] watcher error", obj.Kind(), obj.GetName())
+			}
+			send = true
+			obj.StateOK(false) // dirty
+
+		case event := <-obj.Events():
+			cuid.SetConverged(false)
+			// we avoid sending events on unpause
+			if exit, send = obj.ReadEvent(&event); exit {
+				return nil // exit
+			}
+
+		case <-cuid.ConvergedTimer():
+			cuid.SetConverged(true) // converged!
+			continue
+		}
+
+		// do all our event sending all together to avoid duplicate msgs
+		if send {
+			send = false
+			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
+				return err // we exit or bubble up a NACK...
+			}
+		}
+	}
+}
+
+// CheckApply method for Password resource. Does nothing, returns happy!
+func (obj *PasswordRes) CheckApply(apply bool) (checkOK bool, err error) {
+
+	var refresh = obj.Refresh() // do we have a pending reload to apply?
+	var exists = true           // does the file (aka the token) exist?
+	var generate bool           // do we need to generate a new password?
+	var write bool              // do we need to write out to disk?
+
+	password, err := obj.read() // password might be empty if just a token
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return false, errwrap.Wrapf(err, "unknown read error")
+		}
+		exists = false
+	}
+
+	if exists {
+		if err := obj.check(password); err != nil {
+			if !obj.CheckRecovery {
+				return false, errwrap.Wrapf(err, "check failed")
+			}
+			log.Printf("%s[%s]: Integrity check failed", obj.Kind(), obj.GetName())
+			generate = true // okay to build a new one
+			write = true    // make sure to write over the old one
+		}
+	} else { // doesn't exist, write one
+		write = true
+	}
+
+	// if we previously had !obj.Saved, and now we want it, we re-generate!
+	if refresh || !exists || (obj.Saved && password == "") {
+		generate = true
+	}
+
+	// stored password isn't consistent with memory
+	if p := obj.Password; obj.Saved && (p != nil && *p != password) {
+		write = true
+	}
+
+	if !refresh && exists && !generate && !write { // nothing to do, done!
+		return true, nil
+	}
+	// a refresh was requested, the token doesn't exist, or the check failed
+
+	if !apply {
+		return false, nil
+	}
+
+	if generate {
+		// we'll need to write this out...
+		if obj.Saved || (!obj.Saved && password != "") {
+			write = true
+		}
+		// generate the actual password
+		var err error
+		log.Printf("%s[%s]: Generating new password...", obj.Kind(), obj.GetName())
+		if password, err = obj.generate(); err != nil { // generate one!
+			return false, errwrap.Wrapf(err, "could not generate password")
+		}
+	}
+
+	obj.Password = &password // save in memory
+
+	var output string // the string to write out
+
+	// if memory value != value on disk, save it
+	if write {
+		if obj.Saved { // save password as clear text
+			// TODO: would it make sense to encrypt this password?
+			output = password
+		}
+		// write either an empty token, or the password
+		log.Printf("%s[%s]: Writing password token...", obj.Kind(), obj.GetName())
+		if _, err := obj.write(output); err != nil {
+			return false, errwrap.Wrapf(err, "can't write to file")
+		}
+	}
+
+	return false, nil
+}
+
+// PasswordUID is the UID struct for PasswordRes.
+type PasswordUID struct {
+	BaseUID
+	name string
+}
+
+// AutoEdges returns the AutoEdge interface. In this case no autoedges are used.
+func (obj *PasswordRes) AutoEdges() AutoEdge {
+	return nil
+}
+
+// GetUIDs includes all params to make a unique identification of this object.
+// Most resources only return one, although some resources can return multiple.
+func (obj *PasswordRes) GetUIDs() []ResUID {
+	x := &PasswordUID{
+		BaseUID: BaseUID{name: obj.GetName(), kind: obj.Kind()},
+		name:    obj.Name,
+	}
+	return []ResUID{x}
+}
+
+// GroupCmp returns whether two resources can be grouped together or not.
+func (obj *PasswordRes) GroupCmp(r Res) bool {
+	_, ok := r.(*PasswordRes)
+	if !ok {
+		return false
+	}
+	return false // TODO: this is doable, but probably not very useful
+	// TODO: it could be useful to group our tokens into a single write, and
+	// as a result, we save inotify watches too!
+}
+
+// Compare two resources and return if they are equivalent.
+func (obj *PasswordRes) Compare(res Res) bool {
+	switch res.(type) {
+	// we can only compare PasswordRes to others of the same resource
+	case *PasswordRes:
+		res := res.(*PasswordRes)
+		if !obj.BaseRes.Compare(res) { // call base Compare
+			return false
+		}
+
+		if obj.Name != res.Name {
+			return false
+		}
+		if obj.Length != res.Length {
+			return false
+		}
+		// TODO: we *could* optimize by allowing CheckApply to move from
+		// saved->!saved, by removing the file, but not likely worth it!
+		if obj.Saved != res.Saved {
+			return false
+		}
+		if obj.CheckRecovery != res.CheckRecovery {
+			return false
+		}
+	default:
+		return false
+	}
+	return true
+}

resources/pkg.go

@@ -19,17 +19,16 @@ package resources
 
 import (
 	"encoding/gob"
-	"errors"
 	"fmt"
 	"log"
 	"path"
 	"strings"
-	"time"
 
 	"github.com/purpleidea/mgmt/event"
-	"github.com/purpleidea/mgmt/global" // XXX: package mgmtmain instead?
 	"github.com/purpleidea/mgmt/resources/packagekit"
 	"github.com/purpleidea/mgmt/util"
+
+	errwrap "github.com/pkg/errors"
 )
 
 func init() {
@@ -48,7 +47,7 @@ type PkgRes struct {
 }
 
 // NewPkgRes is a constructor for this resource. It also calls Init() for you.
-func NewPkgRes(name, state string, allowuntrusted, allownonfree, allowunsupported bool) *PkgRes {
+func NewPkgRes(name, state string, allowuntrusted, allownonfree, allowunsupported bool) (*PkgRes, error) {
 	obj := &PkgRes{
 		BaseRes: BaseRes{
 			Name: name,
@@ -58,8 +57,7 @@ func NewPkgRes(name, state string, allowuntrusted, allownonfree, allowunsupporte
 		AllowNonFree:     allownonfree,
 		AllowUnsupported: allowunsupported,
 	}
-	obj.Init() // XXX: on error return nil, or separate error return?
+	return obj, obj.Init()
-	return obj
 }
 
 // Init runs some startup code for this resource.
@@ -77,7 +75,7 @@ func (obj *PkgRes) Init() error {
 
 	result, err := obj.pkgMappingHelper(bus)
 	if err != nil {
-		return fmt.Errorf("The pkgMappingHelper failed with: %v.", err)
+		return errwrap.Wrapf(err, "The pkgMappingHelper failed")
 	}
 
 	data, ok := result[obj.Name] // lookup single package (init does just one)
@@ -89,7 +87,7 @@ func (obj *PkgRes) Init() error {
 	packageIDs := []string{data.PackageID} // just one for now
 	filesMap, err := bus.GetFilesByPackageID(packageIDs)
 	if err != nil {
-		return fmt.Errorf("Can't run GetFilesByPackageID: %v", err)
+		return errwrap.Wrapf(err, "Can't run GetFilesByPackageID")
 	}
 	if files, ok := filesMap[data.PackageID]; ok {
 		obj.fileList = util.DirifyFileList(files, false)
@@ -111,51 +109,40 @@ func (obj *PkgRes) Validate() error {
 // TODO: https://github.com/hughsie/PackageKit/issues/109
 // TODO: https://github.com/hughsie/PackageKit/issues/110
 func (obj *PkgRes) Watch(processChan chan event.Event) error {
-	if obj.IsWatching() {
+	cuid := obj.Converger() // get the converger uid used to report status
-		return nil
-	}
-	obj.SetWatching(true)
-	defer obj.SetWatching(false)
-	cuuid := obj.converger.Register()
-	defer cuuid.Unregister()
-
-	var startup bool
-	Startup := func(block bool) <-chan time.Time {
-		if block {
-			return nil // blocks forever
-			//return make(chan time.Time) // blocks forever
-		}
-		return time.After(time.Duration(500) * time.Millisecond) // 1/2 the resolution of converged timeout
-	}
 
 	bus := packagekit.NewBus()
 	if bus == nil {
-		log.Fatal("Can't connect to PackageKit bus.")
+		return fmt.Errorf("Can't connect to PackageKit bus.")
 	}
 	defer bus.Close()
 
 	ch, err := bus.WatchChanges()
 	if err != nil {
-		log.Fatalf("Error adding signal match: %v", err)
+		return errwrap.Wrapf(err, "Error adding signal match")
+	}
+
+	// notify engine that we're running
+	if err := obj.Running(processChan); err != nil {
+		return err // bubble up a NACK...
 	}
 
 	var send = false // send event?
 	var exit = false
-	var dirty = false
 
 	for {
-		if global.DEBUG {
+		if obj.debug {
-			log.Printf("%v: Watching...", obj.fmtNames(obj.getNames()))
+			log.Printf("%s: Watching...", obj.fmtNames(obj.getNames()))
 		}
 
 		obj.SetState(ResStateWatching) // reset
 		select {
 		case event := <-ch:
-			cuuid.SetConverged(false)
+			cuid.SetConverged(false)
 
 			// FIXME: ask packagekit for info on what packages changed
-			if global.DEBUG {
+			if obj.debug {
-				log.Printf("%v: Event: %v", obj.fmtNames(obj.getNames()), event.Name)
+				log.Printf("%s: Event: %v", obj.fmtNames(obj.getNames()), event.Name)
 			}
 
 			// since the chan is buffered, remove any supplemental
@@ -165,34 +152,23 @@ func (obj *PkgRes) Watch(processChan chan event.Event) error {
 			}
 
 			send = true
-			dirty = true
+			obj.StateOK(false) // dirty
 
-		case event := <-obj.events:
+		case event := <-obj.Events():
-			cuuid.SetConverged(false)
+			cuid.SetConverged(false)
 			if exit, send = obj.ReadEvent(&event); exit {
 				return nil // exit
 			}
-			dirty = false // these events don't invalidate state
+			//obj.StateOK(false) // these events don't invalidate state
 
-		case <-cuuid.ConvergedTimer():
+		case <-cuid.ConvergedTimer():
-			cuuid.SetConverged(true) // converged!
+			cuid.SetConverged(true) // converged!
 			continue
-
-		case <-Startup(startup):
-			cuuid.SetConverged(false)
-			send = true
-			dirty = true
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
-			startup = true // startup finished
 			send = false
-			// only invalid state on certain types of events
-			if dirty {
-				dirty = false
-				obj.isStateOK = false // something made state dirty
-			}
 			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
 				return err // we exit or bubble up a NACK...
 			}
@@ -218,9 +194,9 @@ func (obj *PkgRes) getNames() []string {
 // pretty print for header values
 func (obj *PkgRes) fmtNames(names []string) string {
 	if len(obj.GetGroup()) > 0 { // grouped elements
-		return fmt.Sprintf("%v[autogroup:(%v)]", obj.Kind(), strings.Join(names, ","))
+		return fmt.Sprintf("%s[autogroup:(%v)]", obj.Kind(), strings.Join(names, ","))
 	}
-	return fmt.Sprintf("%v[%v]", obj.Kind(), obj.GetName())
+	return fmt.Sprintf("%s[%s]", obj.Kind(), obj.GetName())
 }
 
 func (obj *PkgRes) groupMappingHelper() map[string]string {
@@ -229,7 +205,7 @@ func (obj *PkgRes) groupMappingHelper() map[string]string {
 		for _, x := range g {
 			pkg, ok := x.(*PkgRes) // convert from Res
 			if !ok {
-				log.Fatalf("Grouped member %v is not a %v", x, obj.Kind())
+				log.Fatalf("Grouped member %v is not a %s", x, obj.Kind())
 			}
 			result[pkg.Name] = pkg.State
 		}
@@ -255,35 +231,27 @@ func (obj *PkgRes) pkgMappingHelper(bus *packagekit.Conn) (map[string]*packageki
 	if !obj.AllowUnsupported {
 		filter += packagekit.PK_FILTER_ENUM_SUPPORTED
 	}
-	result, e := bus.PackagesToPackageIDs(packageMap, filter)
+	result, err := bus.PackagesToPackageIDs(packageMap, filter)
-	if e != nil {
+	if err != nil {
-		return nil, fmt.Errorf("Can't run PackagesToPackageIDs: %v", e)
+		return nil, errwrap.Wrapf(err, "Can't run PackagesToPackageIDs")
 	}
 	return result, nil
 }
 
 // CheckApply checks the resource state and applies the resource if the bool
 // input is true. It returns error info and if the state check passed or not.
-func (obj *PkgRes) CheckApply(apply bool) (checkok bool, err error) {
+func (obj *PkgRes) CheckApply(apply bool) (checkOK bool, err error) {
-	log.Printf("%v: CheckApply(%t)", obj.fmtNames(obj.getNames()), apply)
+	log.Printf("%s: Check", obj.fmtNames(obj.getNames()))
-
-	if obj.State == "" { // TODO: Validate() should replace this check!
-		log.Fatalf("%v: Package state is undefined!", obj.fmtNames(obj.getNames()))
-	}
-
-	if obj.isStateOK { // cache the state
-		return true, nil
-	}
 
 	bus := packagekit.NewBus()
 	if bus == nil {
-		return false, errors.New("Can't connect to PackageKit bus.")
+		return false, fmt.Errorf("Can't connect to PackageKit bus.")
 	}
 	defer bus.Close()
 
 	result, err := obj.pkgMappingHelper(bus)
 	if err != nil {
-		return false, fmt.Errorf("The pkgMappingHelper failed with: %v.", err)
+		return false, errwrap.Wrapf(err, "The pkgMappingHelper failed")
 	}
 
 	packageMap := obj.groupMappingHelper() // map[string]string
@@ -296,7 +264,7 @@ func (obj *PkgRes) CheckApply(apply bool) (checkok bool, err error) {
 	// eventually we might be able to drop this constraint!
 	states, err := packagekit.FilterState(result, packageList, obj.State)
 	if err != nil {
-		return false, fmt.Errorf("The FilterState method failed with: %v.", err)
+		return false, errwrap.Wrapf(err, "The FilterState method failed")
 	}
 	data, _ := result[obj.Name] // if above didn't error, we won't either!
 	validState := util.BoolMapTrue(util.BoolMapValues(states))
@@ -309,12 +277,10 @@ func (obj *PkgRes) CheckApply(apply bool) (checkok bool, err error) {
 		fallthrough
 	case "newest":
 		if validState {
-			obj.isStateOK = true // reset
 			return true, nil // state is correct, exit!
 		}
 	default: // version string
 		if obj.State == data.Version && data.Version != "" {
-			obj.isStateOK = true // reset
 			return true, nil
 		}
 	}
@@ -325,7 +291,7 @@ func (obj *PkgRes) CheckApply(apply bool) (checkok bool, err error) {
 	}
 
 	// apply portion
-	log.Printf("%v: Apply", obj.fmtNames(obj.getNames()))
+	log.Printf("%s: Apply", obj.fmtNames(obj.getNames()))
 	readyPackages, err := packagekit.FilterPackageState(result, packageList, obj.State)
 	if err != nil {
 		return false, err // fail
@@ -339,7 +305,7 @@ func (obj *PkgRes) CheckApply(apply bool) (checkok bool, err error) {
 		transactionFlags += packagekit.PK_TRANSACTION_FLAG_ENUM_ONLY_TRUSTED
 	}
 	// apply correct state!
-	log.Printf("%v: Set: %v...", obj.fmtNames(util.StrListIntersection(applyPackages, obj.getNames())), obj.State)
+	log.Printf("%s: Set: %v...", obj.fmtNames(util.StrListIntersection(applyPackages, obj.getNames())), obj.State)
 	switch obj.State {
 	case "uninstalled": // run remove
 		// NOTE: packageID is different than when installed, because now
@@ -357,21 +323,20 @@ func (obj *PkgRes) CheckApply(apply bool) (checkok bool, err error) {
 	if err != nil {
 		return false, err // fail
 	}
-	log.Printf("%v: Set: %v success!", obj.fmtNames(util.StrListIntersection(applyPackages, obj.getNames())), obj.State)
+	log.Printf("%s: Set: %v success!", obj.fmtNames(util.StrListIntersection(applyPackages, obj.getNames())), obj.State)
-	obj.isStateOK = true // reset
 	return false, nil // success
 }
 
-// PkgUUID is the UUID struct for PkgRes.
+// PkgUID is the UID struct for PkgRes.
-type PkgUUID struct {
+type PkgUID struct {
-	BaseUUID
+	BaseUID
 	name  string // pkg name
 	state string // pkg state or "version"
 }
 
 // IFF aka if and only if they are equivalent, return true. If not, false.
-func (obj *PkgUUID) IFF(uuid ResUUID) bool {
+func (obj *PkgUID) IFF(uid ResUID) bool {
-	res, ok := uuid.(*PkgUUID)
+	res, ok := uid.(*PkgUID)
 	if !ok {
 		return false
 	}
@@ -382,30 +347,30 @@ func (obj *PkgUUID) IFF(uuid ResUUID) bool {
 // PkgResAutoEdges holds the state of the auto edge generator.
 type PkgResAutoEdges struct {
 	fileList   []string
-	svcUUIDs   []ResUUID
+	svcUIDs    []ResUID
 	testIsNext bool   // safety
 	name       string // saved data from PkgRes obj
 	kind       string
 }
 
 // Next returns the next automatic edge.
-func (obj *PkgResAutoEdges) Next() []ResUUID {
+func (obj *PkgResAutoEdges) Next() []ResUID {
 	if obj.testIsNext {
 		log.Fatal("Expecting a call to Test()")
 	}
 	obj.testIsNext = true // set after all the errors paths are past
 
-	// first return any matching svcUUIDs
+	// first return any matching svcUIDs
-	if x := obj.svcUUIDs; len(x) > 0 {
+	if x := obj.svcUIDs; len(x) > 0 {
 		return x
 	}
 
-	var result []ResUUID
+	var result []ResUID
-	// return UUID's for whatever is in obj.fileList
+	// return UID's for whatever is in obj.fileList
 	for _, x := range obj.fileList {
 		var reversed = false // cheat by passing a pointer
-		result = append(result, &FileUUID{
+		result = append(result, &FileUID{
-			BaseUUID: BaseUUID{
+			BaseUID: BaseUID{
 				name:     obj.name,
 				kind:     obj.kind,
 				reversed: &reversed,
@@ -422,12 +387,12 @@ func (obj *PkgResAutoEdges) Test(input []bool) bool {
 		log.Fatal("Expecting a call to Next()")
 	}
 
-	// ack the svcUUID's...
+	// ack the svcUID's...
-	if x := obj.svcUUIDs; len(x) > 0 {
+	if x := obj.svcUIDs; len(x) > 0 {
 		if y := len(x); y != len(input) {
 			log.Fatalf("Expecting %d value(s)!", y)
 		}
-		obj.svcUUIDs = []ResUUID{} // empty
+		obj.svcUIDs = []ResUID{} // empty
 		obj.testIsNext = false
 		return true
 	}
@@ -475,37 +440,37 @@ func (obj *PkgRes) AutoEdges() AutoEdge {
 	// is contained in the Test() method! This design is completely okay!
 
 	// add matches for any svc resources found in pkg definition!
-	var svcUUIDs []ResUUID
+	var svcUIDs []ResUID
 	for _, x := range ReturnSvcInFileList(obj.fileList) {
 		var reversed = false
-		svcUUIDs = append(svcUUIDs, &SvcUUID{
+		svcUIDs = append(svcUIDs, &SvcUID{
-			BaseUUID: BaseUUID{
+			BaseUID: BaseUID{
 				name:     obj.GetName(),
 				kind:     obj.Kind(),
 				reversed: &reversed,
 			},
-			name: x, // the svc name itself in the SvcUUID object!
+			name: x, // the svc name itself in the SvcUID object!
 		}) // build list
 	}
 
 	return &PkgResAutoEdges{
 		fileList:   util.RemoveCommonFilePrefixes(obj.fileList), // clean start!
-		svcUUIDs:   svcUUIDs,
+		svcUIDs:    svcUIDs,
 		testIsNext: false,         // start with Next() call
 		name:       obj.GetName(), // save data for PkgResAutoEdges obj
 		kind:       obj.Kind(),
 	}
 }
 
-// GetUUIDs includes all params to make a unique identification of this object.
+// GetUIDs includes all params to make a unique identification of this object.
 // Most resources only return one, although some resources can return multiple.
-func (obj *PkgRes) GetUUIDs() []ResUUID {
+func (obj *PkgRes) GetUIDs() []ResUID {
-	x := &PkgUUID{
+	x := &PkgUID{
-		BaseUUID: BaseUUID{name: obj.GetName(), kind: obj.Kind()},
+		BaseUID: BaseUID{name: obj.GetName(), kind: obj.Kind()},
 		name:    obj.Name,
 		state:   obj.State,
 	}
-	result := []ResUUID{x}
+	result := []ResUID{x}
 	return result
 }
 

resources/refresh.go

@@ -0,0 +1,104 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strings"
+
+	errwrap "github.com/pkg/errors"
+)
+
+// Refresh returns the pending state of a notification. It should only be called
+// in the CheckApply portion of a resource where a refresh should be acted upon.
+func (obj *BaseRes) Refresh() bool {
+	return obj.refresh
+}
+
+// SetRefresh sets the pending state of a notification. It should only be called
+// by the mgmt engine.
+func (obj *BaseRes) SetRefresh(b bool) {
+	obj.refresh = b
+}
+
+// StatefulBool is an interface for storing a boolean flag in a permanent spot.
+type StatefulBool interface {
+	Get() (bool, error) // get value of token
+	Set() error         // set token to true
+	Del() error         // rm token if it exists
+}
+
+// DiskBool stores a boolean variable on disk for stateful access across runs.
+// The absence of the path is treated as false. If the path contains a special
+// value, then it is treated as true. All the other non-error cases are false.
+type DiskBool struct {
+	Path string // path to token
+}
+
+// str returns the string data which represents true (aka set).
+func (obj *DiskBool) str() string {
+	const TrueToken = "true"
+	const newline = "\n"
+	return TrueToken + newline
+}
+
+// Get returns if the boolean setting, if no error reading the value occurs.
+func (obj *DiskBool) Get() (bool, error) {
+	file, err := os.Open(obj.Path) // open a handle to read the file
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil // no token means value is false
+		}
+		return false, errwrap.Wrapf(err, "could not read token")
+	}
+	defer file.Close()
+	data, err := ioutil.ReadAll(file)
+	if err != nil {
+		return false, errwrap.Wrapf(err, "could not read from file")
+	}
+	return strings.TrimSpace(string(data)) == strings.TrimSpace(obj.str()), nil
+}
+
+// Set stores the true boolean value, if no error setting the value occurs.
+func (obj *DiskBool) Set() error {
+	file, err := os.Create(obj.Path) // open a handle to create the file
+	if err != nil {
+		return errwrap.Wrapf(err, "can't create file")
+	}
+	defer file.Close()
+	str := obj.str()
+	if c, err := file.Write([]byte(str)); err != nil {
+		return errwrap.Wrapf(err, "error writing to file")
+	} else if l := len(str); c != l {
+		return fmt.Errorf("wrote %d bytes instead of %d", c, l)
+	}
+	return file.Sync() // guarantee it!
+}
+
+// Del stores the false boolean value, if no error clearing the value occurs.
+func (obj *DiskBool) Del() error {
+	if err := os.Remove(obj.Path); err != nil { // remove the file
+		if os.IsNotExist(err) {
+			return nil // no file means this is already fine
+		}
+		return errwrap.Wrapf(err, "could not delete token")
+	}
+	return nil
+}

resources/resources.go

@@ -15,6 +15,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+// Package resources provides the resource framework and idempotent primitives.
 package resources
 
 import (
@@ -23,11 +24,14 @@ import (
 	"encoding/gob"
 	"fmt"
 	"log"
+	"os"
+	"path"
 
 	// TODO: should each resource be a sub-package?
 	"github.com/purpleidea/mgmt/converger"
 	"github.com/purpleidea/mgmt/event"
-	"github.com/purpleidea/mgmt/global"
+
+	errwrap "github.com/pkg/errors"
 )
 
 //go:generate stringer -type=ResState -output=resstate_stringer.go
@@ -44,17 +48,29 @@ const (
 	ResStatePoking
 )
 
-// ResUUID is a unique identifier for a resource, namely it's name, and the kind ("type").
+const refreshPathToken = "refresh"
-type ResUUID interface {
+
+// Data is the set of input values passed into the pgraph for the resources.
+type Data struct {
+	//Hostname string         // uuid for the host
+	//Noop     bool
+	Converger converger.Converger
+	Prefix    string // the prefix to be used for the pgraph namespace
+	Debug     bool
+	// NOTE: we can add more fields here if needed for the resources.
+}
+
+// ResUID is a unique identifier for a resource, namely it's name, and the kind ("type").
+type ResUID interface {
 	GetName() string
 	Kind() string
-	IFF(ResUUID) bool
+	IFF(ResUID) bool
 
 	Reversed() bool // true means this resource happens before the generator
 }
 
-// The BaseUUID struct is used to provide a unique resource identifier.
+// The BaseUID struct is used to provide a unique resource identifier.
-type BaseUUID struct {
+type BaseUID struct {
 	name string // name and kind are the values of where this is coming from
 	kind string
 
@@ -63,14 +79,14 @@ type BaseUUID struct {
 
 // The AutoEdge interface is used to implement the autoedges feature.
 type AutoEdge interface {
-	Next() []ResUUID  // call to get list of edges to add
+	Next() []ResUID   // call to get list of edges to add
 	Test([]bool) bool // call until false
 }
 
 // MetaParams is a struct will all params that apply to every resource.
 type MetaParams struct {
-	AutoEdge  bool `yaml:"autoedge"`  // metaparam, should we generate auto edges? // XXX should default to true
+	AutoEdge  bool `yaml:"autoedge"`  // metaparam, should we generate auto edges?
-	AutoGroup bool `yaml:"autogroup"` // metaparam, should we auto group? // XXX should default to true
+	AutoGroup bool `yaml:"autogroup"` // metaparam, should we auto group?
 	Noop      bool `yaml:"noop"`
 	// NOTE: there are separate Watch and CheckApply retry and delay values,
 	// but I've decided to use the same ones for both until there's a proper
@@ -79,6 +95,29 @@ type MetaParams struct {
 	Delay uint64 `yaml:"delay"` // metaparam, number of milliseconds to wait between retries
 }
 
+// UnmarshalYAML is the custom unmarshal handler for the MetaParams struct. It
+// is primarily useful for setting the defaults.
+func (obj *MetaParams) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type rawMetaParams MetaParams           // indirection to avoid infinite recursion
+	raw := rawMetaParams(DefaultMetaParams) // convert; the defaults go here
+
+	if err := unmarshal(&raw); err != nil {
+		return err
+	}
+
+	*obj = MetaParams(raw) // restore from indirection with type conversion!
+	return nil
+}
+
+// DefaultMetaParams are the defaults to be used for undefined metaparams.
+var DefaultMetaParams = MetaParams{
+	AutoEdge:  true,
+	AutoGroup: true,
+	Noop:      false,
+	Retry:     0, // TODO: is this a good default?
+	Delay:     0, // TODO: is this a good default?
+}
+
 // The Base interface is everything that is common to all resources.
 // Everything here only needs to be implemented once, in the BaseRes.
 type Base interface {
@@ -88,20 +127,32 @@ type Base interface {
 	Kind() string
 	Meta() *MetaParams
 	Events() chan event.Event
-	AssociateData(converger.Converger)
+	AssociateData(*Data)
 	IsWatching() bool
 	SetWatching(bool)
+	RegisterConverger()
+	UnregisterConverger()
+	Converger() converger.ConvergerUID
 	GetState() ResState
 	SetState(ResState)
 	DoSend(chan event.Event, string) (bool, error)
 	SendEvent(event.EventName, bool, bool) bool
 	ReadEvent(*event.Event) (bool, bool)   // TODO: optional here?
+	Refresh() bool                         // is there a pending refresh to run?
+	SetRefresh(bool)                       // set the refresh state of this resource
+	SendRecv(Res) (map[string]bool, error) // send->recv data passing function
+	IsStateOK() bool
+	StateOK(b bool)
 	GroupCmp(Res) bool  // TODO: is there a better name for this?
 	GroupRes(Res) error // group resource (arg) into self
 	IsGrouped() bool    // am I grouped?
 	SetGrouped(bool)    // set grouped bool
 	GetGroup() []Res    // return everyone grouped inside me
 	SetGroup([]Res)
+	VarDir(string) (string, error)
+	Running(chan event.Event) error // notify the engine that Watch started
+	Started() <-chan struct{}       // returns when the resource has started
+	Starter(bool)
 }
 
 // Res is the minimum interface you need to implement to define a new resource.
@@ -109,9 +160,9 @@ type Res interface {
 	Base // include everything from the Base interface
 	Init() error
 	//Validate() error    // TODO: this might one day be added
-	GetUUIDs() []ResUUID          // most resources only return one
+	GetUIDs() []ResUID            // most resources only return one
 	Watch(chan event.Event) error // send on channel to signal process() events
-	CheckApply(bool) (bool, error)
+	CheckApply(apply bool) (checkOK bool, err error)
 	AutoEdges() AutoEdge
 	Compare(Res) bool
 	CollectPattern(string) // XXX: temporary until Res collection is more advanced
@@ -121,20 +172,29 @@ type Res interface {
 type BaseRes struct {
 	Name       string           `yaml:"name"`
 	MetaParams MetaParams       `yaml:"meta"` // struct of all the metaparams
+	Recv       map[string]*Send // mapping of key to receive on from value
+
 	kind      string
 	events    chan event.Event
 	converger converger.Converger // converged tracking
+	cuid      converger.ConvergerUID
+	prefix    string // base prefix for this resource
+	debug     bool
 	state     ResState
 	watching  bool          // is Watch() loop running ?
+	started   chan struct{} // closed when worker is started/running
+	starter   bool          // does this have indegree == 0 ? XXX: usually?
 	isStateOK bool          // whether the state is okay based on events or not
 	isGrouped bool          // am i contained within a group?
 	grouped   []Res         // list of any grouped resources
+	refresh   bool          // does this resource have a refresh to run?
+	//refreshState StatefulBool // TODO: future stateful bool
 }
 
-// UUIDExistsInUUIDs wraps the IFF method when used with a list of UUID's.
+// UIDExistsInUIDs wraps the IFF method when used with a list of UID's.
-func UUIDExistsInUUIDs(uuid ResUUID, uuids []ResUUID) bool {
+func UIDExistsInUIDs(uid ResUID, uids []ResUID) bool {
-	for _, u := range uuids {
+	for _, u := range uids {
-		if uuid.IFF(u) {
+		if uid.IFF(u) {
 			return true
 		}
 	}
@@ -142,30 +202,30 @@ func UUIDExistsInUUIDs(uuid ResUUID, uuids []ResUUID) bool {
 }
 
 // GetName returns the name of the resource.
-func (obj *BaseUUID) GetName() string {
+func (obj *BaseUID) GetName() string {
 	return obj.name
 }
 
 // Kind returns the kind of resource.
-func (obj *BaseUUID) Kind() string {
+func (obj *BaseUID) Kind() string {
 	return obj.kind
 }
 
-// IFF looks at two UUID's and if and only if they are equivalent, returns true.
+// IFF looks at two UID's and if and only if they are equivalent, returns true.
 // If they are not equivalent, it returns false.
 // Most resources will want to override this method, since it does the important
 // work of actually discerning if two resources are identical in function.
-func (obj *BaseUUID) IFF(uuid ResUUID) bool {
+func (obj *BaseUID) IFF(uid ResUID) bool {
-	res, ok := uuid.(*BaseUUID)
+	res, ok := uid.(*BaseUID)
 	if !ok {
 		return false
 	}
 	return obj.name == res.name
 }
 
-// Reversed is part of the ResUUID interface, and true means this resource
+// Reversed is part of the ResUID interface, and true means this resource
 // happens before the generator.
-func (obj *BaseUUID) Reversed() bool {
+func (obj *BaseUID) Reversed() bool {
 	if obj.reversed == nil {
 		log.Fatal("Programming error!")
 	}
@@ -174,7 +234,17 @@ func (obj *BaseUUID) Reversed() bool {
 
 // Init initializes structures like channels if created without New constructor.
 func (obj *BaseRes) Init() error {
-	obj.events = make(chan event.Event) // unbuffered chan size to avoid stale events
+	if obj.kind == "" {
+		return fmt.Errorf("Resource did not set kind!")
+	}
+	obj.events = make(chan event.Event) // unbuffered chan to avoid stale events
+	obj.started = make(chan struct{})   // closes when started
+	//dir, err := obj.VarDir("")
+	//if err != nil {
+	//	return errwrap.Wrapf(err, "VarDir failed in Init()")
+	//}
+	// TODO: this StatefulBool implementation could be eventually swappable
+	//obj.refreshState = &DiskBool{Path: path.Join(dir, refreshPathToken)}
 	return nil
 }
 
@@ -209,20 +279,40 @@ func (obj *BaseRes) Events() chan event.Event {
 }
 
 // AssociateData associates some data with the object in question.
-func (obj *BaseRes) AssociateData(converger converger.Converger) {
+func (obj *BaseRes) AssociateData(data *Data) {
-	obj.converger = converger
+	obj.converger = data.Converger
+	obj.prefix = data.Prefix
+	obj.debug = data.Debug
 }
 
-// IsWatching tells us if the Watch() function is running.
+// IsWatching tells us if the Worker() function is running.
 func (obj *BaseRes) IsWatching() bool {
 	return obj.watching
 }
 
-// SetWatching stores the status of if the Watch() function is running.
+// SetWatching stores the status of if the Worker() function is running.
 func (obj *BaseRes) SetWatching(b bool) {
 	obj.watching = b
 }
 
+// RegisterConverger sets up the cuid for the resource. This is a helper
+// function for the engine, and shouldn't be called by the resources directly.
+func (obj *BaseRes) RegisterConverger() {
+	obj.cuid = obj.converger.Register()
+}
+
+// UnregisterConverger tears down the cuid for the resource. This is a helper
+// function for the engine, and shouldn't be called by the resources directly.
+func (obj *BaseRes) UnregisterConverger() {
+	obj.cuid.Unregister()
+}
+
+// Converger returns the ConvergerUID for the resource. This should be called
+// by the Watch method of the resource to set the converged state.
+func (obj *BaseRes) Converger() converger.ConvergerUID {
+	return obj.cuid
+}
+
 // GetState returns the state of the resource.
 func (obj *BaseRes) GetState() ResState {
 	return obj.state
@@ -230,91 +320,20 @@ func (obj *BaseRes) GetState() ResState {
 
 // SetState sets the state of the resource.
 func (obj *BaseRes) SetState(state ResState) {
-	if global.DEBUG {
+	if obj.debug {
-		log.Printf("%v[%v]: State: %v -> %v", obj.Kind(), obj.GetName(), obj.GetState(), state)
+		log.Printf("%s[%s]: State: %v -> %v", obj.Kind(), obj.GetName(), obj.GetState(), state)
 	}
 	obj.state = state
 }
 
-// DoSend sends off an event, but doesn't block the incoming event queue. It can
+// IsStateOK returns the cached state value.
-// also recursively call itself when events need processing during the wait.
+func (obj *BaseRes) IsStateOK() bool {
-// I'm not completely comfortable with this fn, but it will have to do for now.
+	return obj.isStateOK
-func (obj *BaseRes) DoSend(processChan chan event.Event, comment string) (bool, error) {
-	resp := event.NewResp()
-	processChan <- event.Event{Name: event.EventNil, Resp: resp, Msg: comment, Activity: true} // trigger process
-	e := resp.Wait()
-	return false, e // XXX: at the moment, we don't use the exit bool.
-	// XXX: this can cause a deadlock. do we need to recursively send? fix event stuff!
-	//select {
-	//case e := <-resp: // wait for the ACK()
-	//	if e != nil { // we got a NACK
-	//		return true, e // exit with error
-	//	}
-	//case event := <-obj.events:
-	//	// NOTE: this code should match the similar code below!
-	//	//cuuid.SetConverged(false) // TODO ?
-	//	if exit, send := obj.ReadEvent(&event); exit {
-	//		return true, nil // exit, without error
-	//	} else if send {
-	//		return obj.DoSend(processChan, comment) // recurse
-	//	}
-	//}
-	//return false, nil // return, no error or exit signal
 }
 
-// SendEvent pushes an event into the message queue for a particular vertex
+// StateOK sets the cached state value.
-func (obj *BaseRes) SendEvent(ev event.EventName, sync bool, activity bool) bool {
+func (obj *BaseRes) StateOK(b bool) {
-	// TODO: isn't this race-y ?
+	obj.isStateOK = b
-	if !obj.IsWatching() { // element has already exited
-		return false // if we don't return, we'll block on the send
-	}
-	if !sync {
-		obj.events <- event.Event{Name: ev, Resp: nil, Msg: "", Activity: activity}
-		return true
-	}
-
-	resp := event.NewResp()
-	obj.events <- event.Event{Name: ev, Resp: resp, Msg: "", Activity: activity}
-	resp.ACKWait() // waits until true (nil) value
-	return true
-}
-
-// ReadEvent processes events when a select gets one, and handles the pause
-// code too! The return values specify if we should exit and poke respectively.
-func (obj *BaseRes) ReadEvent(ev *event.Event) (exit, poke bool) {
-	ev.ACK()
-	switch ev.Name {
-	case event.EventStart:
-		return false, true
-
-	case event.EventPoke:
-		return false, true
-
-	case event.EventBackPoke:
-		return false, true // forward poking in response to a back poke!
-
-	case event.EventExit:
-		return true, false
-
-	case event.EventPause:
-		// wait for next event to continue
-		select {
-		case e := <-obj.events:
-			e.ACK()
-			if e.Name == event.EventExit {
-				return true, false
-			} else if e.Name == event.EventStart { // eventContinue
-				return false, false // don't poke on unpause!
-			} else {
-				// if we get a poke event here, it's a bug!
-				log.Fatalf("%v[%v]: Unknown event: %v, while paused!", obj.Kind(), obj.GetName(), e)
-			}
-		}
-
-	default:
-		log.Fatal("Unknown event: ", ev)
-	}
-	return true, false // required to keep the stupid go compiler happy
 }
 
 // GroupCmp compares two resources and decides if they're suitable for grouping
@@ -357,7 +376,7 @@ func (obj *BaseRes) SetGroup(g []Res) {
 	obj.grouped = g
 }
 
-// Compare is the base compare method, which also handles the metaparams cmp
+// Compare is the base compare method, which also handles the metaparams cmp.
 func (obj *BaseRes) Compare(res Res) bool {
 	// TODO: should the AutoEdge values be compared?
 	if obj.Meta().AutoEdge != res.Meta().AutoEdge {
@@ -388,6 +407,37 @@ func (obj *BaseRes) CollectPattern(pattern string) {
 	// XXX: default method is empty
 }
 
+// VarDir returns the path to a working directory for the resource. It will try
+// and create the directory first, and return an error if this failed.
+func (obj *BaseRes) VarDir(extra string) (string, error) {
+	// Using extra adds additional dirs onto our namespace. An empty extra
+	// adds no additional directories.
+	if obj.prefix == "" {
+		return "", fmt.Errorf("VarDir prefix is empty!")
+	}
+	if obj.Kind() == "" {
+		return "", fmt.Errorf("VarDir kind is empty!")
+	}
+	if obj.GetName() == "" {
+		return "", fmt.Errorf("VarDir name is empty!")
+	}
+
+	// FIXME: is obj.GetName() sufficiently unique to use as a UID here?
+	uid := obj.GetName()
+	p := fmt.Sprintf("%s/", path.Join(obj.prefix, obj.Kind(), uid, extra))
+	if err := os.MkdirAll(p, 0770); err != nil {
+		return "", errwrap.Wrapf(err, "Can't create prefix for %s[%s]", obj.Kind(), obj.GetName())
+	}
+	return p, nil
+}
+
+// Started returns a channel that closes when the resource has started up.
+func (obj *BaseRes) Started() <-chan struct{} { return obj.started }
+
+// Starter sets the starter bool. This defines if a vertex has an indegree of 0.
+// If we have an indegree of 0, we'll need to be a poke initiator in the graph.
+func (obj *BaseRes) Starter(b bool) { obj.starter = b }
+
 // ResToB64 encodes a resource to a base64 encoded string (after serialization)
 func ResToB64(res Res) (string, error) {
 	b := bytes.Buffer{}

resources/resources_test.go

@@ -105,16 +105,16 @@ func TestMiscEncodeDecode2(t *testing.T) {
 }
 
 func TestIFF(t *testing.T) {
-	uuid := &BaseUUID{name: "/tmp/unit-test"}
+	uid := &BaseUID{name: "/tmp/unit-test"}
-	same := &BaseUUID{name: "/tmp/unit-test"}
+	same := &BaseUID{name: "/tmp/unit-test"}
-	diff := &BaseUUID{name: "/tmp/other-file"}
+	diff := &BaseUID{name: "/tmp/other-file"}
 
-	if !uuid.IFF(same) {
+	if !uid.IFF(same) {
-		t.Error("basic resource UUIDs with the same name should satisfy each other's IFF condition.")
+		t.Error("basic resource UIDs with the same name should satisfy each other's IFF condition.")
 	}
 
-	if uuid.IFF(diff) {
+	if uid.IFF(diff) {
-		t.Error("basic resource UUIDs with different names should NOT satisfy each other's IFF condition.")
+		t.Error("basic resource UIDs with different names should NOT satisfy each other's IFF condition.")
 	}
 }
 

resources/sendrecv.go

@@ -0,0 +1,216 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"fmt"
+	"log"
+	"reflect"
+
+	"github.com/purpleidea/mgmt/event"
+
+	multierr "github.com/hashicorp/go-multierror"
+	errwrap "github.com/pkg/errors"
+)
+
+// SendEvent pushes an event into the message queue for a particular vertex
+func (obj *BaseRes) SendEvent(ev event.EventName, sync bool, activity bool) bool {
+	// TODO: isn't this race-y ?
+	if !obj.IsWatching() { // element has already exited
+		return false // if we don't return, we'll block on the send
+	}
+	if !sync {
+		obj.events <- event.Event{Name: ev, Resp: nil, Msg: "", Activity: activity}
+		return true
+	}
+
+	resp := event.NewResp()
+	obj.events <- event.Event{Name: ev, Resp: resp, Msg: "", Activity: activity}
+	resp.ACKWait() // waits until true (nil) value
+	return true
+}
+
+// DoSend sends off an event, but doesn't block the incoming event queue.
+func (obj *BaseRes) DoSend(processChan chan event.Event, comment string) (exit bool, err error) {
+	resp := event.NewResp()
+	processChan <- event.Event{Name: event.EventNil, Resp: resp, Activity: false, Msg: comment} // trigger process
+	e := resp.Wait()
+	return false, e // XXX: at the moment, we don't use the exit bool.
+}
+
+// ReadEvent processes events when a select gets one, and handles the pause
+// code too! The return values specify if we should exit and poke respectively.
+func (obj *BaseRes) ReadEvent(ev *event.Event) (exit, send bool) {
+	ev.ACK()
+	var poke bool
+	// ensure that a CheckApply runs by sending with a dirty state...
+	if ev.GetActivity() { // if previous node did work, and we were notified...
+		//obj.StateOK(false) // not necessarily
+		poke = true // poke!
+		// XXX: this should be elsewhere in case Watch isn't used (eg: Polling instead...)
+		// XXX: unless this is used in our "fallback" polling implementation???
+		//obj.SetRefresh(true) // TODO: is this redundant?
+	}
+
+	switch ev.Name {
+	case event.EventStart:
+		send = true || poke
+		return
+
+	case event.EventPoke:
+		send = true || poke
+		return
+
+	case event.EventBackPoke:
+		send = true || poke
+		return // forward poking in response to a back poke!
+
+	case event.EventExit:
+		// FIXME: what do we do if we have a pending refresh (poke) and an exit?
+		return true, false
+
+	case event.EventPause:
+		// wait for next event to continue
+		select {
+		case e, ok := <-obj.Events():
+			if !ok { // shutdown
+				return true, false
+			}
+			e.ACK()
+			if e.Name == event.EventExit {
+				return true, false
+			} else if e.Name == event.EventStart { // eventContinue
+				return false, false // don't poke on unpause!
+			} else {
+				// if we get a poke event here, it's a bug!
+				log.Fatalf("%s[%s]: Unknown event: %v, while paused!", obj.Kind(), obj.GetName(), e)
+			}
+		}
+
+	default:
+		log.Fatal("Unknown event: ", ev)
+	}
+	return true, false // required to keep the stupid go compiler happy
+}
+
+// Running is called by the Watch method of the resource once it has started up.
+// This signals to the engine to kick off the initial CheckApply resource check.
+func (obj *BaseRes) Running(processChan chan event.Event) error {
+	obj.StateOK(false)       // assume we're initially dirty
+	cuid := obj.Converger()  // get the converger uid used to report status
+	cuid.SetConverged(false) // a reasonable initial assumption
+	close(obj.started)       // send started signal
+
+	// FIXME: exit return value is unused atm, so ignore it for now...
+	//if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
+	var err error
+	if obj.starter { // vertices of indegree == 0 should send initial pokes
+		_, err = obj.DoSend(processChan, "") // trigger a CheckApply
+	}
+	return err // bubble up any possible error (or nil)
+}
+
+// Send points to a value that a resource will send.
+type Send struct {
+	Res Res    // a handle to the resource which is sending a value
+	Key string // the key in the resource that we're sending
+
+	Changed bool // set to true if this key was updated, read only!
+}
+
+// SendRecv pulls in the sent values into the receive slots. It is called by the
+// receiver and must be given as input the full resource struct to receive on.
+func (obj *BaseRes) SendRecv(res Res) (map[string]bool, error) {
+	if obj.debug {
+		// NOTE: this could expose private resource data like passwords
+		log.Printf("%s[%s]: SendRecv: %+v", obj.Kind(), obj.GetName(), obj.Recv)
+	}
+	var updated = make(map[string]bool) // list of updated keys
+	var err error
+	for k, v := range obj.Recv {
+		updated[k] = false // default
+		v.Changed = false  // reset to the default
+		// send
+		obj1 := reflect.Indirect(reflect.ValueOf(v.Res))
+		type1 := obj1.Type()
+		value1 := obj1.FieldByName(v.Key)
+		kind1 := value1.Kind()
+
+		// recv
+		obj2 := reflect.Indirect(reflect.ValueOf(res)) // pass in full struct
+		type2 := obj2.Type()
+		value2 := obj2.FieldByName(k)
+		kind2 := value2.Kind()
+
+		if obj.debug {
+			log.Printf("Send(%s) has %v: %v", type1, kind1, value1)
+			log.Printf("Recv(%s) has %v: %v", type2, kind2, value2)
+		}
+
+		// i think we probably want the same kind, at least for now...
+		if kind1 != kind2 {
+			e := fmt.Errorf("Kind mismatch between %s[%s]: %s and %s[%s]: %s", v.Res.Kind(), v.Res.GetName(), kind1, obj.Kind(), obj.GetName(), kind2)
+			err = multierr.Append(err, e) // list of errors
+			continue
+		}
+
+		// if the types don't match, we can't use send->recv
+		// TODO: do we want to relax this for string -> *string ?
+		if e := TypeCmp(value1, value2); e != nil {
+			e := errwrap.Wrapf(e, "Type mismatch between %s[%s] and %s[%s]", v.Res.Kind(), v.Res.GetName(), obj.Kind(), obj.GetName())
+			err = multierr.Append(err, e) // list of errors
+			continue
+		}
+
+		// if we can't set, then well this is pointless!
+		if !value2.CanSet() {
+			e := fmt.Errorf("Can't set %s[%s].%s", obj.Kind(), obj.GetName(), k)
+			err = multierr.Append(err, e) // list of errors
+			continue
+		}
+
+		// if we can't interface, we can't compare...
+		if !value1.CanInterface() || !value2.CanInterface() {
+			e := fmt.Errorf("Can't interface %s[%s].%s", obj.Kind(), obj.GetName(), k)
+			err = multierr.Append(err, e) // list of errors
+			continue
+		}
+
+		// if the values aren't equal, we're changing the receiver
+		if !reflect.DeepEqual(value1.Interface(), value2.Interface()) {
+			// TODO: can we catch the panics here in case they happen?
+			value2.Set(value1) // do it for all types that match
+			updated[k] = true  // we updated this key!
+			v.Changed = true   // tag this key as updated!
+			log.Printf("SendRecv: %s[%s].%s -> %s[%s].%s", v.Res.Kind(), v.Res.GetName(), v.Key, obj.Kind(), obj.GetName(), k)
+		}
+	}
+	return updated, err
+}
+
+// TypeCmp compares two reflect values to see if they are the same Kind. It can
+// look into a ptr Kind to see if the underlying pair of ptr's can TypeCmp too!
+func TypeCmp(a, b reflect.Value) error {
+	ta, tb := a.Type(), b.Type()
+	if ta != tb {
+		return fmt.Errorf("Type mismatch: %s != %s", ta, tb)
+	}
+	// NOTE: it seems we don't need to recurse into pointers to sub check!
+
+	return nil // identical Type()'s
+}

resources/svc.go

@@ -21,10 +21,8 @@ package resources
 
 import (
 	"encoding/gob"
-	"errors"
 	"fmt"
 	"log"
-	"time"
 
 	"github.com/purpleidea/mgmt/event"
 	"github.com/purpleidea/mgmt/util"
@@ -32,6 +30,7 @@ import (
 	systemd "github.com/coreos/go-systemd/dbus" // change namespace
 	systemdUtil "github.com/coreos/go-systemd/util"
 	"github.com/godbus/dbus" // namespace collides with systemd wrapper
+	errwrap "github.com/pkg/errors"
 )
 
 func init() {
@@ -46,7 +45,7 @@ type SvcRes struct {
 }
 
 // NewSvcRes is a constructor for this resource. It also calls Init() for you.
-func NewSvcRes(name, state, startup string) *SvcRes {
+func NewSvcRes(name, state, startup string) (*SvcRes, error) {
 	obj := &SvcRes{
 		BaseRes: BaseRes{
 			Name: name,
@@ -54,8 +53,7 @@ func NewSvcRes(name, state, startup string) *SvcRes {
 		State:   state,
 		Startup: startup,
 	}
-	obj.Init()
+	return obj, obj.Init()
-	return obj
 }
 
 // Init runs some startup code for this resource.
@@ -77,22 +75,7 @@ func (obj *SvcRes) Validate() error {
 
 // Watch is the primary listener for this resource and it outputs events.
 func (obj *SvcRes) Watch(processChan chan event.Event) error {
-	if obj.IsWatching() {
+	cuid := obj.Converger() // get the converger uid used to report status
-		return nil
-	}
-	obj.SetWatching(true)
-	defer obj.SetWatching(false)
-	cuuid := obj.converger.Register()
-	defer cuuid.Unregister()
-
-	var startup bool
-	Startup := func(block bool) <-chan time.Time {
-		if block {
-			return nil // blocks forever
-			//return make(chan time.Time) // blocks forever
-		}
-		return time.After(time.Duration(500) * time.Millisecond) // 1/2 the resolution of converged timeout
-	}
 
 	// obj.Name: svc name
 	if !systemdUtil.IsRunningSystemd() {
@@ -101,14 +84,14 @@ func (obj *SvcRes) Watch(processChan chan event.Event) error {
 
 	conn, err := systemd.NewSystemdConnection() // needs root access
 	if err != nil {
-		return fmt.Errorf("Failed to connect to systemd: %s", err)
+		return errwrap.Wrapf(err, "Failed to connect to systemd")
 	}
 	defer conn.Close()
 
 	// if we share the bus with others, we will get each others messages!!
 	bus, err := util.SystemBusPrivateUsable() // don't share the bus connection!
 	if err != nil {
-		return fmt.Errorf("Failed to connect to bus: %s", err)
+		return errwrap.Wrapf(err, "Failed to connect to bus")
 	}
 
 	// XXX: will this detect new units?
@@ -117,10 +100,14 @@ func (obj *SvcRes) Watch(processChan chan event.Event) error {
 	buschan := make(chan *dbus.Signal, 10)
 	bus.Signal(buschan)
 
-	var svc = fmt.Sprintf("%v.service", obj.Name) // systemd name
+	// notify engine that we're running
+	if err := obj.Running(processChan); err != nil {
+		return err // bubble up a NACK...
+	}
+
+	var svc = fmt.Sprintf("%s.service", obj.Name) // systemd name
 	var send = false                              // send event?
 	var exit = false
-	var dirty = false
 	var invalid = false              // does the svc exist or not?
 	var previous bool                // previous invalid value
 	set := conn.NewSubscriptionSet() // no error should be returned
@@ -144,18 +131,18 @@ func (obj *SvcRes) Watch(processChan chan event.Event) error {
 		if !invalid {
 			var notFound = (loadstate.Value == dbus.MakeVariant("not-found"))
 			if notFound { // XXX: in the loop we'll handle changes better...
-				log.Printf("Failed to find svc: %v", svc)
+				log.Printf("Failed to find svc: %s", svc)
-				invalid = true // XXX ?
+				invalid = true // XXX: ?
 			}
 		}
 
 		if previous != invalid { // if invalid changed, send signal
 			send = true
-			dirty = true
+			obj.StateOK(false) // dirty
 		}
 
 		if invalid {
-			log.Printf("Waiting for: %v", svc) // waiting for svc to appear...
+			log.Printf("Waiting for: %s", svc) // waiting for svc to appear...
 			if activeSet {
 				activeSet = false
 				set.Remove(svc) // no return value should ever occur
@@ -163,28 +150,20 @@ func (obj *SvcRes) Watch(processChan chan event.Event) error {
 
 			obj.SetState(ResStateWatching) // reset
 			select {
-			case <-buschan: // XXX wait for new units event to unstick
+			case <-buschan: // XXX: wait for new units event to unstick
-				cuuid.SetConverged(false)
+				cuid.SetConverged(false)
 				// loop so that we can see the changed invalid signal
-				log.Printf("Svc[%v]->DaemonReload()", svc)
+				log.Printf("Svc[%s]->DaemonReload()", svc)
 
-			case event := <-obj.events:
+			case event := <-obj.Events():
-				cuuid.SetConverged(false)
+				cuid.SetConverged(false)
 				if exit, send = obj.ReadEvent(&event); exit {
 					return nil // exit
 				}
-				if event.GetActivity() {
-					dirty = true
-				}
 
-			case <-cuuid.ConvergedTimer():
+			case <-cuid.ConvergedTimer():
-				cuuid.SetConverged(true) // converged!
+				cuid.SetConverged(true) // converged!
 				continue
-
-			case <-Startup(startup):
-				cuuid.SetConverged(false)
-				send = true
-				dirty = true
 			}
 		} else {
 			if !activeSet {
@@ -192,7 +171,7 @@ func (obj *SvcRes) Watch(processChan chan event.Event) error {
 				set.Add(svc) // no return value should ever occur
 			}
 
-			log.Printf("Watching: %v", svc) // attempting to watch...
+			log.Printf("Watching: %s", svc) // attempting to watch...
 			obj.SetState(ResStateWatching)  // reset
 			select {
 			case event := <-subChannel:
@@ -204,52 +183,39 @@ func (obj *SvcRes) Watch(processChan chan event.Event) error {
 
 					switch event[svc].ActiveState {
 					case "active":
-						log.Printf("Svc[%v]->Started", svc)
+						log.Printf("Svc[%s]->Started", svc)
 					case "inactive":
-						log.Printf("Svc[%v]->Stopped", svc)
+						log.Printf("Svc[%s]->Stopped", svc)
 					case "reloading":
-						log.Printf("Svc[%v]->Reloading", svc)
+						log.Printf("Svc[%s]->Reloading", svc)
 					default:
 						log.Fatalf("Unknown svc state: %s", event[svc].ActiveState)
 					}
 				} else {
 					// svc stopped (and ActiveState is nil...)
-					log.Printf("Svc[%v]->Stopped", svc)
+					log.Printf("Svc[%s]->Stopped", svc)
 				}
 				send = true
-				dirty = true
+				obj.StateOK(false) // dirty
 
 			case err := <-subErrors:
-				cuuid.SetConverged(false)
+				cuid.SetConverged(false)
-				return fmt.Errorf("Unknown %s[%s] error: %v", obj.Kind(), obj.GetName(), err)
+				return errwrap.Wrapf(err, "Unknown %s[%s] error", obj.Kind(), obj.GetName())
 
-			case event := <-obj.events:
+			case event := <-obj.Events():
-				cuuid.SetConverged(false)
+				cuid.SetConverged(false)
 				if exit, send = obj.ReadEvent(&event); exit {
 					return nil // exit
 				}
-				if event.GetActivity() {
-					dirty = true
-				}
 
-			case <-cuuid.ConvergedTimer():
+			case <-cuid.ConvergedTimer():
-				cuuid.SetConverged(true) // converged!
+				cuid.SetConverged(true) // converged!
 				continue
-
-			case <-Startup(startup):
-				cuuid.SetConverged(false)
-				send = true
-				dirty = true
 			}
 		}
 
 		if send {
-			startup = true // startup finished
 			send = false
-			if dirty {
-				dirty = false
-				obj.isStateOK = false // something made state dirty
-			}
 			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
 				return err // we exit or bubble up a NACK...
 			}
@@ -259,34 +225,28 @@ func (obj *SvcRes) Watch(processChan chan event.Event) error {
 
 // CheckApply checks the resource state and applies the resource if the bool
 // input is true. It returns error info and if the state check passed or not.
-func (obj *SvcRes) CheckApply(apply bool) (checkok bool, err error) {
+func (obj *SvcRes) CheckApply(apply bool) (checkOK bool, err error) {
-	log.Printf("%v[%v]: CheckApply(%t)", obj.Kind(), obj.GetName(), apply)
-
-	if obj.isStateOK { // cache the state
-		return true, nil
-	}
-
 	if !systemdUtil.IsRunningSystemd() {
-		return false, errors.New("Systemd is not running.")
+		return false, fmt.Errorf("Systemd is not running.")
 	}
 
 	conn, err := systemd.NewSystemdConnection() // needs root access
 	if err != nil {
-		return false, fmt.Errorf("Failed to connect to systemd: %v", err)
+		return false, errwrap.Wrapf(err, "Failed to connect to systemd")
 	}
 	defer conn.Close()
 
-	var svc = fmt.Sprintf("%v.service", obj.Name) // systemd name
+	var svc = fmt.Sprintf("%s.service", obj.Name) // systemd name
 
 	loadstate, err := conn.GetUnitProperty(svc, "LoadState")
 	if err != nil {
-		return false, fmt.Errorf("Failed to get load state: %v", err)
+		return false, errwrap.Wrapf(err, "Failed to get load state")
 	}
 
 	// NOTE: we have to compare variants with other variants, they are really strings...
 	var notFound = (loadstate.Value == dbus.MakeVariant("not-found"))
 	if notFound {
-		return false, fmt.Errorf("Failed to find svc: %v", svc)
+		return false, errwrap.Wrapf(err, "Failed to find svc: %s", svc)
 	}
 
 	// XXX: check svc "enabled at boot" or not status...
@@ -294,14 +254,15 @@ func (obj *SvcRes) CheckApply(apply bool) (checkok bool, err error) {
 	//conn.GetUnitProperties(svc)
 	activestate, err := conn.GetUnitProperty(svc, "ActiveState")
 	if err != nil {
-		return false, fmt.Errorf("Failed to get active state: %v", err)
+		return false, errwrap.Wrapf(err, "Failed to get active state")
 	}
 
 	var running = (activestate.Value == dbus.MakeVariant("active"))
 	var stateOK = ((obj.State == "") || (obj.State == "running" && running) || (obj.State == "stopped" && !running))
-	var startupOK = true // XXX DETECT AND SET
+	var startupOK = true        // XXX: DETECT AND SET
+	var refresh = obj.Refresh() // do we have a pending reload to apply?
 
-	if stateOK && startupOK {
+	if stateOK && startupOK && !refresh {
 		return true, nil // we are in the correct state
 	}
 
@@ -311,7 +272,7 @@ func (obj *SvcRes) CheckApply(apply bool) (checkok bool, err error) {
 	}
 
 	// apply portion
-	log.Printf("%v[%v]: Apply", obj.Kind(), obj.GetName())
+	log.Printf("%s[%s]: Apply", obj.Kind(), obj.GetName())
 	var files = []string{svc} // the svc represented in a list
 	if obj.Startup == "enabled" {
 		_, _, err = conn.EnableUnitFiles(files, false, true)
@@ -321,7 +282,7 @@ func (obj *SvcRes) CheckApply(apply bool) (checkok bool, err error) {
 	}
 
 	if err != nil {
-		return false, fmt.Errorf("Unable to change startup status: %v", err)
+		return false, errwrap.Wrapf(err, "Unable to change startup status")
 	}
 
 	// XXX: do we need to use a buffered channel here?
@@ -330,41 +291,54 @@ func (obj *SvcRes) CheckApply(apply bool) (checkok bool, err error) {
 	if obj.State == "running" {
 		_, err = conn.StartUnit(svc, "fail", result)
 		if err != nil {
-			return false, fmt.Errorf("Failed to start unit: %v", err)
+			return false, errwrap.Wrapf(err, "Failed to start unit")
 		}
+		if refresh {
+			log.Printf("%s[%s]: Skipping reload, due to pending start", obj.Kind(), obj.GetName())
+		}
+		refresh = false // we did a start, so a reload is not needed
 	} else if obj.State == "stopped" {
 		_, err = conn.StopUnit(svc, "fail", result)
 		if err != nil {
-			return false, fmt.Errorf("Failed to stop unit: %v", err)
+			return false, errwrap.Wrapf(err, "Failed to stop unit")
 		}
+		if refresh {
+			log.Printf("%s[%s]: Skipping reload, due to pending stop", obj.Kind(), obj.GetName())
+		}
+		refresh = false // we did a stop, so a reload is not needed
 	}
 
 	status := <-result
 	if &status == nil {
-		return false, errors.New("Systemd service action result is nil")
+		return false, fmt.Errorf("Systemd service action result is nil")
 	}
 	if status != "done" {
 		return false, fmt.Errorf("Unknown systemd return string: %v", status)
 	}
 
+	if refresh { // we need to reload the service
+		// XXX: run a svc reload here!
+		log.Printf("%s[%s]: Reloading...", obj.Kind(), obj.GetName())
+	}
+
 	// XXX: also set enabled on boot
 
 	return false, nil // success
 }
 
-// SvcUUID is the UUID struct for SvcRes.
+// SvcUID is the UID struct for SvcRes.
-type SvcUUID struct {
+type SvcUID struct {
-	// NOTE: there is also a name variable in the BaseUUID struct, this is
+	// NOTE: there is also a name variable in the BaseUID struct, this is
-	// information about where this UUID came from, and is unrelated to the
+	// information about where this UID came from, and is unrelated to the
 	// information about the resource we're matching. That data which is
 	// used in the IFF function, is what you see in the struct fields here.
-	BaseUUID
+	BaseUID
 	name string // the svc name
 }
 
 // IFF aka if and only if they are equivalent, return true. If not, false.
-func (obj *SvcUUID) IFF(uuid ResUUID) bool {
+func (obj *SvcUID) IFF(uid ResUID) bool {
-	res, ok := uuid.(*SvcUUID)
+	res, ok := uid.(*SvcUID)
 	if !ok {
 		return false
 	}
@@ -373,13 +347,13 @@ func (obj *SvcUUID) IFF(uuid ResUUID) bool {
 
 // SvcResAutoEdges holds the state of the auto edge generator.
 type SvcResAutoEdges struct {
-	data    []ResUUID
+	data    []ResUID
 	pointer int
 	found   bool
 }
 
 // Next returns the next automatic edge.
-func (obj *SvcResAutoEdges) Next() []ResUUID {
+func (obj *SvcResAutoEdges) Next() []ResUID {
 	if obj.found {
 		log.Fatal("Shouldn't be called anymore!")
 	}
@@ -388,7 +362,7 @@ func (obj *SvcResAutoEdges) Next() []ResUUID {
 	}
 	value := obj.data[obj.pointer]
 	obj.pointer++
-	return []ResUUID{value} // we return one, even though api supports N
+	return []ResUID{value} // we return one, even though api supports N
 }
 
 // Test gets results of the earlier Next() call, & returns if we should continue!
@@ -412,15 +386,15 @@ func (obj *SvcResAutoEdges) Test(input []bool) bool {
 
 // AutoEdges returns the AutoEdge interface. In this case the systemd units.
 func (obj *SvcRes) AutoEdges() AutoEdge {
-	var data []ResUUID
+	var data []ResUID
 	svcFiles := []string{
 		fmt.Sprintf("/etc/systemd/system/%s.service", obj.Name),     // takes precedence
 		fmt.Sprintf("/usr/lib/systemd/system/%s.service", obj.Name), // pkg default
 	}
 	for _, x := range svcFiles {
 		var reversed = true
-		data = append(data, &FileUUID{
+		data = append(data, &FileUID{
-			BaseUUID: BaseUUID{
+			BaseUID: BaseUID{
 				name:     obj.GetName(),
 				kind:     obj.Kind(),
 				reversed: &reversed,
@@ -435,14 +409,14 @@ func (obj *SvcRes) AutoEdges() AutoEdge {
 	}
 }
 
-// GetUUIDs includes all params to make a unique identification of this object.
+// GetUIDs includes all params to make a unique identification of this object.
 // Most resources only return one, although some resources can return multiple.
-func (obj *SvcRes) GetUUIDs() []ResUUID {
+func (obj *SvcRes) GetUIDs() []ResUID {
-	x := &SvcUUID{
+	x := &SvcUID{
-		BaseUUID: BaseUUID{name: obj.GetName(), kind: obj.Kind()},
+		BaseUID: BaseUID{name: obj.GetName(), kind: obj.Kind()},
 		name:    obj.Name, // svc name
 	}
-	return []ResUUID{x}
+	return []ResUID{x}
 }
 
 // GroupCmp returns whether two resources can be grouped together or not.

resources/timer.go

@@ -33,24 +33,25 @@ func init() {
 type TimerRes struct {
 	BaseRes  `yaml:",inline"`
 	Interval int `yaml:"interval"` // Interval : Interval between runs
+
+	ticker *time.Ticker
 }
 
-// TimerUUID is the UUID struct for TimerRes.
+// TimerUID is the UID struct for TimerRes.
-type TimerUUID struct {
+type TimerUID struct {
-	BaseUUID
+	BaseUID
 	name string
 }
 
 // NewTimerRes is a constructor for this resource. It also calls Init() for you.
-func NewTimerRes(name string, interval int) *TimerRes {
+func NewTimerRes(name string, interval int) (*TimerRes, error) {
 	obj := &TimerRes{
 		BaseRes: BaseRes{
 			Name: name,
 		},
 		Interval: interval,
 	}
-	obj.Init()
+	return obj, obj.Init()
-	return obj
 }
 
 // Init runs some startup code for this resource.
@@ -66,54 +67,46 @@ func (obj *TimerRes) Validate() error {
 	return nil
 }
 
+// newTicker creates a new ticker
+func (obj *TimerRes) newTicker() *time.Ticker {
+	return time.NewTicker(time.Duration(obj.Interval) * time.Second)
+}
+
 // Watch is the primary listener for this resource and it outputs events.
 func (obj *TimerRes) Watch(processChan chan event.Event) error {
-	if obj.IsWatching() {
+	cuid := obj.Converger() // get the converger uid used to report status
-		return nil
-	}
-	obj.SetWatching(true)
-	defer obj.SetWatching(false)
-	cuuid := obj.converger.Register()
-	defer cuuid.Unregister()
 
-	var startup bool
+	// create a time.Ticker for the given interval
-	Startup := func(block bool) <-chan time.Time {
+	obj.ticker = obj.newTicker()
-		if block {
+	defer obj.ticker.Stop()
-			return nil // blocks forever
-			//return make(chan time.Time) // blocks forever
-		}
-		return time.After(time.Duration(500) * time.Millisecond) // 1/2 the resolution of converged timeout
-	}
 
-	// Create a time.Ticker for the given interval
+	// notify engine that we're running
-	ticker := time.NewTicker(time.Duration(obj.Interval) * time.Second)
+	if err := obj.Running(processChan); err != nil {
-	defer ticker.Stop()
+		return err // bubble up a NACK...
+	}
 
 	var send = false
 
 	for {
 		obj.SetState(ResStateWatching)
 		select {
-		case <-ticker.C: // received the timer event
+		case <-obj.ticker.C: // received the timer event
 			send = true
-			log.Printf("%v[%v]: received tick", obj.Kind(), obj.GetName())
+			log.Printf("%s[%s]: received tick", obj.Kind(), obj.GetName())
-		case event := <-obj.events:
+
-			cuuid.SetConverged(false)
+		case event := <-obj.Events():
+			cuid.SetConverged(false)
 			if exit, _ := obj.ReadEvent(&event); exit {
 				return nil
 			}
-		case <-cuuid.ConvergedTimer():
-			cuuid.SetConverged(true)
-			continue
 
-		case <-Startup(startup):
+		case <-cuid.ConvergedTimer():
-			cuuid.SetConverged(false)
+			cuid.SetConverged(true)
-			send = true
+			continue
 		}
+
 		if send {
-			startup = true // startup finished
 			send = false
-			obj.isStateOK = false
 			if exit, err := obj.DoSend(processChan, "timer ticked"); exit || err != nil {
 				return err // we exit or bubble up a NACK...
 			}
@@ -121,17 +114,33 @@ func (obj *TimerRes) Watch(processChan chan event.Event) error {
 	}
 }
 
-// GetUUIDs includes all params to make a unique identification of this object.
+// CheckApply method for Timer resource. Triggers a timer reset on notify.
+func (obj *TimerRes) CheckApply(apply bool) (bool, error) {
+	// because there are no checks to run, this resource has a less
+	// traditional pattern than what is seen in most resources...
+	if !obj.Refresh() { // this works for apply || !apply
+		return true, nil // state is always okay if no refresh to do
+	} else if !apply { // we had a refresh to do
+		return false, nil // therefore state is wrong
+	}
+
+	// reset the timer since apply && refresh
+	obj.ticker.Stop()
+	obj.ticker = obj.newTicker()
+	return false, nil
+}
+
+// GetUIDs includes all params to make a unique identification of this object.
 // Most resources only return one, although some resources can return multiple.
-func (obj *TimerRes) GetUUIDs() []ResUUID {
+func (obj *TimerRes) GetUIDs() []ResUID {
-	x := &TimerUUID{
+	x := &TimerUID{
-		BaseUUID: BaseUUID{
+		BaseUID: BaseUID{
 			name: obj.GetName(),
 			kind: obj.Kind(),
 		},
 		name: obj.Name,
 	}
-	return []ResUUID{x}
+	return []ResUID{x}
 }
 
 // AutoEdges returns the AutoEdge interface. In this case no autoedges are used.
@@ -158,9 +167,3 @@ func (obj *TimerRes) Compare(res Res) bool {
 	}
 	return true
 }
-
-// CheckApply method for Timer resource. Does nothing, returns happy!
-func (obj *TimerRes) CheckApply(apply bool) (bool, error) {
-	log.Printf("%v[%v]: CheckApply(%t)", obj.Kind(), obj.GetName(), apply)
-	return true, nil // state is always okay
-}

resources/virt.go

@@ -0,0 +1,763 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"encoding/gob"
+	"fmt"
+	"log"
+	"math/rand"
+	"net/url"
+	"time"
+
+	"github.com/purpleidea/mgmt/event"
+
+	errwrap "github.com/pkg/errors"
+	"github.com/rgbkrk/libvirt-go"
+)
+
+func init() {
+	gob.Register(&VirtRes{})
+}
+
+var (
+	libvirtInitialized = false
+)
+
+type virtURISchemeType int
+
+const (
+	defaultURI virtURISchemeType = iota
+	lxcURI
+)
+
+// VirtAuth is used to pass credentials to libvirt.
+type VirtAuth struct {
+	Username string `yaml:"username"`
+	Password string `yaml:"password"`
+}
+
+// VirtRes is a libvirt resource. A transient virt resource, which has its state
+// set to `shutoff` is one which does not exist. The parallel equivalent is a
+// file resource which removes a particular path.
+type VirtRes struct {
+	BaseRes    `yaml:",inline"`
+	URI        string             `yaml:"uri"`       // connection uri, eg: qemu:///session
+	State      string             `yaml:"state"`     // running, paused, shutoff
+	Transient  bool               `yaml:"transient"` // defined (false) or undefined (true)
+	CPUs       uint16             `yaml:"cpus"`
+	Memory     uint64             `yaml:"memory"` // in KBytes
+	OSInit     string             `yaml:"osinit"` // init used by lxc
+	Boot       []string           `yaml:"boot"`   // boot order. values: fd, hd, cdrom, network
+	Disk       []diskDevice       `yaml:"disk"`
+	CDRom      []cdRomDevice      `yaml:"cdrom"`
+	Network    []networkDevice    `yaml:"network"`
+	Filesystem []filesystemDevice `yaml:"filesystem"`
+	Auth       *VirtAuth          `yaml:"auth"`
+
+	conn      libvirt.VirConnection
+	absent    bool // cached state
+	uriScheme virtURISchemeType
+}
+
+// NewVirtRes is a constructor for this resource. It also calls Init() for you.
+func NewVirtRes(name string, uri, state string, transient bool, cpus uint16, memory uint64, osinit string) (*VirtRes, error) {
+	obj := &VirtRes{
+		BaseRes: BaseRes{
+			Name: name,
+		},
+		URI:       uri,
+		State:     state,
+		Transient: transient,
+		CPUs:      cpus,
+		Memory:    memory,
+		OSInit:    osinit,
+	}
+	return obj, obj.Init()
+}
+
+// Init runs some startup code for this resource.
+func (obj *VirtRes) Init() error {
+	if !libvirtInitialized {
+		if err := libvirt.EventRegisterDefaultImpl(); err != nil {
+			return errwrap.Wrapf(err, "EventRegisterDefaultImpl failed")
+		}
+		libvirtInitialized = true
+	}
+	var u *url.URL
+	var err error
+	if u, err = url.Parse(obj.URI); err != nil {
+		return errwrap.Wrapf(err, "%s[%s]: Parsing URI failed: %s", obj.Kind(), obj.GetName(), obj.URI)
+	}
+	switch u.Scheme {
+	case "lxc":
+		obj.uriScheme = lxcURI
+	}
+
+	obj.absent = (obj.Transient && obj.State == "shutoff") // machine shouldn't exist
+
+	obj.BaseRes.kind = "Virt"
+	return obj.BaseRes.Init() // call base init, b/c we're overriding
+}
+
+// Validate if the params passed in are valid data.
+func (obj *VirtRes) Validate() error {
+	return nil
+}
+
+func (obj *VirtRes) connect() (conn libvirt.VirConnection, err error) {
+	if obj.Auth != nil {
+		conn, err = libvirt.NewVirConnectionWithAuth(obj.URI, obj.Auth.Username, obj.Auth.Password)
+	}
+	if obj.Auth == nil || err != nil {
+		conn, err = libvirt.NewVirConnection(obj.URI)
+	}
+	return
+}
+
+// Watch is the primary listener for this resource and it outputs events.
+func (obj *VirtRes) Watch(processChan chan event.Event) error {
+	cuid := obj.Converger() // get the converger uid used to report status
+
+	conn, err := obj.connect()
+	if err != nil {
+		return fmt.Errorf("Connection to libvirt failed with: %s", err)
+	}
+
+	eventChan := make(chan int) // TODO: do we need to buffer this?
+	errorChan := make(chan error)
+	exitChan := make(chan struct{})
+	defer close(exitChan)
+
+	// run libvirt event loop
+	// TODO: *trigger* EventRunDefaultImpl to unblock so it can shut down...
+	// at the moment this isn't a major issue because it seems to unblock in
+	// bursts every 5 seconds! we can do this by writing to an event handler
+	// in the meantime, terminating the program causes it to exit anyways...
+	go func() {
+		for {
+			// TODO: can we merge this into our main for loop below?
+			select {
+			case <-exitChan:
+				log.Printf("EventRunDefaultImpl exited!")
+				return
+			default:
+			}
+			//log.Printf("EventRunDefaultImpl started!")
+			if err := libvirt.EventRunDefaultImpl(); err != nil {
+				errorChan <- errwrap.Wrapf(err, "EventRunDefaultImpl failed")
+				return
+			}
+			//log.Printf("EventRunDefaultImpl looped!")
+		}
+	}()
+
+	callback := libvirt.DomainEventCallback(
+		func(c *libvirt.VirConnection, d *libvirt.VirDomain, eventDetails interface{}, f func()) int {
+			if lifecycleEvent, ok := eventDetails.(libvirt.DomainLifecycleEvent); ok {
+				domName, _ := d.GetName()
+				if domName == obj.GetName() {
+					eventChan <- lifecycleEvent.Event
+				}
+			} else if obj.debug {
+				log.Printf("%s[%s]: Event details isn't DomainLifecycleEvent", obj.Kind(), obj.GetName())
+			}
+			return 0
+		},
+	)
+	callbackID := conn.DomainEventRegister(
+		libvirt.VirDomain{},
+		libvirt.VIR_DOMAIN_EVENT_ID_LIFECYCLE,
+		&callback,
+		nil,
+	)
+	defer conn.DomainEventDeregister(callbackID)
+
+	// notify engine that we're running
+	if err := obj.Running(processChan); err != nil {
+		return err // bubble up a NACK...
+	}
+
+	var send = false
+	var exit = false
+
+	for {
+		select {
+		case event := <-eventChan:
+			// TODO: shouldn't we do these checks in CheckApply ?
+			switch event {
+			case libvirt.VIR_DOMAIN_EVENT_DEFINED:
+				if obj.Transient {
+					obj.StateOK(false) // dirty
+					send = true
+				}
+			case libvirt.VIR_DOMAIN_EVENT_UNDEFINED:
+				if !obj.Transient {
+					obj.StateOK(false) // dirty
+					send = true
+				}
+			case libvirt.VIR_DOMAIN_EVENT_STARTED:
+				fallthrough
+			case libvirt.VIR_DOMAIN_EVENT_RESUMED:
+				if obj.State != "running" {
+					obj.StateOK(false) // dirty
+					send = true
+				}
+			case libvirt.VIR_DOMAIN_EVENT_SUSPENDED:
+				if obj.State != "paused" {
+					obj.StateOK(false) // dirty
+					send = true
+				}
+			case libvirt.VIR_DOMAIN_EVENT_STOPPED:
+				fallthrough
+			case libvirt.VIR_DOMAIN_EVENT_SHUTDOWN:
+				if obj.State != "shutoff" {
+					obj.StateOK(false) // dirty
+					send = true
+				}
+			case libvirt.VIR_DOMAIN_EVENT_PMSUSPENDED:
+				fallthrough
+			case libvirt.VIR_DOMAIN_EVENT_CRASHED:
+				obj.StateOK(false) // dirty
+				send = true
+			}
+
+		case err := <-errorChan:
+			cuid.SetConverged(false)
+			return fmt.Errorf("Unknown %s[%s] libvirt error: %s", obj.Kind(), obj.GetName(), err)
+
+		case event := <-obj.Events():
+			cuid.SetConverged(false)
+			if exit, send = obj.ReadEvent(&event); exit {
+				return nil // exit
+			}
+
+		case <-cuid.ConvergedTimer():
+			cuid.SetConverged(true) // converged!
+			continue
+		}
+
+		if send {
+			send = false
+			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
+				return err // we exit or bubble up a NACK...
+			}
+		}
+	}
+}
+
+// attrCheckApply performs the CheckApply functions for CPU, Memory and others.
+// This shouldn't be called when the machine is absent; it won't be found!
+func (obj *VirtRes) attrCheckApply(apply bool) (bool, error) {
+	var checkOK = true
+
+	dom, err := obj.conn.LookupDomainByName(obj.GetName())
+	if err != nil {
+		return false, errwrap.Wrapf(err, "conn.LookupDomainByName failed")
+	}
+
+	domInfo, err := dom.GetInfo()
+	if err != nil {
+		// we don't know if the state is ok
+		return false, errwrap.Wrapf(err, "domain.GetInfo failed")
+	}
+
+	// check memory
+	if domInfo.GetMemory() != obj.Memory {
+		checkOK = false
+		if !apply {
+			return false, nil
+		}
+		if err := dom.SetMemory(obj.Memory); err != nil {
+			return false, errwrap.Wrapf(err, "domain.SetMemory failed")
+		}
+		log.Printf("%s[%s]: Memory changed", obj.Kind(), obj.GetName())
+	}
+
+	// check cpus
+	if domInfo.GetNrVirtCpu() != obj.CPUs {
+		checkOK = false
+		if !apply {
+			return false, nil
+		}
+		if err := dom.SetVcpus(obj.CPUs); err != nil {
+			return false, errwrap.Wrapf(err, "domain.SetVcpus failed")
+		}
+		log.Printf("%s[%s]: CPUs changed", obj.Kind(), obj.GetName())
+	}
+
+	return checkOK, nil
+}
+
+// domainCreate creates a transient or persistent domain in the correct state. It
+// doesn't check the state before hand, as it is a simple helper function.
+func (obj *VirtRes) domainCreate() (libvirt.VirDomain, bool, error) {
+
+	if obj.Transient {
+		var flag uint32
+		var state string
+		switch obj.State {
+		case "running":
+			flag = libvirt.VIR_DOMAIN_NONE
+			state = "started"
+		case "paused":
+			flag = libvirt.VIR_DOMAIN_START_PAUSED
+			state = "paused"
+		case "shutoff":
+			// a transient, shutoff machine, means machine is absent
+			return libvirt.VirDomain{}, true, nil // returned dom is invalid
+		}
+		dom, err := obj.conn.DomainCreateXML(obj.getDomainXML(), flag)
+		if err != nil {
+			return dom, false, err // returned dom is invalid
+		}
+		log.Printf("%s[%s]: Domain transient %s", state, obj.Kind(), obj.GetName())
+		return dom, false, nil
+	}
+
+	dom, err := obj.conn.DomainDefineXML(obj.getDomainXML())
+	if err != nil {
+		return dom, false, err // returned dom is invalid
+	}
+	log.Printf("%s[%s]: Domain defined", obj.Kind(), obj.GetName())
+
+	if obj.State == "running" {
+		if err := dom.Create(); err != nil {
+			return dom, false, err
+		}
+		log.Printf("%s[%s]: Domain started", obj.Kind(), obj.GetName())
+	}
+
+	if obj.State == "paused" {
+		if err := dom.CreateWithFlags(libvirt.VIR_DOMAIN_START_PAUSED); err != nil {
+			return dom, false, err
+		}
+		log.Printf("%s[%s]: Domain created paused", obj.Kind(), obj.GetName())
+	}
+
+	return dom, false, nil
+}
+
+// CheckApply checks the resource state and applies the resource if the bool
+// input is true. It returns error info and if the state check passed or not.
+func (obj *VirtRes) CheckApply(apply bool) (bool, error) {
+	var err error
+	obj.conn, err = obj.connect()
+	if err != nil {
+		return false, fmt.Errorf("Connection to libvirt failed with: %s", err)
+	}
+
+	var checkOK = true
+
+	dom, err := obj.conn.LookupDomainByName(obj.GetName())
+	if err == nil {
+		// pass
+	} else if virErr, ok := err.(libvirt.VirError); ok && virErr.Code == libvirt.VIR_ERR_NO_DOMAIN {
+		// domain not found
+		if obj.absent {
+			return true, nil
+		}
+
+		if !apply {
+			return false, nil
+		}
+
+		var c = true
+		dom, c, err = obj.domainCreate() // create the domain
+		if err != nil {
+			return false, errwrap.Wrapf(err, "domainCreate failed")
+		} else if !c {
+			checkOK = false
+		}
+
+	} else {
+		return false, errwrap.Wrapf(err, "LookupDomainByName failed")
+	}
+	defer dom.Free()
+	// domain exists
+
+	domInfo, err := dom.GetInfo()
+	if err != nil {
+		// we don't know if the state is ok
+		return false, errwrap.Wrapf(err, "domain.GetInfo failed")
+	}
+	isPersistent, err := dom.IsPersistent()
+	if err != nil {
+		// we don't know if the state is ok
+		return false, errwrap.Wrapf(err, "domain.IsPersistent failed")
+	}
+	isActive, err := dom.IsActive()
+	if err != nil {
+		// we don't know if the state is ok
+		return false, errwrap.Wrapf(err, "domain.IsActive failed")
+	}
+
+	// check for persistence
+	if isPersistent == obj.Transient { // if they're different!
+		if !apply {
+			return false, nil
+		}
+		if isPersistent {
+			if err := dom.Undefine(); err != nil {
+				return false, errwrap.Wrapf(err, "domain.Undefine failed")
+			}
+			log.Printf("%s[%s]: Domain undefined", obj.Kind(), obj.GetName())
+		} else {
+			domXML, err := dom.GetXMLDesc(libvirt.VIR_DOMAIN_XML_INACTIVE)
+			if err != nil {
+				return false, errwrap.Wrapf(err, "domain.GetXMLDesc failed")
+			}
+			if _, err = obj.conn.DomainDefineXML(domXML); err != nil {
+				return false, errwrap.Wrapf(err, "conn.DomainDefineXML failed")
+			}
+			log.Printf("%s[%s]: Domain defined", obj.Kind(), obj.GetName())
+		}
+		checkOK = false
+	}
+
+	// check for valid state
+	domState := domInfo.GetState()
+	switch obj.State {
+	case "running":
+		if domState == libvirt.VIR_DOMAIN_RUNNING {
+			break
+		}
+		if domState == libvirt.VIR_DOMAIN_BLOCKED {
+			// TODO: what should happen?
+			return false, fmt.Errorf("Domain %s is blocked!", obj.GetName())
+		}
+		if !apply {
+			return false, nil
+		}
+		if isActive { // domain must be paused ?
+			if err := dom.Resume(); err != nil {
+				return false, errwrap.Wrapf(err, "domain.Resume failed")
+			}
+			checkOK = false
+			log.Printf("%s[%s]: Domain resumed", obj.Kind(), obj.GetName())
+			break
+		}
+		if err := dom.Create(); err != nil {
+			return false, errwrap.Wrapf(err, "domain.Create failed")
+		}
+		checkOK = false
+		log.Printf("%s[%s]: Domain created", obj.Kind(), obj.GetName())
+
+	case "paused":
+		if domState == libvirt.VIR_DOMAIN_PAUSED {
+			break
+		}
+		if !apply {
+			return false, nil
+		}
+		if isActive { // domain must be running ?
+			if err := dom.Suspend(); err != nil {
+				return false, errwrap.Wrapf(err, "domain.Suspend failed")
+			}
+			checkOK = false
+			log.Printf("%s[%s]: Domain paused", obj.Kind(), obj.GetName())
+			break
+		}
+		if err := dom.CreateWithFlags(libvirt.VIR_DOMAIN_START_PAUSED); err != nil {
+			return false, errwrap.Wrapf(err, "domain.CreateWithFlags failed")
+		}
+		checkOK = false
+		log.Printf("%s[%s]: Domain created paused", obj.Kind(), obj.GetName())
+
+	case "shutoff":
+		if domState == libvirt.VIR_DOMAIN_SHUTOFF || domState == libvirt.VIR_DOMAIN_SHUTDOWN {
+			break
+		}
+		if !apply {
+			return false, nil
+		}
+
+		if err := dom.Destroy(); err != nil {
+			return false, errwrap.Wrapf(err, "domain.Destroy failed")
+		}
+		checkOK = false
+		log.Printf("%s[%s]: Domain destroyed", obj.Kind(), obj.GetName())
+	}
+
+	if !apply {
+		return false, nil
+	}
+	// remaining apply portion
+
+	// mem & cpu checks...
+	if !obj.absent {
+		if c, err := obj.attrCheckApply(apply); err != nil {
+			return false, errwrap.Wrapf(err, "attrCheckApply failed")
+		} else if !c {
+			checkOK = false
+		}
+	}
+
+	return checkOK, nil // w00t
+}
+
+// Return the correct domain type based on the uri
+func (obj VirtRes) getDomainType() string {
+	switch obj.uriScheme {
+	case lxcURI:
+		return "<domain type='lxc'>"
+	default:
+		return "<domain type='kvm'>"
+	}
+}
+
+// Return the correct os type based on the uri
+func (obj VirtRes) getOSType() string {
+	switch obj.uriScheme {
+	case lxcURI:
+		return "<type>exe</type>"
+	default:
+		return "<type>hvm</type>"
+	}
+}
+
+func (obj VirtRes) getOSInit() string {
+	switch obj.uriScheme {
+	case lxcURI:
+		return fmt.Sprintf("<init>%s</init>", obj.OSInit)
+	default:
+		return ""
+	}
+}
+
+func (obj *VirtRes) getDomainXML() string {
+	var b string
+	b += obj.getDomainType() // start domain
+
+	b += fmt.Sprintf("<name>%s</name>", obj.GetName())
+	b += fmt.Sprintf("<memory unit='KiB'>%d</memory>", obj.Memory)
+	b += fmt.Sprintf("<vcpu>%d</vcpu>", obj.CPUs)
+
+	b += "<os>"
+	b += obj.getOSType()
+	b += obj.getOSInit()
+	if obj.Boot != nil {
+		for _, boot := range obj.Boot {
+			b += fmt.Sprintf("<boot dev='%s'/>", boot)
+		}
+	}
+	b += fmt.Sprintf("</os>")
+
+	b += fmt.Sprintf("<devices>") // start devices
+
+	if obj.Disk != nil {
+		for i, disk := range obj.Disk {
+			b += fmt.Sprintf(disk.GetXML(i))
+		}
+	}
+
+	if obj.CDRom != nil {
+		for i, cdrom := range obj.CDRom {
+			b += fmt.Sprintf(cdrom.GetXML(i))
+		}
+	}
+
+	if obj.Network != nil {
+		for i, net := range obj.Network {
+			b += fmt.Sprintf(net.GetXML(i))
+		}
+	}
+
+	if obj.Filesystem != nil {
+		for i, fs := range obj.Filesystem {
+			b += fmt.Sprintf(fs.GetXML(i))
+		}
+	}
+
+	b += "<serial type='pty'><target port='0'/></serial>"
+	b += "<console type='pty'><target type='serial' port='0'/></console>"
+	b += "</devices>" // end devices
+	b += "</domain>"  // end domain
+	return b
+}
+
+type virtDevice interface {
+	GetXML(idx int) string
+}
+
+type diskDevice struct {
+	Source string `yaml:"source"`
+	Type   string `yaml:"type"`
+}
+
+type cdRomDevice struct {
+	Source string `yaml:"source"`
+	Type   string `yaml:"type"`
+}
+
+type networkDevice struct {
+	Name string `yaml:"name"`
+	MAC  string `yaml:"mac"`
+}
+
+type filesystemDevice struct {
+	Access   string `yaml:"access"`
+	Source   string `yaml:"source"`
+	Target   string `yaml:"target"`
+	ReadOnly bool   `yaml:"read_only"`
+}
+
+func (d *diskDevice) GetXML(idx int) string {
+	var b string
+	b += "<disk type='file' device='disk'>"
+	b += fmt.Sprintf("<driver name='qemu' type='%s'/>", d.Type)
+	b += fmt.Sprintf("<source file='%s'/>", d.Source)
+	b += fmt.Sprintf("<target dev='vd%s' bus='virtio'/>", (string)(idx+97)) // TODO: 26, 27... should be 'aa', 'ab'...
+	b += "</disk>"
+	return b
+}
+
+func (d *cdRomDevice) GetXML(idx int) string {
+	var b string
+	b += "<disk type='file' device='cdrom'>"
+	b += fmt.Sprintf("<driver name='qemu' type='%s'/>", d.Type)
+	b += fmt.Sprintf("<source file='%s'/>", d.Source)
+	b += fmt.Sprintf("<target dev='hd%s' bus='ide'/>", (string)(idx+97)) // TODO: 26, 27... should be 'aa', 'ab'...
+	b += "<readonly/>"
+	b += "</disk>"
+	return b
+}
+
+func (d *networkDevice) GetXML(idx int) string {
+	if d.MAC == "" {
+		d.MAC = randMAC()
+	}
+	var b string
+	b += "<interface type='network'>"
+	b += fmt.Sprintf("<mac address='%s'/>", d.MAC)
+	b += fmt.Sprintf("<source network='%s'/>", d.Name)
+	b += "</interface>"
+	return b
+}
+
+func (d *filesystemDevice) GetXML(idx int) string {
+	var b string
+	b += "<filesystem" // open
+	if d.Access != "" {
+		b += fmt.Sprintf(" accessmode='%s'", d.Access)
+	}
+	b += ">" // close
+	b += fmt.Sprintf("<source dir='%s'/>", d.Source)
+	b += fmt.Sprintf("<target dir='%s'/>", d.Target)
+	if d.ReadOnly {
+		b += "<readonly/>"
+	}
+	b += "</filesystem>"
+	return b
+}
+
+// VirtUID is the UID struct for FileRes.
+type VirtUID struct {
+	BaseUID
+}
+
+// GetUIDs includes all params to make a unique identification of this object.
+// Most resources only return one, although some resources can return multiple.
+func (obj *VirtRes) GetUIDs() []ResUID {
+	x := &VirtUID{
+		BaseUID: BaseUID{name: obj.GetName(), kind: obj.Kind()},
+		// TODO: add more properties here so we can link to vm dependencies
+	}
+	return []ResUID{x}
+}
+
+// GroupCmp returns whether two resources can be grouped together or not.
+func (obj *VirtRes) GroupCmp(r Res) bool {
+	_, ok := r.(*VirtRes)
+	if !ok {
+		return false
+	}
+	return false // not possible atm
+}
+
+// AutoEdges returns the AutoEdge interface. In this case no autoedges are used.
+func (obj *VirtRes) AutoEdges() AutoEdge {
+	return nil
+}
+
+// Compare two resources and return if they are equivalent.
+func (obj *VirtRes) Compare(res Res) bool {
+	switch res.(type) {
+	case *VirtRes:
+		res := res.(*VirtRes)
+		if !obj.BaseRes.Compare(res) { // call base Compare
+			return false
+		}
+
+		if obj.Name != res.Name {
+			return false
+		}
+		if obj.URI != res.URI {
+			return false
+		}
+		if obj.State != res.State {
+			return false
+		}
+		if obj.Transient != res.Transient {
+			return false
+		}
+		if obj.CPUs != res.CPUs {
+			return false
+		}
+		// TODO: can we skip the compare of certain properties such as
+		// Memory because this object (but with different memory) can be
+		// *converted* into the new version that has more/less memory?
+		// We would need to run some sort of "old struct update", to get
+		// the new values, but that's easy to add.
+		if obj.Memory != res.Memory {
+			return false
+		}
+		// TODO:
+		//if obj.Boot != res.Boot {
+		//	return false
+		//}
+		//if obj.Disk != res.Disk {
+		//	return false
+		//}
+		//if obj.CDRom != res.CDRom {
+		//	return false
+		//}
+		//if obj.Network != res.Network {
+		//	return false
+		//}
+		//if obj.Filesystem != res.Filesystem {
+		//	return false
+		//}
+	default:
+		return false
+	}
+	return true
+}
+
+// CollectPattern applies the pattern for collection resources.
+func (obj *VirtRes) CollectPattern(string) {
+}
+
+// randMAC returns a random mac address in the libvirt range.
+func randMAC() string {
+	rand.Seed(time.Now().UnixNano())
+	return "52:54:00" +
+		fmt.Sprintf(":%x", rand.Intn(255)) +
+		fmt.Sprintf(":%x", rand.Intn(255)) +
+		fmt.Sprintf(":%x", rand.Intn(255))
+}

spec.in

@@ -12,11 +12,14 @@ Source0:	https://dl.fedoraproject.org/pub/alt/purpleidea/__PROGRAM__/SOURCES/__P
 # graphviz should really be a "suggests", since technically it's optional
 Requires: graphviz
 
-BuildRequires:	golang
+# If go_compiler is not set to 1, there is no virtual provide. Use golang instead.
+BuildRequires:  %{?go_compiler:compiler(go-compiler)}%{!?go_compiler:golang}
 BuildRequires:	golang-googlecode-tools-stringer
 BuildRequires:	git-core
 BuildRequires:	mercurial
 
+ExclusiveArch:  %{go_arches}
+
 %description
 A next generation config management prototype!
 

test/omv/pkg1a.yaml

@@ -32,7 +32,7 @@
   - iptables -F
   - cd /vagrant/mgmt/ && make path
   - cd /vagrant/mgmt/ && make deps && make build && cp mgmt ~/bin/
-  - cd && mgmt run --file /vagrant/mgmt/examples/pkg1.yaml --converged-timeout=5
+  - cd && mgmt run --yaml /vagrant/mgmt/examples/pkg1.yaml --converged-timeout=5
 :namespace: omv
 :count: 0
 :username: ''

test/omv/pkg1b.yaml

@@ -33,7 +33,7 @@
   - iptables -F
   - cd /vagrant/mgmt/ && make path
   - cd /vagrant/mgmt/ && make deps && make build && cp mgmt ~/bin/
-  - cd && mgmt run --file /vagrant/mgmt/examples/pkg1.yaml --converged-timeout=5
+  - cd && mgmt run --yaml /vagrant/mgmt/examples/pkg1.yaml --converged-timeout=5
 :namespace: omv
 :count: 0
 :username: ''

test/shell/t2.sh

@@ -7,7 +7,7 @@ if env | grep -q -e '^TRAVIS=true$'; then
 fi
 
 # run till completion
-timeout --kill-after=15s 10s ./mgmt run --file t2.yaml --converged-timeout=5 --no-watch --tmp-prefix &
+timeout --kill-after=15s 10s ./mgmt run --yaml t2.yaml --converged-timeout=5 --no-watch --tmp-prefix &
 pid=$!
 wait $pid	# get exit status
 e=$?

test/shell/t3.sh

@@ -10,11 +10,11 @@ fi
 mkdir -p "${MGMT_TMPDIR}"mgmt{A..C}
 
 # run till completion
-timeout --kill-after=15s 10s ./mgmt run --file t3-a.yaml --converged-timeout=5 --no-watch --tmp-prefix &
+timeout --kill-after=15s 10s ./mgmt run --yaml t3-a.yaml --converged-timeout=5 --no-watch --tmp-prefix &
 pid1=$!
-timeout --kill-after=15s 10s ./mgmt run --file t3-b.yaml --converged-timeout=5 --no-watch --tmp-prefix &
+timeout --kill-after=15s 10s ./mgmt run --yaml t3-b.yaml --converged-timeout=5 --no-watch --tmp-prefix &
 pid2=$!
-timeout --kill-after=15s 10s ./mgmt run --file t3-c.yaml --converged-timeout=5 --no-watch --tmp-prefix &
+timeout --kill-after=15s 10s ./mgmt run --yaml t3-c.yaml --converged-timeout=5 --no-watch --tmp-prefix &
 pid3=$!
 
 wait $pid1	# get exit status

test/shell/t4.sh

@@ -1,7 +1,7 @@
 #!/bin/bash -e
 
 # should take slightly more than 25s, but fail if we take 35s)
-timeout --kill-after=35s 30s ./mgmt run --file t4.yaml --converged-timeout=5 --no-watch --tmp-prefix &
+timeout --kill-after=35s 30s ./mgmt run --yaml t4.yaml --converged-timeout=5 --no-watch --tmp-prefix &
 pid=$!
 wait $pid	# get exit status
 exit $?

test/shell/t5.sh

@@ -1,7 +1,7 @@
 #!/bin/bash -e
 
 # should take slightly more than 35s, but fail if we take 45s)
-timeout --kill-after=45s 40s ./mgmt run --file t5.yaml --converged-timeout=5 --no-watch --tmp-prefix &
+timeout --kill-after=45s 40s ./mgmt run --yaml t5.yaml --converged-timeout=5 --no-watch --tmp-prefix &
 pid=$!
 wait $pid	# get exit status
 exit $?

test/shell/t6.sh

@@ -7,7 +7,7 @@ if env | grep -q -e '^TRAVIS=true$'; then
 fi
 
 # run till completion
-timeout --kill-after=20s 15s ./mgmt run --file t6.yaml --no-watch --tmp-prefix &
+timeout --kill-after=20s 15s ./mgmt run --yaml t6.yaml --no-watch --tmp-prefix &
 pid=$!
 sleep 1s	# let it converge
 test -e /tmp/mgmt/f1

test/test-gofmt.sh

@@ -9,9 +9,9 @@ ROOT=$(dirname "${BASH_SOURCE}")/..
 
 GO_VERSION=($(go version))
 
-if [[ -z $(echo "${GO_VERSION[2]}" | grep -E 'go1.2|go1.3|go1.4|go1.5|go1.6') ]]; then
+if [[ -z $(echo "${GO_VERSION[2]}" | grep -E 'go1.2|go1.3|go1.4|go1.5|go1.6|go1.7|go1.8|devel') ]]; then
-	echo "Unknown go version '${GO_VERSION}', skipping gofmt."
+	echo "Unknown go version '${GO_VERSION[2]}', failing gofmt."
-	exit 0
+	exit 1
 fi
 
 cd "${ROOT}"

test/test-golint.sh

@@ -20,9 +20,10 @@ if [ "$COMMITS" != "" ] && [ "$COMMITS" -gt "1" ]; then
 	HACK="yes"
 fi
 
-LINT=`golint`	# current golint output
+LINT=`find . -maxdepth 3 -iname '*.go' -not -path './old/*' -not -path './tmp/*' -exec golint {} \;`	# current golint output
 COUNT=`echo -e "$LINT" | wc -l`	# number of golint problems in current branch
 [ "$LINT" = "" ] && echo PASS && exit	# everything is "perfect"
+echo "$LINT"	# display the issues
 
 T=`mktemp --tmpdir -d tmp.XXX`
 [ "$T" = "" ] && exit 1
@@ -46,7 +47,7 @@ while read -r line; do
 done <<< "$NUMSTAT1"	# three < is the secret to putting a variable into read
 
 git checkout "$PREVIOUS" &>/dev/null	# previous commit
-LINT1=`golint`
+LINT1=`find . -maxdepth 3 -iname '*.go' -not -path './old/*' -not -path './tmp/*' -exec golint {} \;`
 COUNT1=`echo -e "$LINT1" | wc -l`	# number of golint problems in older branch
 
 # clean up

test/test-govet.sh

@@ -8,4 +8,5 @@ for file in `find . -maxdepth 3 -type f -name '*.go' -not -path './old/*' -not -
 	go vet "$file" && echo PASS || exit 1	# since it doesn't output an ok message on pass
 	grep 'log.' "$file" | grep '\\n"' && echo 'no \n needed in log.Printf()' && exit 1 || echo PASS	# no \n needed in log.Printf()
 	grep 'case _ = <-' "$file" && echo 'case _ = <- can be simplified to: case <-' && exit 1 || echo PASS	# this can be simplified
+	grep -Ei "[\/]+[\/]+[ ]*+(FIXME[^:]|TODO[^:]|XXX[^:])" "$file" && echo 'Token is missing a colon' && exit 1 || echo PASS	# tokens must end with a colon
 done

test/test-headerfmt.sh

@@ -11,7 +11,7 @@ done < "$FILE"
 cd "${ROOT}"
 
 find_files() {
-	git ls-files | grep '\.go$'
+	git ls-files | grep '\.go$' | grep -v '^examples/'
 }
 
 bad_files=$(

yamlgraph/gapi.go

@@ -0,0 +1,121 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package yamlgraph
+
+import (
+	"fmt"
+	"log"
+	"sync"
+
+	"github.com/purpleidea/mgmt/gapi"
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/recwatch"
+)
+
+// GAPI implements the main yamlgraph GAPI interface.
+type GAPI struct {
+	File *string // yaml graph definition to use; nil if undefined
+
+	data        gapi.Data
+	initialized bool
+	closeChan   chan struct{}
+	wg          sync.WaitGroup // sync group for tunnel go routines
+}
+
+// NewGAPI creates a new yamlgraph GAPI struct and calls Init().
+func NewGAPI(data gapi.Data, file *string) (*GAPI, error) {
+	obj := &GAPI{
+		File: file,
+	}
+	return obj, obj.Init(data)
+}
+
+// Init initializes the yamlgraph GAPI struct.
+func (obj *GAPI) Init(data gapi.Data) error {
+	if obj.initialized {
+		return fmt.Errorf("Already initialized!")
+	}
+	if obj.File == nil {
+		return fmt.Errorf("The File param must be specified!")
+	}
+	obj.data = data // store for later
+	obj.closeChan = make(chan struct{})
+	obj.initialized = true
+	return nil
+}
+
+// Graph returns a current Graph.
+func (obj *GAPI) Graph() (*pgraph.Graph, error) {
+	if !obj.initialized {
+		return nil, fmt.Errorf("yamlgraph: GAPI is not initialized")
+	}
+
+	config := ParseConfigFromFile(*obj.File)
+	if config == nil {
+		return nil, fmt.Errorf("yamlgraph: ParseConfigFromFile returned nil")
+	}
+
+	g, err := config.NewGraphFromConfig(obj.data.Hostname, obj.data.World, obj.data.Noop)
+	return g, err
+}
+
+// Next returns nil errors every time there could be a new graph.
+func (obj *GAPI) Next() chan error {
+	if obj.data.NoWatch {
+		return nil
+	}
+	ch := make(chan error)
+	obj.wg.Add(1)
+	go func() {
+		defer obj.wg.Done()
+		defer close(ch) // this will run before the obj.wg.Done()
+		if !obj.initialized {
+			ch <- fmt.Errorf("yamlgraph: GAPI is not initialized")
+			return
+		}
+		configWatcher := recwatch.NewConfigWatcher()
+		configChan := configWatcher.ConfigWatch(*obj.File) // simple
+		for {
+			select {
+			case err, ok := <-configChan: // returns nil events on ok!
+				if !ok { // the channel closed!
+					return
+				}
+				log.Printf("yamlgraph: Generating new graph...")
+				ch <- err // trigger a run
+				if err != nil {
+					return
+				}
+			case <-obj.closeChan:
+				return
+			}
+		}
+	}()
+	return ch
+}
+
+// Close shuts down the yamlgraph GAPI.
+func (obj *GAPI) Close() error {
+	if !obj.initialized {
+		return fmt.Errorf("yamlgraph: GAPI is not initialized")
+	}
+	close(obj.closeChan)
+	obj.wg.Wait()
+	obj.initialized = false // closed = true
+	return nil
+}

yamlgraph/gconfig.go

@@ -15,8 +15,8 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-// Package gconfig provides the facilities for loading a graph from a yaml file.
+// Package yamlgraph provides the facilities for loading a graph from a yaml file.
-package gconfig
+package yamlgraph
 
 import (
 	"errors"
@@ -26,9 +26,7 @@ import (
 	"reflect"
 	"strings"
 
-	"github.com/purpleidea/mgmt/etcd"
+	"github.com/purpleidea/mgmt/gapi"
-	"github.com/purpleidea/mgmt/event"
-	"github.com/purpleidea/mgmt/global"
 	"github.com/purpleidea/mgmt/pgraph"
 	"github.com/purpleidea/mgmt/resources"
 	"github.com/purpleidea/mgmt/util"
@@ -41,33 +39,42 @@ type collectorResConfig struct {
 	Pattern string `yaml:"pattern"` // XXX: Not Implemented
 }
 
-type vertexConfig struct {
+// Vertex is the data structure of a vertex.
+type Vertex struct {
 	Kind string `yaml:"kind"`
 	Name string `yaml:"name"`
 }
 
-type edgeConfig struct {
+// Edge is the data structure of an edge.
+type Edge struct {
 	Name string `yaml:"name"`
-	From vertexConfig `yaml:"from"`
+	From Vertex `yaml:"from"`
-	To   vertexConfig `yaml:"to"`
+	To   Vertex `yaml:"to"`
+}
+
+// Resources is the data structure of the set of resources.
+type Resources struct {
+	// in alphabetical order
+	Exec     []*resources.ExecRes     `yaml:"exec"`
+	File     []*resources.FileRes     `yaml:"file"`
+	Hostname []*resources.HostnameRes `yaml:"hostname"`
+	Msg      []*resources.MsgRes      `yaml:"msg"`
+	Noop     []*resources.NoopRes     `yaml:"noop"`
+	Nspawn   []*resources.NspawnRes   `yaml:"nspawn"`
+	Password []*resources.PasswordRes `yaml:"password"`
+	Pkg      []*resources.PkgRes      `yaml:"pkg"`
+	Svc      []*resources.SvcRes      `yaml:"svc"`
+	Timer    []*resources.TimerRes    `yaml:"timer"`
+	Virt     []*resources.VirtRes     `yaml:"virt"`
 }
 
 // GraphConfig is the data structure that describes a single graph to run.
 type GraphConfig struct {
 	Graph     string               `yaml:"graph"`
-	Resources struct {
+	Resources Resources            `yaml:"resources"`
-		Noop  []*resources.NoopRes  `yaml:"noop"`
-		Pkg   []*resources.PkgRes   `yaml:"pkg"`
-		File  []*resources.FileRes  `yaml:"file"`
-		Svc   []*resources.SvcRes   `yaml:"svc"`
-		Exec  []*resources.ExecRes  `yaml:"exec"`
-		Timer []*resources.TimerRes `yaml:"timer"`
-		Msg   []*resources.MsgRes   `yaml:"msg"`
-	} `yaml:"resources"`
 	Collector []collectorResConfig `yaml:"collect"`
-	Edges     []edgeConfig         `yaml:"edges"`
+	Edges     []Edge               `yaml:"edges"`
 	Comment   string               `yaml:"comment"`
-	Hostname  string               `yaml:"hostname"` // uuid for the host
 	Remote    string               `yaml:"remote"`
 }
 
@@ -82,36 +89,13 @@ func (c *GraphConfig) Parse(data []byte) error {
 	return nil
 }
 
-// ParseConfigFromFile takes a filename and returns the graph config structure.
+// NewGraphFromConfig transforms a GraphConfig struct into a new graph.
-func ParseConfigFromFile(filename string) *GraphConfig {
+// FIXME: remove any possibly left over, now obsolete graph diff code from here!
-	data, err := ioutil.ReadFile(filename)
+func (c *GraphConfig) NewGraphFromConfig(hostname string, world gapi.World, noop bool) (*pgraph.Graph, error) {
-	if err != nil {
+	// hostname is the uuid for the host
-		log.Printf("Config: Error: ParseConfigFromFile: File: %v", err)
-		return nil
-	}
-
-	var config GraphConfig
-	if err := config.Parse(data); err != nil {
-		log.Printf("Config: Error: ParseConfigFromFile: Parse: %v", err)
-		return nil
-	}
-
-	return &config
-}
-
-// NewGraphFromConfig returns a new graph from existing input, such as from the
-// existing graph, and a GraphConfig struct.
-func (c *GraphConfig) NewGraphFromConfig(g *pgraph.Graph, embdEtcd *etcd.EmbdEtcd, noop bool) (*pgraph.Graph, error) {
-	if c.Hostname == "" {
-		return nil, fmt.Errorf("Config: Error: Hostname can't be empty!")
-	}
 
 	var graph *pgraph.Graph          // new graph to return
-	if g == nil {           // FIXME: how can we check for an empty graph?
 	graph = pgraph.NewGraph("Graph") // give graph a default name
-	} else {
-		graph = g.Copy() // same vertices, since they're pointers!
-	}
 
 	var lookup = make(map[string]map[string]*pgraph.Vertex)
 
@@ -132,27 +116,21 @@ func (c *GraphConfig) NewGraphFromConfig(g *pgraph.Graph, embdEtcd *etcd.EmbdEtc
 		slice := reflect.ValueOf(iface)
 		// XXX: should we just drop these everywhere and have the kind strings be all lowercase?
 		kind := util.FirstToUpper(name)
-		if global.DEBUG {
-			log.Printf("Config: Processing: %v...", kind)
-		}
 		for j := 0; j < slice.Len(); j++ { // loop through resources of same kind
 			x := slice.Index(j).Interface()
 			res, ok := x.(resources.Res) // convert to Res type
 			if !ok {
 				return nil, fmt.Errorf("Config: Error: Can't convert: %v of type: %T to Res.", x, x)
 			}
-			if noop {
+			//if noop { // now done in mgmtmain
-				res.Meta().Noop = noop
+			//	res.Meta().Noop = noop
-			}
+			//}
 			if _, exists := lookup[kind]; !exists {
 				lookup[kind] = make(map[string]*pgraph.Vertex)
 			}
 			// XXX: should we export based on a @@ prefix, or a metaparam
 			// like exported => true || exported => (host pattern)||(other pattern?)
 			if !strings.HasPrefix(res.GetName(), "@@") { // not exported resource
-				// XXX: we don't have a way of knowing if any of the
-				// metaparams are undefined, and as a result to set the
-				// defaults that we want! I hate the go yaml parser!!!
 				v := graph.GetVertexMatch(res)
 				if v == nil { // no match found
 					res.Init()
@@ -163,19 +141,19 @@ func (c *GraphConfig) NewGraphFromConfig(g *pgraph.Graph, embdEtcd *etcd.EmbdEtc
 				keep = append(keep, v)          // append
 
 			} else if !noop { // do not export any resources if noop
-				// store for addition to etcd storage...
+				// store for addition to backend storage...
 				res.SetName(res.GetName()[2:]) //slice off @@
 				res.SetKind(kind)              // cheap init
 				resourceList = append(resourceList, res)
 			}
 		}
 	}
-	// store in etcd
+	// store in backend (usually etcd)
-	if err := etcd.EtcdSetResources(embdEtcd, c.Hostname, resourceList); err != nil {
+	if err := world.ResExport(resourceList); err != nil {
 		return nil, fmt.Errorf("Config: Could not export resources: %v", err)
 	}
 
-	// lookup from etcd
+	// lookup from backend (usually etcd)
 	var hostnameFilter []string // empty to get from everyone
 	kindFilter := []string{}
 	for _, t := range c.Collector {
@@ -183,11 +161,11 @@ func (c *GraphConfig) NewGraphFromConfig(g *pgraph.Graph, embdEtcd *etcd.EmbdEtc
 		kind := util.FirstToUpper(t.Kind)
 		kindFilter = append(kindFilter, kind)
 	}
-	// do all the graph look ups in one single step, so that if the etcd
+	// do all the graph look ups in one single step, so that if the backend
 	// database changes, we don't have a partial state of affairs...
 	if len(kindFilter) > 0 { // if kindFilter is empty, don't need to do lookups!
 		var err error
-		resourceList, err = etcd.EtcdGetResources(embdEtcd, hostnameFilter, kindFilter)
+		resourceList, err = world.ResCollect(hostnameFilter, kindFilter)
 		if err != nil {
 			return nil, fmt.Errorf("Config: Could not collect resources: %v", err)
 		}
@@ -198,7 +176,7 @@ func (c *GraphConfig) NewGraphFromConfig(g *pgraph.Graph, embdEtcd *etcd.EmbdEtc
 		for _, t := range c.Collector {
 			// XXX: should we just drop these everywhere and have the kind strings be all lowercase?
 			kind := util.FirstToUpper(t.Kind)
-			// use t.Kind and optionally t.Pattern to collect from etcd storage
+			// use t.Kind and optionally t.Pattern to collect from storage
 			log.Printf("Collect: %v; Pattern: %v", kind, t.Pattern)
 
 			// XXX: expand to more complex pattern matching here...
@@ -213,9 +191,9 @@ func (c *GraphConfig) NewGraphFromConfig(g *pgraph.Graph, embdEtcd *etcd.EmbdEtc
 			matched = true
 
 			// collect resources but add the noop metaparam
-			if noop {
+			//if noop { // now done in mgmtmain
-				res.Meta().Noop = noop
+			//	res.Meta().Noop = noop
-			}
+			//}
 
 			if t.Pattern != "" { // XXX: simplistic for now
 				res.CollectPattern(t.Pattern) // res.Dirname = t.Pattern
@@ -240,15 +218,6 @@ func (c *GraphConfig) NewGraphFromConfig(g *pgraph.Graph, embdEtcd *etcd.EmbdEtc
 		}
 	}
 
-	// get rid of any vertices we shouldn't "keep" (that aren't in new graph)
-	for _, v := range graph.GetVertices() {
-		if !pgraph.VertexContains(v, keep) {
-			// wait for exit before starting new graph!
-			v.SendEvent(event.EventExit, true, false)
-			graph.DeleteVertex(v)
-		}
-	}
-
 	for _, e := range c.Edges {
 		if _, ok := lookup[util.FirstToUpper(e.From.Kind)]; !ok {
 			return nil, fmt.Errorf("Can't find 'from' resource!")
@@ -267,3 +236,20 @@ func (c *GraphConfig) NewGraphFromConfig(g *pgraph.Graph, embdEtcd *etcd.EmbdEtc
 
 	return graph, nil
 }
+
+// ParseConfigFromFile takes a filename and returns the graph config structure.
+func ParseConfigFromFile(filename string) *GraphConfig {
+	data, err := ioutil.ReadFile(filename)
+	if err != nil {
+		log.Printf("Config: Error: ParseConfigFromFile: File: %v", err)
+		return nil
+	}
+
+	var config GraphConfig
+	if err := config.Parse(data); err != nil {
+		log.Printf("Config: Error: ParseConfigFromFile: Parse: %v", err)
+		return nil
+	}
+
+	return &config
+}