lang: unification: Improve type unification algorithm

The simple type unification algorithm suffered from some serious
performance and memory problems when used with certain code bases. This
adds some crucial optimizations that improve performance drastically.

.github/FUNDING.yml

@@ -0,0 +1,2 @@
+# You can add one username per supported platform and one custom link
+patreon: purpleidea

.travis.yml

@@ -1,10 +1,6 @@
 language: go
 os:
   - linux
-go:
-  - 1.10.x
-  - 1.11.x
-  - tip
 go_import_path: github.com/purpleidea/mgmt
 sudo: true
 dist: xenial
@@ -25,17 +21,27 @@ before_install:
   - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
   - git fetch --unshallow
 install: 'make deps'
-script: 'make test'
 matrix:
   fast_finish: false
   allow_failures:
-    - go: 1.11.x
-    - go: tip
-    - os: osx
+  - go: 1.11.x
+  - go: tip
+  - os: osx
   # include only one build for osx for a quicker build as the nr. of these runners are sparse
   include:
-    - os: osx
-      go: 1.10.x
+  - name: "basic tests"
+    go: 1.10.x
+    env: TEST_BLOCK=basic
+  - name: "shell tests"
+    go: 1.10.x
+    env: TEST_BLOCK=shell
+  - name: "race tests"
+    go: 1.10.x
+    env: TEST_BLOCK=race
+  - go: 1.11.x
+  - go: tip
+  - os: osx
+script: 'TEST_BLOCK="$TEST_BLOCK" make test'
 
 # the "secure" channel value is the result of running: ./misc/travis-encrypt.sh
 # with a value of: irc.freenode.net#mgmtconfig to eliminate noise from forks...

COPYRIGHT

@@ -1,5 +1,5 @@
 Mgmt
-Copyright (C) 2013-2018+ James Shubin and the project contributors
+Copyright (C) 2013-2019+ James Shubin and the project contributors
 Written by James Shubin <james@shubin.ca> and the project contributors
 
 This program is free software: you can redistribute it and/or modify

Makefile

@@ -1,5 +1,5 @@
 # Mgmt
-# Copyright (C) 2013-2018+ James Shubin and the project contributors
+# Copyright (C) 2013-2019+ James Shubin and the project contributors
 # Written by James Shubin <james@shubin.ca> and the project contributors
 #
 # This program is free software: you can redistribute it and/or modify
@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 SHELL = /usr/bin/env bash
-.PHONY: all art cleanart version program lang path deps run race bindata generate build build-debug crossbuild clean test gofmt yamlfmt format docs rpmbuild mkdirs rpm srpm spec tar upload upload-sources upload-srpms upload-rpms copr tag release
+.PHONY: all art cleanart version program lang path deps run race bindata generate build build-debug crossbuild clean test gofmt yamlfmt format docs rpmbuild mkdirs rpm srpm spec tar upload upload-sources upload-srpms upload-rpms copr tag release funcgen
 .SILENT: clean bindata
 
 # a large amount of output from this `find`, can cause `make` to be much slower!
@@ -117,7 +117,6 @@ race:
 
 # generate go files from non-go source
 bindata: ## generate go files from non-go sources
-	@echo "Generating: bindata..."
 	$(MAKE) --quiet -C bindata
 	$(MAKE) --quiet -C lang/funcs
 
@@ -126,8 +125,7 @@ generate:
 
 lang: ## generates the lexer/parser for the language frontend
 	@# recursively run make in child dir named lang
-	@echo "Generating: lang..."
-	$(MAKE) --quiet -C lang
+	@$(MAKE) --quiet -C lang
 
 # build a `mgmt` binary for current host os/arch
 $(PROGRAM): build/mgmt-${GOHOSTOS}-${GOHOSTARCH} ## build an mgmt binary for current host os/arch
@@ -148,7 +146,7 @@ build-debug: $(PROGRAM)
 # extract os and arch from target pattern
 GOOS=$(firstword $(subst -, ,$*))
 GOARCH=$(lastword $(subst -, ,$*))
-build/mgmt-%: $(GO_FILES) | bindata lang
+build/mgmt-%: $(GO_FILES) | bindata lang funcgen
 	@echo "Building: $(PROGRAM), os/arch: $*, version: $(SVERSION)..."
 	@# reassigning GOOS and GOARCH to make build command copy/pastable
 	@# go 1.10 requires specifying the package for ldflags
@@ -166,6 +164,8 @@ clean: ## clean things up
 	$(MAKE) --quiet -C bindata clean
 	$(MAKE) --quiet -C lang/funcs clean
 	$(MAKE) --quiet -C lang clean
+	rm -f lang/funcs/core/generated_funcs.go || true
+	rm -f lang/funcs/core/generated_funcs_test.go || true
 	[ ! -e $(PROGRAM) ] || rm $(PROGRAM)
 	rm -f *_stringer.go	# generated by `go generate`
 	rm -f *_mock.go		# generated by `go generate`
@@ -188,8 +188,8 @@ $(addprefix test-shell-,${test_shell}): test-shell-%: build
 
 gofmt:
 	# TODO: remove gofmt once goimports has a -s option
-	find . -maxdepth 6 -type f -name '*.go' -not -path './old/*' -not -path './tmp/*' -not -path './vendor/*' -exec gofmt -s -w {} \;
-	find . -maxdepth 6 -type f -name '*.go' -not -path './old/*' -not -path './tmp/*' -not -path './vendor/*' -exec goimports -w {} \;
+	find . -maxdepth 9 -type f -name '*.go' -not -path './old/*' -not -path './tmp/*' -not -path './vendor/*' -exec gofmt -s -w {} \;
+	find . -maxdepth 9 -type f -name '*.go' -not -path './old/*' -not -path './tmp/*' -not -path './vendor/*' -exec goimports -w {} \;
 
 yamlfmt:
 	find . -maxdepth 3 -type f -name '*.yaml' -not -path './old/*' -not -path './tmp/*' -not -path './omv.yaml' -exec ruby -e "require 'yaml'; x=YAML.load_file('{}').to_yaml.each_line.map(&:rstrip).join(10.chr)+10.chr; File.open('{}', 'w').write x" \;
@@ -360,28 +360,28 @@ releases/$(VERSION)/.mkdir:
 	mkdir -p releases/$(VERSION)/{deb,rpm,pacman}/ && touch releases/$(VERSION)/.mkdir
 
 releases/$(VERSION)/rpm/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir
-	@echo "Generating rpm changelog..."
+	@echo "Generating: rpm changelog..."
 	./misc/make-rpm-changelog.sh $(VERSION)
 
 $(RPM_PKG): releases/$(VERSION)/rpm/changelog
-	@echo "Building rpm package..."
+	@echo "Building: rpm package..."
 	./misc/fpm-pack.sh rpm $(VERSION) libvirt-devel augeas-devel
 
 releases/$(VERSION)/deb/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir
-	@echo "Generating deb changelog..."
+	@echo "Generating: deb changelog..."
 	./misc/make-deb-changelog.sh $(VERSION)
 
 $(DEB_PKG): releases/$(VERSION)/deb/changelog
-	@echo "Building deb package..."
+	@echo "Building: deb package..."
 	./misc/fpm-pack.sh deb $(VERSION) libvirt-dev libaugeas-dev
 
 $(PACMAN_PKG): $(PROGRAM) releases/$(VERSION)/.mkdir
-	@echo "Building pacman package..."
+	@echo "Building: pacman package..."
 	./misc/fpm-pack.sh pacman $(VERSION) libvirt augeas
 
 $(SHA256SUMS): $(RPM_PKG) $(DEB_PKG) $(PACMAN_PKG)
 	@# remove the directory separator in the SHA256SUMS file
-	@echo "Generating sha256 sum..."
+	@echo "Generating: sha256 sum..."
 	sha256sum $(RPM_PKG) $(DEB_PKG) $(PACMAN_PKG) | awk -F '/| ' '{print $$1"  "$$6}' > $(SHA256SUMS)
 
 $(SHA256SUMS_ASC): $(SHA256SUMS)
@@ -408,4 +408,14 @@ help: ## show this help screen
 		awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
 	@echo ''
 
+funcgen: lang/funcs/core/generated_funcs_test.go lang/funcs/core/generated_funcs.go
+
+lang/funcs/core/generated_funcs_test.go: lang/funcs/funcgen/*.go lang/funcs/core/funcgen.yaml lang/funcs/funcgen/templates/generated_funcs_test.go.tpl
+	@echo "Generating: funcs test..."
+	@go run lang/funcs/funcgen/*.go -templates lang/funcs/funcgen/templates/generated_funcs_test.go.tpl 2>/dev/null
+
+lang/funcs/core/generated_funcs.go: lang/funcs/funcgen/*.go lang/funcs/core/funcgen.yaml lang/funcs/funcgen/templates/generated_funcs.go.tpl
+	@echo "Generating: funcs..."
+	@go run lang/funcs/funcgen/*.go -templates lang/funcs/funcgen/templates/generated_funcs.go.tpl 2>/dev/null
+
 # vim: ts=8

bindata/Makefile

@@ -1,5 +1,5 @@
 # Mgmt
-# Copyright (C) 2013-2018+ James Shubin and the project contributors
+# Copyright (C) 2013-2019+ James Shubin and the project contributors
 # Written by James Shubin <james@shubin.ca> and the project contributors
 #
 # This program is free software: you can redistribute it and/or modify
@@ -30,6 +30,7 @@ build: bindata.go
 
 # add more input files as dependencies at the end here...
 bindata.go: ../COPYING
+	@echo "Generating: bindata..."
 	# go-bindata --pkg bindata -o <OUTPUT> <INPUT>
 	go-bindata --pkg bindata -o ./$@ $^
 	# gofmt the output file

converger/converger.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -25,139 +25,251 @@ import (
 	"time"
 
 	"github.com/purpleidea/mgmt/util"
-
-	multierr "github.com/hashicorp/go-multierror"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
-// TODO: we could make a new function that masks out the state of certain
-// UID's, but at the moment the new Timer code has obsoleted the need...
+// New builds a new converger coordinator.
+func New(timeout int64) *Coordinator {
+	return &Coordinator{
+		timeout: timeout,
 
-// Converger is the general interface for implementing a convergence watcher.
-type Converger interface { // TODO: need a better name
-	Register() UID
-	IsConverged(UID) bool         // is the UID converged ?
-	SetConverged(UID, bool) error // set the converged state of the UID
-	Unregister(UID)
-	Start()
-	Pause()
-	Loop(bool)
-	ConvergedTimer(UID) <-chan time.Time
-	Status() map[uint64]bool
-	Timeout() int                              // returns the timeout that this was created with
-	AddStateFn(string, func(bool) error) error // adds a stateFn with a name
-	RemoveStateFn(string) error                // remove a stateFn with a given name
-}
+		mutex: &sync.RWMutex{},
 
-// UID is the interface resources can use to notify with if converged. You'll
-// need to use part of the Converger interface to Register initially too.
-type UID interface {
-	ID() uint64   // get Id
-	Name() string // get a friendly name
-	SetName(string)
-	IsValid() bool // has Id been initialized ?
-	InvalidateID() // set Id to nil
-	IsConverged() bool
-	SetConverged(bool) error
-	Unregister()
-	ConvergedTimer() <-chan time.Time
-	StartTimer() (func() error, error) // cancellable is the same as StopTimer()
-	ResetTimer() error                 // resets counter to zero
-	StopTimer() error
-}
+		//lastid: 0,
+		status: make(map[*UID]struct{}),
 
-// converger is an implementation of the Converger interface.
-type converger struct {
-	timeout   int           // must be zero (instant) or greater seconds to run
-	converged bool          // did we converge (state changes of this run Fn)
-	channel   chan struct{} // signal here to run an isConverged check
-	control   chan bool     // control channel for start/pause
-	mutex     *sync.RWMutex // used for controlling access to status and lastid
-	lastid    uint64
-	status    map[uint64]bool
-	stateFns  map[string]func(bool) error // run on converged state changes with state bool
-	smutex    *sync.RWMutex               // used for controlling access to stateFns
-}
+		//converged: false, // initial state
 
-// cuid is an implementation of the UID interface.
-type cuid struct {
-	converger Converger
-	id        uint64
-	name      string // user defined, friendly name
-	mutex     *sync.Mutex
-	timer     chan struct{}
-	running   bool // is the above timer running?
-	wg        *sync.WaitGroup
-}
+		pokeChan: make(chan struct{}, 1), // must be buffered
+
+		readyChan: make(chan struct{}), // ready signal
+
+		//paused: false, // starts off as started
+		pauseSignal: make(chan struct{}),
+		//resumeSignal: make(chan struct{}), // happens on pause
+		//pausedAck: util.NewEasyAck(), // happens on pause
 
-// NewConverger builds a new converger struct.
-func NewConverger(timeout int) Converger {
-	return &converger{
-		timeout:  timeout,
-		channel:  make(chan struct{}),
-		control:  make(chan bool),
-		mutex:    &sync.RWMutex{},
-		lastid:   0,
-		status:   make(map[uint64]bool),
 		stateFns: make(map[string]func(bool) error),
 		smutex:   &sync.RWMutex{},
-	}
-}
 
-// Register assigns a UID to the caller.
-func (obj *converger) Register() UID {
-	obj.mutex.Lock()
-	defer obj.mutex.Unlock()
-	obj.lastid++
-	obj.status[obj.lastid] = false // initialize as not converged
-	return &cuid{
-		converger: obj,
-		id:        obj.lastid,
-		name:      fmt.Sprintf("%d", obj.lastid), // some default
-		mutex:     &sync.Mutex{},
-		timer:     nil,
-		running:   false,
+		closeChan: make(chan struct{}),
 		wg:        &sync.WaitGroup{},
 	}
 }
 
-// IsConverged gets the converged status of a uid.
-func (obj *converger) IsConverged(uid UID) bool {
-	if !uid.IsValid() {
-		panic(fmt.Sprintf("the ID of UID(%s) is nil", uid.Name()))
-	}
-	obj.mutex.RLock()
-	isConverged, found := obj.status[uid.ID()] // lookup
-	obj.mutex.RUnlock()
-	if !found {
-		panic("the ID of UID is unregistered")
-	}
-	return isConverged
+// Coordinator is the central converger engine.
+type Coordinator struct {
+	// timeout must be zero (instant) or greater seconds to run. If it's -1
+	// then this is disabled, and we never run stateFns.
+	timeout int64
+
+	// mutex is used for controlling access to status and lastid.
+	mutex *sync.RWMutex
+
+	// lastid contains the last uid we used for registration.
+	//lastid    uint64
+	// status contains a reference to each active UID.
+	status map[*UID]struct{}
+
+	// converged stores the last convergence state. When this changes, we
+	// run the stateFns.
+	converged bool
+
+	// pokeChan receives a message every time we might need to re-calculate.
+	pokeChan chan struct{}
+
+	// readyChan closes to notify any interested parties that the main loop
+	// is running.
+	readyChan chan struct{}
+
+	// paused represents if this coordinator is paused or not.
+	paused bool
+	// pauseSignal closes to request a pause of this coordinator.
+	pauseSignal chan struct{}
+	// resumeSignal closes to request a resume of this coordinator.
+	resumeSignal chan struct{}
+	// pausedAck is used to send an ack message saying that we've paused.
+	pausedAck *util.EasyAck
+
+	// stateFns run on converged state changes.
+	stateFns map[string]func(bool) error
+	// smutex is used for controlling access to the stateFns map.
+	smutex *sync.RWMutex
+
+	// closeChan closes when we've been requested to shutdown.
+	closeChan chan struct{}
+	// wg waits for everything to finish.
+	wg *sync.WaitGroup
 }
 
-// SetConverged updates the converger with the converged state of the UID.
-func (obj *converger) SetConverged(uid UID, isConverged bool) error {
-	if !uid.IsValid() {
-		return fmt.Errorf("the ID of UID(%s) is nil", uid.Name())
-	}
+// Register creates a new UID which can be used to report converged state. You
+// must Unregister each UID before Shutdown will be able to finish running.
+func (obj *Coordinator) Register() *UID {
+	obj.wg.Add(1) // additional tracking for each UID
 	obj.mutex.Lock()
-	if _, found := obj.status[uid.ID()]; !found {
-		panic("the ID of UID is unregistered")
+	defer obj.mutex.Unlock()
+	//obj.lastid++
+	uid := &UID{
+		timeout: obj.timeout, // copy the timeout here
+		//id: obj.lastid,
+		//name: fmt.Sprintf("%d", obj.lastid), // some default
+
+		poke: obj.poke,
+
+		// timer
+		mutex:   &sync.Mutex{},
+		timer:   nil,
+		running: false,
+		wg:      &sync.WaitGroup{},
 	}
-	obj.status[uid.ID()] = isConverged // set
-	obj.mutex.Unlock()                 // unlock *before* poke or deadlock!
-	if isConverged != obj.converged {  // only poke if it would be helpful
-		// run in a go routine so that we never block... just queue up!
-		// this allows us to send events, even if we haven't started...
-		go func() { obj.channel <- struct{}{} }()
+	uid.unregister = func() { obj.Unregister(uid) } // add unregister func
+	obj.status[uid] = struct{}{}                    // TODO: add converged state here?
+	return uid
+}
+
+// Unregister removes the UID from the converger coordinator. If you supply an
+// invalid or unregistered uid to this function, it will panic. An unregistered
+// UID is no longer part of the convergence checking.
+func (obj *Coordinator) Unregister(uid *UID) {
+	defer obj.wg.Done() // additional tracking for each UID
+	obj.mutex.Lock()
+	defer obj.mutex.Unlock()
+
+	if _, exists := obj.status[uid]; !exists {
+		panic("uid is not registered")
 	}
+	uid.StopTimer() // ignore any errors
+	delete(obj.status, uid)
+}
+
+// Run starts the main loop for the converger coordinator. It is commonly run
+// from a go routine. It blocks until the Shutdown method is run to close it.
+// NOTE: when we have very short timeouts, if we start before all the resources
+// have joined the map, then it might appear as if we converged before we did!
+func (obj *Coordinator) Run(startPaused bool) {
+	obj.wg.Add(1)
+	wg := &sync.WaitGroup{} // needed for the startPaused
+	defer wg.Wait()         // don't leave any leftover go routines running
+	if startPaused {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			obj.Pause() // ignore any errors
+			close(obj.readyChan)
+		}()
+	} else {
+		close(obj.readyChan) // we must wait till the wg.Add(1) has happened...
+	}
+	defer obj.wg.Done()
+	for {
+		// pause if one was requested...
+		select {
+		case <-obj.pauseSignal: // channel closes
+			obj.pausedAck.Ack() // send ack
+			// we are paused now, and waiting for resume or exit...
+			select {
+			case <-obj.resumeSignal: // channel closes
+				// resumed!
+
+			case <-obj.closeChan: // we can always escape
+				return
+			}
+
+		case _, ok := <-obj.pokeChan: // we got an event (re-calculate)
+			if !ok {
+				return
+			}
+
+			if err := obj.test(); err != nil {
+				// FIXME: what to do on error ?
+			}
+
+		case <-obj.closeChan: // we can always escape
+			return
+		}
+	}
+}
+
+// Ready blocks until the Run loop has started up. This is useful so that we
+// don't run Shutdown before we've even started up properly.
+func (obj *Coordinator) Ready() {
+	select {
+	case <-obj.readyChan:
+	}
+}
+
+// Shutdown sends a signal to the Run loop that it should exit. This blocks
+// until it does.
+func (obj *Coordinator) Shutdown() {
+	close(obj.closeChan)
+	obj.wg.Wait()
+	close(obj.pokeChan) // free memory?
+}
+
+// Pause pauses the coordinator. It should not be called on an already paused
+// coordinator. It will block until the coordinator pauses with an
+// acknowledgment, or until an exit is requested. If the latter happens it will
+// error. It is NOT thread-safe with the Resume() method so only call either one
+// at a time.
+func (obj *Coordinator) Pause() error {
+	if obj.paused {
+		return fmt.Errorf("already paused")
+	}
+
+	obj.pausedAck = util.NewEasyAck()
+	obj.resumeSignal = make(chan struct{}) // build the resume signal
+	close(obj.pauseSignal)
+
+	// wait for ack (or exit signal)
+	select {
+	case <-obj.pausedAck.Wait(): // we got it!
+		// we're paused
+	case <-obj.closeChan:
+		return fmt.Errorf("closing")
+	}
+	obj.paused = true
+
 	return nil
 }
 
-// isConverged returns true if *every* registered uid has converged.
-func (obj *converger) isConverged() bool {
-	obj.mutex.RLock() // take a read lock
-	defer obj.mutex.RUnlock()
-	for _, v := range obj.status {
+// Resume unpauses the coordinator. It can be safely called on a brand-new
+// coordinator that has just started running without incident. It is NOT
+// thread-safe with the Pause() method, so only call either one at a time.
+func (obj *Coordinator) Resume() {
+	// TODO: do we need a mutex around Resume?
+	if !obj.paused { // no need to unpause brand-new resources
+		return
+	}
+
+	obj.pauseSignal = make(chan struct{}) // rebuild for next pause
+	close(obj.resumeSignal)
+	obj.poke() // unblock and notice the resume if necessary
+
+	obj.paused = false
+
+	// no need to wait for it to resume
+	//return // implied
+}
+
+// poke sends a message to the coordinator telling it that it should re-evaluate
+// whether we're converged or not. This does not block. Do not run this in a
+// goroutine. It must not be called after Shutdown has been called.
+func (obj *Coordinator) poke() {
+	// redundant
+	//if len(obj.pokeChan) > 0 {
+	//	return
+	//}
+
+	select {
+	case obj.pokeChan <- struct{}{}:
+	default: // if chan is now full because more than one poke happened...
+	}
+}
+
+// IsConverged returns true if *every* registered uid has converged. If there
+// are no registered UID's, then this will return true.
+func (obj *Coordinator) IsConverged() bool {
+	for _, v := range obj.Status() {
 		if !v { // everyone must be converged for this to be true
 			return false
 		}
@@ -165,145 +277,40 @@ func (obj *converger) isConverged() bool {
 	return true
 }
 
-// Unregister dissociates the ConvergedUID from the converged checking.
-func (obj *converger) Unregister(uid UID) {
-	if !uid.IsValid() {
-		panic(fmt.Sprintf("the ID of UID(%s) is nil", uid.Name()))
+// test evaluates whether we're converged or not and runs the state change. It
+// is NOT thread-safe.
+func (obj *Coordinator) test() error {
+	// TODO: add these checks elsewhere to prevent anything from running?
+	if obj.timeout < 0 {
+		return nil // nothing to do (only run if timeout is valid)
 	}
-	obj.mutex.Lock()
-	uid.StopTimer() // ignore any errors
-	delete(obj.status, uid.ID())
-	obj.mutex.Unlock()
-	uid.InvalidateID()
-}
 
-// Start causes a Converger object to start or resume running.
-func (obj *converger) Start() {
-	obj.control <- true
-}
+	converged := obj.IsConverged()
+	defer func() {
+		obj.converged = converged // set this only at the end...
+	}()
 
-// Pause causes a Converger object to stop running temporarily.
-func (obj *converger) Pause() { // FIXME: add a sync ACK on pause before return
-	obj.control <- false
-}
-
-// Loop is the main loop for a Converger object. It usually runs in a goroutine.
-// TODO: we could eventually have each resource tell us as soon as it converges,
-// and then keep track of the time delays here, to avoid callers needing select.
-// NOTE: when we have very short timeouts, if we start before all the resources
-// have joined the map, then it might appear as if we converged before we did!
-func (obj *converger) Loop(startPaused bool) {
-	if obj.control == nil {
-		panic("converger not initialized correctly")
-	}
-	if startPaused { // start paused without racing
-		select {
-		case e := <-obj.control:
-			if !e {
-				panic("converger expected true")
-			}
+	if !converged {
+		if !obj.converged { // were we previously also not converged?
+			return nil // nothing to do
 		}
-	}
-	for {
-		select {
-		case e := <-obj.control: // expecting "false" which means pause!
-			if e {
-				panic("converger expected false")
-			}
-			// now i'm paused...
-			select {
-			case e := <-obj.control:
-				if !e {
-					panic("converger expected true")
-				}
-				// restart
-				// kick once to refresh the check...
-				go func() { obj.channel <- struct{}{} }()
-				continue
-			}
 
-		case <-obj.channel:
-			if !obj.isConverged() {
-				if obj.converged { // we're doing a state change
-					// call the arbitrary functions (takes a read lock!)
-					if err := obj.runStateFns(false); err != nil {
-						// FIXME: what to do on error ?
-					}
-				}
-				obj.converged = false
-				continue
-			}
-
-			// we have converged!
-			if obj.timeout >= 0 { // only run if timeout is valid
-				if !obj.converged { // we're doing a state change
-					// call the arbitrary functions (takes a read lock!)
-					if err := obj.runStateFns(true); err != nil {
-						// FIXME: what to do on error ?
-					}
-				}
-			}
-			obj.converged = true
-			// loop and wait again...
-		}
+		// we're doing a state change
+		// call the arbitrary functions (takes a read lock!)
+		return obj.runStateFns(false)
 	}
+
+	// we have converged!
+	if obj.converged { // were we previously also converged?
+		return nil // nothing to do
+	}
+
+	// call the arbitrary functions (takes a read lock!)
+	return obj.runStateFns(true)
 }
 
-// ConvergedTimer adds a timeout to a select call and blocks until then.
-// TODO: this means we could eventually have per resource converged timeouts
-func (obj *converger) ConvergedTimer(uid UID) <-chan time.Time {
-	// be clever: if i'm already converged, this timeout should block which
-	// avoids unnecessary new signals being sent! this avoids fast loops if
-	// we have a low timeout, or in particular a timeout == 0
-	if uid.IsConverged() {
-		// blocks the case statement in select forever!
-		return util.TimeAfterOrBlock(-1)
-	}
-	return util.TimeAfterOrBlock(obj.timeout)
-}
-
-// Status returns a map of the converged status of each UID.
-func (obj *converger) Status() map[uint64]bool {
-	status := make(map[uint64]bool)
-	obj.mutex.RLock() // take a read lock
-	defer obj.mutex.RUnlock()
-	for k, v := range obj.status { // make a copy to avoid the mutex
-		status[k] = v
-	}
-	return status
-}
-
-// Timeout returns the timeout in seconds that converger was created with. This
-// is useful to avoid passing in the timeout value separately when you're
-// already passing in the Converger struct.
-func (obj *converger) Timeout() int {
-	return obj.timeout
-}
-
-// AddStateFn adds a state function to be run on change of converged state.
-func (obj *converger) AddStateFn(name string, stateFn func(bool) error) error {
-	obj.smutex.Lock()
-	defer obj.smutex.Unlock()
-	if _, exists := obj.stateFns[name]; exists {
-		return fmt.Errorf("a stateFn with that name already exists")
-	}
-	obj.stateFns[name] = stateFn
-	return nil
-}
-
-// RemoveStateFn adds a state function to be run on change of converged state.
-func (obj *converger) RemoveStateFn(name string) error {
-	obj.smutex.Lock()
-	defer obj.smutex.Unlock()
-	if _, exists := obj.stateFns[name]; !exists {
-		return fmt.Errorf("a stateFn with that name doesn't exist")
-	}
-	delete(obj.stateFns, name)
-	return nil
-}
-
-// runStateFns runs the listed of stored state functions.
-func (obj *converger) runStateFns(converged bool) error {
+// runStateFns runs the list of stored state functions.
+func (obj *Coordinator) runStateFns(converged bool) error {
 	obj.smutex.RLock()
 	defer obj.smutex.RUnlock()
 	var keys []string
@@ -315,77 +322,125 @@ func (obj *converger) runStateFns(converged bool) error {
 	for _, name := range keys { // run in deterministic order
 		fn := obj.stateFns[name]
 		// call an arbitrary function
-		if e := fn(converged); e != nil {
-			err = multierr.Append(err, e) // list of errors
-		}
+		e := fn(converged)
+		err = errwrap.Append(err, e) // list of errors
 	}
 	return err
 }
 
-// ID returns the unique id of this UID object.
-func (obj *cuid) ID() uint64 {
-	return obj.id
+// AddStateFn adds a state function to be run on change of converged state.
+func (obj *Coordinator) AddStateFn(name string, stateFn func(bool) error) error {
+	obj.smutex.Lock()
+	defer obj.smutex.Unlock()
+	if _, exists := obj.stateFns[name]; exists {
+		return fmt.Errorf("a stateFn with that name already exists")
+	}
+	obj.stateFns[name] = stateFn
+	return nil
 }
 
-// Name returns a user defined name for the specific cuid.
-func (obj *cuid) Name() string {
-	return obj.name
+// RemoveStateFn removes a state function from running on change of converged
+// state.
+func (obj *Coordinator) RemoveStateFn(name string) error {
+	obj.smutex.Lock()
+	defer obj.smutex.Unlock()
+	if _, exists := obj.stateFns[name]; !exists {
+		return fmt.Errorf("a stateFn with that name doesn't exist")
+	}
+	delete(obj.stateFns, name)
+	return nil
 }
 
-// SetName sets a user defined name for the specific cuid.
-func (obj *cuid) SetName(name string) {
-	obj.name = name
+// Status returns a map of the converged status of each UID.
+func (obj *Coordinator) Status() map[*UID]bool {
+	status := make(map[*UID]bool)
+	obj.mutex.RLock() // take a read lock
+	defer obj.mutex.RUnlock()
+	for k := range obj.status {
+		status[k] = k.IsConverged()
+	}
+	return status
 }
 
-// IsValid tells us if the id is valid or has already been destroyed.
-func (obj *cuid) IsValid() bool {
-	return obj.id != 0 // an id of 0 is invalid
+// Timeout returns the timeout in seconds that converger was created with. This
+// is useful to avoid passing in the timeout value separately when you're
+// already passing in the Coordinator struct.
+func (obj *Coordinator) Timeout() int64 {
+	return obj.timeout
 }
 
-// InvalidateID marks the id as no longer valid.
-func (obj *cuid) InvalidateID() {
-	obj.id = 0 // an id of 0 is invalid
+// UID represents one of the probes for the converger coordinator. It is created
+// by calling the Register method of the Coordinator struct. It should be freed
+// after use with Unregister.
+type UID struct {
+	// timeout is a copy of the main timeout. It could eventually be used
+	// for per-UID timeouts too.
+	timeout int64
+	// isConverged stores the convergence state of this particular UID.
+	isConverged bool
+
+	// poke stores a reference to the main poke function.
+	poke func()
+	// unregister stores a reference to the unregister function.
+	unregister func()
+
+	// timer
+	mutex   *sync.Mutex
+	timer   chan struct{}
+	running bool // is the timer running?
+	wg      *sync.WaitGroup
 }
 
-// IsConverged is a helper function to the regular IsConverged method.
-func (obj *cuid) IsConverged() bool {
-	return obj.converger.IsConverged(obj)
+// Unregister removes this UID from the converger coordinator. An unregistered
+// UID is no longer part of the convergence checking.
+func (obj *UID) Unregister() {
+	obj.unregister()
 }
 
-// SetConverged is a helper function to the regular SetConverged notification.
-func (obj *cuid) SetConverged(isConverged bool) error {
-	return obj.converger.SetConverged(obj, isConverged)
+// IsConverged reports whether this UID is converged or not.
+func (obj *UID) IsConverged() bool {
+	return obj.isConverged
 }
 
-// Unregister is a helper function to unregister myself.
-func (obj *cuid) Unregister() {
-	obj.converger.Unregister(obj)
+// SetConverged sets the convergence state of this UID. This is used by the
+// running timer if one is started. The timer will overwrite any value set by
+// this method.
+func (obj *UID) SetConverged(isConverged bool) {
+	obj.isConverged = isConverged
+	obj.poke() // notify of change
 }
 
-// ConvergedTimer is a helper around the regular ConvergedTimer method.
-func (obj *cuid) ConvergedTimer() <-chan time.Time {
-	return obj.converger.ConvergedTimer(obj)
+// ConvergedTimer adds a timeout to a select call and blocks until then.
+// TODO: this means we could eventually have per resource converged timeouts
+func (obj *UID) ConvergedTimer() <-chan time.Time {
+	// be clever: if i'm already converged, this timeout should block which
+	// avoids unnecessary new signals being sent! this avoids fast loops if
+	// we have a low timeout, or in particular a timeout == 0
+	if obj.IsConverged() {
+		// blocks the case statement in select forever!
+		return util.TimeAfterOrBlock(-1)
+	}
+	return util.TimeAfterOrBlock(int(obj.timeout))
 }
 
-// StartTimer runs an invisible timer that automatically converges on timeout.
-func (obj *cuid) StartTimer() (func() error, error) {
+// StartTimer runs a timer that sets us as converged on timeout. It also returns
+// a handle to the StopTimer function which should be run before exit.
+func (obj *UID) StartTimer() (func() error, error) {
 	obj.mutex.Lock()
-	if !obj.running {
-		obj.timer = make(chan struct{})
-		obj.running = true
-	} else {
-		obj.mutex.Unlock()
+	defer obj.mutex.Unlock()
+	if obj.running {
 		return obj.StopTimer, fmt.Errorf("timer already started")
 	}
-	obj.mutex.Unlock()
+	obj.timer = make(chan struct{})
+	obj.running = true
 	obj.wg.Add(1)
 	go func() {
 		defer obj.wg.Done()
 		for {
 			select {
 			case _, ok := <-obj.timer: // reset signal channel
-				if !ok { // channel is closed
-					return // false to exit
+				if !ok {
+					return
 				}
 				obj.SetConverged(false)
 
@@ -393,8 +448,8 @@ func (obj *cuid) StartTimer() (func() error, error) {
 				obj.SetConverged(true) // converged!
 				select {
 				case _, ok := <-obj.timer: // reset signal channel
-					if !ok { // channel is closed
-						return // false to exit
+					if !ok {
+						return
 					}
 				}
 			}
@@ -403,8 +458,8 @@ func (obj *cuid) StartTimer() (func() error, error) {
 	return obj.StopTimer, nil
 }
 
-// ResetTimer resets the counter to zero if using a StartTimer internally.
-func (obj *cuid) ResetTimer() error {
+// ResetTimer resets the timer to zero.
+func (obj *UID) ResetTimer() error {
 	obj.mutex.Lock()
 	defer obj.mutex.Unlock()
 	if obj.running {
@@ -414,8 +469,8 @@ func (obj *cuid) ResetTimer() error {
 	return fmt.Errorf("timer hasn't been started")
 }
 
-// StopTimer stops the running timer permanently until a StartTimer is run.
-func (obj *cuid) StopTimer() error {
+// StopTimer stops the running timer.
+func (obj *UID) StopTimer() error {
 	obj.mutex.Lock()
 	defer obj.mutex.Unlock()
 	if !obj.running {

converger/converger_test.go

@@ -0,0 +1,31 @@
+// Mgmt
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// +build !root
+
+package converger
+
+import (
+	"testing"
+)
+
+func TestBufferedChan1(t *testing.T) {
+	ch := make(chan bool, 1)
+	ch <- true
+	close(ch) // closing a channel that's not empty should not block
+	// must be able to exit without blocking anywhere
+}

debian/copyright

@@ -3,7 +3,7 @@ Upstream-Name: mgmt
 Source: <https://github.com/purpleidea/mgmt>
 
 Files: *
-Copyright: Copyright (C) 2013-2018+ James Shubin and the project contributors
+Copyright: Copyright (C) 2013-2019+ James Shubin and the project contributors
 License: GPL-3.0
 
 License: GPL-3.0

doc.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

docker/Dockerfile.development

@@ -1,10 +1,10 @@
-FROM golang:1.9
+FROM golang:1.11
 
 MAINTAINER Michał Czeraszkiewicz <contact@czerasz.com>
 
 # Set the reset cache variable
 # Read more here: http://czerasz.com/2014/11/13/docker-tip-and-tricks/#use-refreshedat-variable-for-better-cache-control
-ENV REFRESHED_AT 2017-11-16
+ENV REFRESHED_AT 2019-02-06
 
 RUN apt-get update
 

docs/conf.py

@@ -51,7 +51,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'mgmt'
-copyright = u'2013-2018+ James Shubin and the project contributors'
+copyright = u'2013-2019+ James Shubin and the project contributors'
 author = u'James Shubin'
 
 # The version info for the project you're documenting, acts as replacement for

docs/development.md

@@ -3,7 +3,7 @@
 This document contains some additional information and help regarding
 developing `mgmt`. Useful tools, conventions, etc.
 
-Be sure to read [quick start guide](docs/quick-start-guide.md) first.
+Be sure to read [quick start guide](quick-start-guide.md) first.
 
 ## Testing
 

docs/documentation.md

@@ -455,7 +455,7 @@ To report any bugs, please file a ticket at: [https://github.com/purpleidea/mgmt
 
 ## Authors
 
-Copyright (C) 2013-2018+ James Shubin and the project contributors
+Copyright (C) 2013-2019+ James Shubin and the project contributors
 
 Please see the
 [AUTHORS](https://github.com/purpleidea/mgmt/tree/master/AUTHORS) file

docs/faq.md

@@ -9,6 +9,18 @@ I wanted a next generation config management solution that didn't have all of
 the design flaws or limitations that the current generation of tools do, and no
 tool existed!
 
+### Why did you choose `golang` for the project?
+
+When I started working on the project, I needed to choose a language that
+already had an implementation of a distributed consensus algorithm available.
+That meant [Paxos](https://en.wikipedia.org/wiki/Paxos_(computer_science)) or
+[Raft](https://en.wikipedia.org/wiki/Raft_(computer_science)). Golang was one
+language that actually had two different Raft implementations, `etcd`, and
+`consul`. Other design requirements included something that was reasonably fast,
+typed and memory-safe, and suited for systems engineering. After a reasonably
+extensive search, I chose `golang`. I think it was the right decision. There are
+a number of other features of the language which helped influence the decision.
+
 ### How do I contribute to the project if I don't know `golang`?
 
 There are many different ways you can contribute to the project. They can be
@@ -125,6 +137,58 @@ The downside to this approach is that you won't benefit from the automatic
 elastic nature of the embedded etcd servers, and that you're responsible if you
 accidentally break your etcd cluster, or if you use an unsupported version.
 
+### In `mgmt` you talk about events. What is this referring to?
+
+Mgmt has two main concepts that involve "events":
+1. Events in the [resource primitive](resource-guide.md).
+2. Events in the [reactive language](language-guide.md).
+
+Each resource primitive in mgmt can test (check) and set (apply) the desired
+state that was requested of it. This is familiar to what is common with existing
+tools such as `Puppet`, `Ansible`, `Chef`, `Terraform`, etc... In addition,
+`mgmt` can also **watch** the state and detect changes. As a result, it never
+has to waste time and cpu resources by polling to test and set state, leading to
+a design which is algorithmically much faster than the existing generation of
+tools.
+
+To describe the set of resources to apply, mgmt describes this collection with a
+language. In order to model the time component of infrastructure, we use a
+special kind of language called an [FRP](https://en.wikipedia.org/wiki/Functional_reactive_programming).
+This language has a built-in concept that we call "events", and which means that
+we re-evaluate the relevant portions of the code whenever a value or function
+has an event that tells us that it changed. The `R` in `FRP` stands for
+reactive. This is similar to how a spreadsheet updates dependent cells when a
+pre-requisite value is modified. [This article](https://en.wikipedia.org/wiki/Reactive_programming)
+provides a bit more background.
+
+Whenever any of the streams of values in the language change, the program is
+partially re-evaluated. The output of any mgmt program is a [DAG](https://en.wikipedia.org/wiki/Directed_acyclic_graph)
+of resources, or more precisely, a stream of resource graphs. Since we have
+events per-resource, we can efficiently switch from one desired-state resource
+graph to the next without re-checking their individual states, since we've been
+monitoring them all along.
+
+One side-effect of all this, is that if a rogue systems administrator manually
+changes the state of any managed resource, mgmt will detect this and attempt to
+revert the change. This makes for excellent live demos, but is not the primary
+design goal. It is a consequence of tracking state so that graph changes are
+efficient. We implement the event detection via an intentional per-resource
+[main loop](https://en.wikipedia.org/wiki/Event_loop) which can enable other
+interesting functionality too!
+
+Make sure to get rid of your rogue sysadmin! ;)
+
+### Do I need to run `mgmt` as `root`?
+
+No and yes. It depends. Nothing in mgmt explicitly requires root in the design,
+however mgmt will require root only if the changes to your system that you want
+it to make require root.
+
+For example, if you use it to manage files that require root access to modify,
+then you'll need root. If you only use it to manage files and resources
+elsewhere, then it shouldn't need root. Many resources are perfectly usable
+without root, and virtually all of my live demos are done without root.
+
 ### How can I run `mgmt` on-demand, or in `cron`, instead of continuously?
 
 By default, `mgmt` will run continuously in an attempt to keep your machine in a
@@ -151,40 +215,38 @@ requires a number of seconds as an argument.
 ./mgmt run lang --lang examples/lang/hello0.mcl --converged-timeout=5
 ```
 
-### What does the error message about an inconsistent dataDir mean?
+### On startup `mgmt` hangs after: `etcd: server: starting...`.
 
 If you get an error message similar to:
 
 ```
-Etcd: Connect: CtxError...
-Etcd: CtxError: Reason: CtxDelayErr(5s): No endpoints available yet!
-Etcd: Connect: Endpoints: []
-Etcd: The dataDir (/var/lib/mgmt/etcd) might be inconsistent or corrupt.
+etcd: server: starting...
+etcd: server: start timeout of 1m0s reached
+etcd: server: close timeout of 15s reached
 ```
 
-This happens when there are a series of fatal connect errors in a row. This can
-happen when you start `mgmt` using a dataDir that doesn't correspond to the
-current cluster view. As a result, the embedded etcd server never finishes
-starting up, and as a result, a default endpoint never gets added. The solution
-is to either reconcile the mistake, and if there is no important data saved, you
-can remove the etcd dataDir. This is typically `/var/lib/mgmt/etcd/member/`.
+But nothing happens afterwards, this can be due to a corrupt etcd storage
+directory. Each etcd server embedded in mgmt must have a special directory where
+it stores local state. It must not be shared by more than one individual member.
+This dir is typically `/var/lib/mgmt/etcd/member/`. If you accidentally use it
+(for example during testing) with a different cluster view, then you can corrupt
+it. This can happen if you use it with more than one different hostname.
 
-### Why do resources have both a `Cmp` method and an `IFF` (on the UID) method?
+The solution is to avoid making this mistake, and if there is no important data
+saved, you can remove the etcd member dir and start over.
 
-The `Cmp()` methods are for determining if two resources are effectively the
-same, which is used to make graph change delta's efficient. This is when we want
-to change from the current running graph to a new graph, but preserve the common
-vertices. Since we want to make this process efficient, we only update the parts
-that are different, and leave everything else alone. This `Cmp()` method can
-tell us if two resources are the same. In case it is not obvious, `cmp` is an
-abbrev. for compare.
+### On running `make` to build a new version, it errors with: `Text file busy`.
 
-The `IFF()` method is part of the whole UID system, which is for discerning if a
-resource meets the requirements another expects for an automatic edge. This is
-because the automatic edge system assumes a unified UID pattern to test for
-equality. In the future it might be helpful or sane to merge the two similar
-comparison functions although for now they are separate because they are
-actually answer different questions.
+If you get an error like:
+
+```
+cp: cannot create regular file 'mgmt': Text file busy
+```
+
+This can happen if you ran `make build` (or just `make`) when there was already
+an instance of mgmt running, or if a related file locking issue occurred. To
+solve this, shutdown and running mgmt process, run `rm mgmt` to remove the file,
+and then get a new one by running `make` again.
 
 ### Does this support Windows? OSX? GNU Hurd?
 
@@ -262,9 +324,8 @@ which definitely existed before the band did.
 ### You didn't answer my question, or I have a question!
 
 It's best to ask on [IRC](https://webchat.freenode.net/?channels=#mgmtconfig)
-to see if someone can help you. Once we get a big enough community going, we'll
-add a mailing list. If you don't get any response from the above, you can
-contact me through my [technical blog](https://purpleidea.com/contact/)
-and I'll do my best to help. If you have a good question, please add it as a
-patch to this documentation. I'll merge your question, and add a patch with the
-answer!
+to see if someone can help you. If you don't get a response from IRC, you can
+contact me through my [technical blog](https://purpleidea.com/contact/) and I'll
+do my best to help. If you have a good question, please add it as a patch to
+this documentation. I'll merge your question, and add a patch with the answer!
+For news and updates, subscribe to the [mailing list](https://www.redhat.com/mailman/listinfo/mgmtconfig-list).

docs/resource-guide.md

@@ -96,8 +96,10 @@ Default() engine.Res
 ```
 
 This returns a populated resource struct as a `Res`. It shouldn't populate any
-values which already have the correct default as the golang zero value. In
+values which already get a good default as the respective golang zero value. In
 general it is preferable if the zero values make for the correct defaults.
+(This is to say, resources are designed to behave safely and intuitively
+when parameters take a zero value, whenever this is possible.)
 
 #### Example
 
@@ -307,21 +309,18 @@ running.
 The lifetime of most resources `Watch` method should be spent in an infinite
 loop that is bounded by a `select` call. The `select` call is the point where
 our method hands back control to the engine (and the kernel) so that we can
-sleep until something of interest wakes us up. In this loop we must process
-events from the engine via the `<-obj.init.Events` channel, and receive events
-for our resource itself!
+sleep until something of interest wakes us up. In this loop we must wait until
+we get a shutdown event from the engine via the `<-obj.init.Done` channel, which
+closes when we'd like to shut everything down. At this point you should cleanup,
+and let `Watch` close.
 
 #### Events
 
-If we receive an internal event from the `<-obj.init.Events` channel, we should
-read it with the `obj.init.Read` helper function. This function tells us if we
-should shutdown our resource. It also handles pause functionality which blocks
-our resource temporarily in this method. If this channel shuts down, then we
-should treat that as an exit signal.
-
-When we want to send an event, we use the `Event` helper function. It is also
-important to mark the resource state as `dirty` if we believe it might have
-changed. We do this by calling the `obj.init.Dirty` function.
+If the  `<-obj.init.Done` channel closes, we should shutdown our resource. When
+When we want to send an event, we use the `Event` helper function. This
+automatically marks the resource state as `dirty`. If you're unsure, it's not
+harmful to send the event. This will ultimately cause `CheckApply` to run. This
+method can block if the resource is being paused.
 
 #### Startup
 
@@ -330,8 +329,7 @@ to generate one event to notify the `mgmt` engine that we're now listening
 successfully, so that it can run an initial `CheckApply` to ensure we're safely
 tracking a healthy state and that we didn't miss anything when `Watch` was down
 or from before `mgmt` was running. You must do this by calling the
-`obj.init.Running` method. If it returns an error, you must exit and return that
-error.
+`obj.init.Running` method.
 
 #### Converged
 
@@ -358,41 +356,29 @@ func (obj *FooRes) Watch() error {
 	defer obj.whatever.CloseFoo() // shutdown our Foo
 
 	// notify engine that we're running
-	if err := obj.init.Running(); err != nil {
-		return err // exit if requested
-	}
+	obj.init.Running() // when started, notify engine that we're running
 
 	var send = false // send event?
 	for {
 		select {
-		case event, ok := <-obj.init.Events:
-			if !ok {
-				// shutdown engine
-				// (it is okay if some `defer` code runs first)
-				return nil
-			}
-			if err := obj.init.Read(event); err != nil {
-				return err
-			}
-
 		// the actual events!
 		case event := <-obj.foo.Events:
 			if is_an_event {
 				send = true
-				obj.init.Dirty() // dirty
 			}
 
 		// event errors
 		case err := <-obj.foo.Errors:
 			return err // will cause a retry or permanent failure
+
+		case <-obj.init.Done: // signal for shutdown request
+			return nil
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
 			send = false
-			if err := obj.init.Event(); err != nil {
-				return err // exit if requested
-			}
+			obj.init.Event()
 		}
 	}
 }
@@ -567,23 +553,10 @@ ready to detect changes.
 Event sends an event notifying the engine of a possible state change. It is
 only called from within `Watch`.
 
-### Events
+### Done
 
-Events is a channel that we must watch for messages from the engine. When it
-closes, this is a signal to shutdown. It is
-only called from within `Watch`.
-
-### Read
-
-Read processes messages that come in from the `Events` channel. It is a helper
-method that knows how to handle the pause mechanism correctly. It is
-only called from within `Watch`.
-
-### Dirty
-
-Dirty marks the resource state as dirty. This signals to the engine that
-CheckApply will have some work to do in order to converge it. It is
-only called from within `Watch`.
+Done is a channel that closes when the engine wants us to shutdown. It is only
+called from within `Watch`.
 
 ### Refresh
 
@@ -783,6 +756,23 @@ Feel free to use this pattern if you're convinced it's necessary. Alternatively,
 if you think I got the `Res` API wrong and you have an improvement, please let
 us know!
 
+### Why do resources have both a `Cmp` method and an `IFF` (on the UID) method?
+
+The `Cmp()` methods are for determining if two resources are effectively the
+same, which is used to make graph change delta's efficient. This is when we want
+to change from the current running graph to a new graph, but preserve the common
+vertices. Since we want to make this process efficient, we only update the parts
+that are different, and leave everything else alone. This `Cmp()` method can
+tell us if two resources are the same. In case it is not obvious, `cmp` is an
+abbrev. for compare.
+
+The `IFF()` method is part of the whole UID system, which is for discerning if a
+resource meets the requirements another expects for an automatic edge. This is
+because the automatic edge system assumes a unified UID pattern to test for
+equality. In the future it might be helpful or sane to merge the two similar
+comparison functions although for now they are separate because they are
+actually answer different questions.
+
 ### What new resource primitives need writing?
 
 There are still many ideas for new resources that haven't been written yet. If

engine/autoedge.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/autoedge_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/autogroup.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/cmp.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/copy.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -20,7 +20,7 @@ package engine
 import (
 	"fmt"
 
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 // ResCopy copies a resource. This is the main entry point for copying a

engine/edge.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/error.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -24,9 +24,6 @@ type Error string
 func (e Error) Error() string { return string(e) }
 
 const (
-	// ErrWatchExit represents an exit from the Watch loop via chan closure.
-	ErrWatchExit = Error("watch exit")
-
-	// ErrSignalExit represents an exit from the Watch loop via exit signal.
-	ErrSignalExit = Error("signal exit")
+	// ErrClosed means we couldn't complete a task because we had closed.
+	ErrClosed = Error("closed")
 )

engine/event/event.go

@@ -1,83 +0,0 @@
-// Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
-// Written by James Shubin <james@shubin.ca> and the project contributors
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-// Package event provides some primitives that are used for message passing.
-package event
-
-//go:generate stringer -type=Kind -output=kind_stringer.go
-
-// Kind represents the type of event being passed.
-type Kind int
-
-// The different event kinds are used in different contexts.
-const (
-	KindNil Kind = iota
-	KindStart
-	KindPause
-	KindPoke
-	KindExit
-)
-
-// Pre-built messages so they can be used directly without having to use NewMsg.
-// These are useful when we don't want a response via ACK().
-var (
-	Start = &Msg{Kind: KindStart}
-	Pause = &Msg{Kind: KindPause} // probably unused b/c we want a resp
-	Poke  = &Msg{Kind: KindPoke}
-	Exit  = &Msg{Kind: KindExit}
-)
-
-// Msg is an event primitive that represents a kind of event, and optionally a
-// request for an ACK.
-type Msg struct {
-	Kind Kind
-
-	resp chan struct{}
-}
-
-// NewMsg builds a new message struct. It will want an ACK. If you don't want an
-// ACK then use the pre-built messages in the package variable globals.
-func NewMsg(kind Kind) *Msg {
-	return &Msg{
-		Kind: kind,
-		resp: make(chan struct{}),
-	}
-}
-
-// CanACK determines if an ACK is possible for this message. It does not say
-// whether one has already been sent or not.
-func (obj *Msg) CanACK() bool {
-	return obj.resp != nil
-}
-
-// ACK acknowledges the event. It must not be called more than once for the same
-// event. It unblocks the past and future calls of Wait for this event.
-func (obj *Msg) ACK() {
-	close(obj.resp)
-}
-
-// Wait on ACK for this event. It doesn't matter if this runs before or after
-// the ACK. It will unblock either way.
-// TODO: consider adding a context if it's ever useful.
-func (obj *Msg) Wait() error {
-	select {
-	//case <-ctx.Done():
-	//	return ctx.Err()
-	case <-obj.resp:
-		return nil
-	}
-}

engine/fs.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/actions.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -24,11 +24,9 @@ import (
 	"time"
 
 	"github.com/purpleidea/mgmt/engine"
-	"github.com/purpleidea/mgmt/engine/event"
 	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/util/errwrap"
 
-	//multierr "github.com/hashicorp/go-multierror"
-	errwrap "github.com/pkg/errors"
 	"golang.org/x/time/rate"
 )
 
@@ -67,26 +65,24 @@ func (obj *Engine) Process(vertex pgraph.Vertex) error {
 		return fmt.Errorf("vertex is not a Res")
 	}
 
-	// Engine Guarantee: Do not allow CheckApply to run while we are paused.
-	// This makes the resource able to know that synchronous channel sending
-	// to the main loop select in Watch from within CheckApply, will succeed
-	// without blocking because the resource went into a paused state. If we
-	// are using the Poll metaparam, then Watch will (of course) not be run.
-	// FIXME: should this lock be here, or wrapped right around CheckApply ?
-	obj.state[vertex].eventsLock.Lock() // this lock is taken within Event()
-	defer obj.state[vertex].eventsLock.Unlock()
-
 	// backpoke! (can be async)
 	if vs := obj.BadTimestamps(vertex); len(vs) > 0 {
 		// back poke in parallel (sync b/c of waitgroup)
+		wg := &sync.WaitGroup{}
 		for _, v := range obj.graph.IncomingGraphVertices(vertex) {
 			if !pgraph.VertexContains(v, vs) { // only poke what's needed
 				continue
 			}
 
-			go obj.state[v].Poke() // async
+			// doesn't really need to be in parallel, but we can...
+			wg.Add(1)
+			go func(vv pgraph.Vertex) {
+				defer wg.Done()
+				obj.state[vv].Poke() // async
+			}(v)
 
 		}
+		wg.Wait()
 		return nil // can't continue until timestamp is in sequence
 	}
 
@@ -244,14 +240,22 @@ func (obj *Engine) Process(vertex pgraph.Vertex) error {
 
 // Worker is the common run frontend of the vertex. It handles all of the retry
 // and retry delay common code, and ultimately returns the final status of this
-// vertex execution.
+// vertex execution. This function cannot be "re-run" for the same vertex. The
+// retry mechanism stuff happens inside of this. To actually "re-run" you need
+// to remove the vertex and build a new one. The engine guarantees that we do
+// not allow CheckApply to run while we are paused. That is enforced here.
 func (obj *Engine) Worker(vertex pgraph.Vertex) error {
 	res, isRes := vertex.(engine.Res)
 	if !isRes {
 		return fmt.Errorf("vertex is not a resource")
 	}
 
-	defer close(obj.state[vertex].stopped) // done signal
+	// bonus safety check
+	if res.MetaParams().Burst == 0 && !(res.MetaParams().Limit == rate.Inf) { // blocked
+		return fmt.Errorf("permanently limited (rate != Inf, burst = 0)")
+	}
+
+	//defer close(obj.state[vertex].stopped) // done signal
 
 	obj.state[vertex].cuid = obj.Converger.Register()
 	obj.state[vertex].tuid = obj.Converger.Register()
@@ -265,7 +269,28 @@ func (obj *Engine) Worker(vertex pgraph.Vertex) error {
 	obj.state[vertex].wg.Add(1)
 	go func() {
 		defer obj.state[vertex].wg.Done()
-		defer close(obj.state[vertex].outputChan) // we close this on behalf of res
+		defer close(obj.state[vertex].eventsChan) // we close this on behalf of res
+
+		// This is a close reverse-multiplexer. If any of the channels
+		// close, then it will cause the doneChan to close. That way,
+		// multiple different folks can send a close signal, without
+		// every worrying about duplicate channel close panics.
+		obj.state[vertex].wg.Add(1)
+		go func() {
+			defer obj.state[vertex].wg.Done()
+
+			// reverse-multiplexer: any close, causes *the* close!
+			select {
+			case <-obj.state[vertex].processDone:
+			case <-obj.state[vertex].watchDone:
+			case <-obj.state[vertex].limitDone:
+			case <-obj.state[vertex].removeDone:
+			case <-obj.state[vertex].eventsDone:
+			}
+
+			// the main "done" signal gets activated here!
+			close(obj.state[vertex].doneChan)
+		}()
 
 		var err error
 		var retry = res.MetaParams().Retry // lookup the retry value
@@ -283,13 +308,8 @@ func (obj *Engine) Worker(vertex pgraph.Vertex) error {
 						case <-timer.C: // the wait is over
 							return errDelayExpired // special
 
-						case event, ok := <-obj.state[vertex].init.Events:
-							if !ok {
-								return nil
-							}
-							if err := obj.state[vertex].init.Read(event); err != nil {
-								return err
-							}
+						case <-obj.state[vertex].init.Done:
+							return nil
 						}
 					}
 				}()
@@ -308,68 +328,121 @@ func (obj *Engine) Worker(vertex pgraph.Vertex) error {
 				obj.Logf("Watch(%s): Exited(%+v)", vertex, err)
 				obj.state[vertex].cuid.StopTimer() // clean up nicely
 			}
-			if err == nil || err == engine.ErrWatchExit || err == engine.ErrSignalExit {
+			if err == nil { // || err == engine.ErrClosed
 				return // exited cleanly, we're done
 			}
 			// we've got an error...
 			delay = res.MetaParams().Delay
 
 			if retry < 0 { // infinite retries
-				obj.state[vertex].reset()
 				continue
 			}
 			if retry > 0 { // don't decrement past 0
 				retry--
 				obj.state[vertex].init.Logf("retrying Watch after %.4f seconds (%d left)", float64(delay)/1000, retry)
-				obj.state[vertex].reset()
 				continue
 			}
 			//if retry == 0 { // optional
 			//	err = errwrap.Wrapf(err, "permanent watch error")
 			//}
 			break // break out of this and send the error
-		}
+		} // for retry loop
+
 		// this section sends an error...
 		// If the CheckApply loop exits and THEN the Watch fails with an
 		// error, then we'd be stuck here if exit signal didn't unblock!
 		select {
-		case obj.state[vertex].outputChan <- errwrap.Wrapf(err, "watch failed"):
+		case obj.state[vertex].eventsChan <- errwrap.Wrapf(err, "watch failed"):
 			// send
-		case <-obj.state[vertex].exit.Signal():
-			// pass
 		}
 	}()
 
-	// bonus safety check
-	if res.MetaParams().Burst == 0 && !(res.MetaParams().Limit == rate.Inf) { // blocked
-		return fmt.Errorf("permanently limited (rate != Inf, burst = 0)")
-	}
-	var limiter = rate.NewLimiter(res.MetaParams().Limit, res.MetaParams().Burst)
-	// It is important that we shutdown the Watch loop if this exits.
-	// Example, if Process errors permanently, we should ask Watch to exit.
-	defer obj.state[vertex].Event(event.Exit) // signal an exit
-	for {
+	// If this exits cleanly, we must unblock the reverse-multiplexer.
+	// I think this additional close is unnecessary, but it's not harmful.
+	defer close(obj.state[vertex].eventsDone) // causes doneChan to close
+	limiter := rate.NewLimiter(res.MetaParams().Limit, res.MetaParams().Burst)
+	var reserv *rate.Reservation
+	var reterr error
+	var failed bool // has Process permanently failed?
+Loop:
+	for { // process loop
 		select {
-		case err, ok := <-obj.state[vertex].outputChan: // read from watch channel
+		case err, ok := <-obj.state[vertex].eventsChan: // read from watch channel
 			if !ok {
-				return nil
+				return reterr // we only return when chan closes
 			}
+			// If the Watch method exits with an error, then this
+			// channel will get that error propagated to it, which
+			// we then save so we can return it to the caller of us.
 			if err != nil {
-				return err // permanent failure
+				failed = true
+				close(obj.state[vertex].watchDone)   // causes doneChan to close
+				reterr = errwrap.Append(reterr, err) // permanent failure
+				continue
 			}
+			if obj.Debug {
+				obj.Logf("event received")
+			}
+			reserv = limiter.ReserveN(time.Now(), 1) // one event
+			// reserv.OK() seems to always be true here!
 
-			// safe to go run the process...
-		case <-obj.state[vertex].exit.Signal(): // TODO: is this needed?
-			return nil
+		case _, ok := <-obj.state[vertex].pokeChan: // read from buffered poke channel
+			if !ok { // we never close it
+				panic("unexpected close of poke channel")
+			}
+			if obj.Debug {
+				obj.Logf("poke received")
+			}
+			reserv = nil // we didn't receive a real event here...
+		}
+		if failed { // don't Process anymore if we've already failed...
+			continue Loop
 		}
 
-		now := time.Now()
-		r := limiter.ReserveN(now, 1) // one event
-		// r.OK() seems to always be true here!
-		d := r.DelayFrom(now)
-		if d > 0 { // delay
+		// drop redundant pokes
+		for len(obj.state[vertex].pokeChan) > 0 {
+			select {
+			case <-obj.state[vertex].pokeChan:
+			default:
+				// race, someone else read one!
+			}
+		}
+
+		// pause if one was requested...
+		select {
+		case <-obj.state[vertex].pauseSignal: // channel closes
+			// NOTE: If we allowed a doneChan below to let us out
+			// of the resumeSignal wait, then we could loop around
+			// and run this again, causing a panic. Instead of this
+			// being made safe with a sync.Once, we instead run a
+			// Resume() call inside of the vertexRemoveFn function,
+			// which should unblock it when we're going to need to.
+			obj.state[vertex].pausedAck.Ack() // send ack
+			// we are paused now, and waiting for resume or exit...
+			select {
+			case <-obj.state[vertex].resumeSignal: // channel closes
+				// resumed!
+				// pass through to allow a Process to try to run
+				// TODO: consider adding this fast pause here...
+				//if obj.fastPause {
+				//	obj.Logf("fast pausing on resume")
+				//	continue
+				//}
+			}
+		default:
+			// no pause requested, keep going...
+		}
+		if failed { // don't Process anymore if we've already failed...
+			continue Loop
+		}
+
+		// limit delay
+		d := time.Duration(0)
+		if reserv != nil {
+			d = reserv.DelayFrom(time.Now())
+		}
+		if reserv != nil && d > 0 { // delay
 			obj.state[vertex].init.Logf("limited (rate: %v/sec, burst: %d, next: %v)", res.MetaParams().Limit, res.MetaParams().Burst, d)
-			var count int
 			timer := time.NewTimer(time.Duration(d) * time.Millisecond)
 		LimitWait:
 			for {
@@ -378,35 +451,38 @@ func (obj *Engine) Worker(vertex pgraph.Vertex) error {
 					break LimitWait
 
 				// consume other events while we're waiting...
-				case e, ok := <-obj.state[vertex].outputChan: // read from watch channel
+				case e, ok := <-obj.state[vertex].eventsChan: // read from watch channel
 					if !ok {
-						// FIXME: is this logic correct?
-						if count == 0 {
-							return nil
-						}
-						// loop, because we have
-						// the previous event to
-						// run process on first!
-						continue
+						return reterr // we only return when chan closes
 					}
 					if e != nil {
-						return e // permanent failure
+						failed = true
+						close(obj.state[vertex].limitDone) // causes doneChan to close
+						reterr = errwrap.Append(reterr, e) // permanent failure
+						break LimitWait
 					}
-					count++                         // count the events...
+					if obj.Debug {
+						obj.Logf("event received in limit")
+					}
+					// TODO: does this get added in properly?
 					limiter.ReserveN(time.Now(), 1) // one event
 				}
 			}
 			timer.Stop() // it's nice to cleanup
 			obj.state[vertex].init.Logf("rate limiting expired!")
 		}
+		if failed { // don't Process anymore if we've already failed...
+			continue Loop
+		}
+		// end of limit delay
 
+		// retry...
 		var err error
 		var retry = res.MetaParams().Retry // lookup the retry value
 		var delay uint64
-	Loop:
+	RetryLoop:
 		for { // retry loop
 			if delay > 0 {
-				var count int
 				timer := time.NewTimer(time.Duration(delay) * time.Millisecond)
 			RetryWait:
 				for {
@@ -415,22 +491,20 @@ func (obj *Engine) Worker(vertex pgraph.Vertex) error {
 						break RetryWait
 
 					// consume other events while we're waiting...
-					case e, ok := <-obj.state[vertex].outputChan: // read from watch channel
+					case e, ok := <-obj.state[vertex].eventsChan: // read from watch channel
 						if !ok {
-							// FIXME: is this logic correct?
-							if count == 0 {
-								// last process error
-								return err
-							}
-							// loop, because we have
-							// the previous event to
-							// run process on first!
-							continue
+							return reterr // we only return when chan closes
 						}
 						if e != nil {
-							return e // permanent failure
+							failed = true
+							close(obj.state[vertex].limitDone) // causes doneChan to close
+							reterr = errwrap.Append(reterr, e) // permanent failure
+							break RetryWait
 						}
-						count++                         // count the events...
+						if obj.Debug {
+							obj.Logf("event received in retry")
+						}
+						// TODO: does this get added in properly?
 						limiter.ReserveN(time.Now(), 1) // one event
 					}
 				}
@@ -438,6 +512,9 @@ func (obj *Engine) Worker(vertex pgraph.Vertex) error {
 				delay = 0    // reset
 				obj.state[vertex].init.Logf("the CheckApply delay expired!")
 			}
+			if failed { // don't Process anymore if we've already failed...
+				continue Loop
+			}
 
 			if obj.Debug {
 				obj.Logf("Process(%s)", vertex)
@@ -447,7 +524,7 @@ func (obj *Engine) Worker(vertex pgraph.Vertex) error {
 				obj.Logf("Process(%s): Return(%+v)", vertex, err)
 			}
 			if err == nil {
-				break Loop
+				break RetryLoop
 			}
 			// we've got an error...
 			delay = res.MetaParams().Delay
@@ -464,15 +541,23 @@ func (obj *Engine) Worker(vertex pgraph.Vertex) error {
 			//	err = errwrap.Wrapf(err, "permanent process error")
 			//}
 
-			// If this exits, defer calls: obj.Event(event.Exit),
-			// which will cause the Watch loop to shutdown. Also,
-			// if the Watch loop shuts down, that will cause this
-			// Process loop to shut down. Also the graph sync can
-			// run an: obj.Event(event.Exit) which causes this to
-			// shutdown as well. Lastly, it is possible that more
-			// that one of these scenarios happens simultaneously.
-			return err
-		}
-	}
+			// It is important that we shutdown the Watch loop if
+			// this dies. If Process fails permanently, we ask it
+			// to exit right here... (It happens when we loop...)
+			failed = true
+			close(obj.state[vertex].processDone) // causes doneChan to close
+			reterr = errwrap.Append(reterr, err) // permanent failure
+			continue
+
+		} // retry loop
+
+		// When this Process loop exits, it's because something has
+		// caused Watch() to shutdown (even if it's our permanent
+		// failure from Process), which caused this channel to close.
+		// On or more exit signals are possible, and more than one can
+		// happen simultaneously.
+
+	} // process loop
+
 	//return nil // unreachable
 }

engine/graph/autoedge.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/autoedge/autoedge.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -22,9 +22,7 @@ import (
 
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/pgraph"
-
-	multierr "github.com/hashicorp/go-multierror"
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 // AutoEdge adds the automatic edges to the graph.
@@ -49,7 +47,7 @@ func AutoEdge(graph *pgraph.Graph, debug bool, logf func(format string, v ...int
 	for _, res := range sorted { // for each vertexes autoedges
 		autoEdgeObj, e := res.AutoEdges()
 		if e != nil {
-			err = multierr.Append(err, e) // collect all errors
+			err = errwrap.Append(err, e) // collect all errors
 			continue
 		}
 		if autoEdgeObj == nil {

engine/graph/autogroup.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -24,8 +24,7 @@ import (
 	"github.com/purpleidea/mgmt/engine/graph/autogroup"
 	"github.com/purpleidea/mgmt/pgraph"
 	"github.com/purpleidea/mgmt/util"
-
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 // AutoGroup runs the auto grouping on the loaded graph.

engine/graph/autogroup/autogroup.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -22,8 +22,7 @@ import (
 
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/pgraph"
-
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 // AutoGroup is the mechanical auto group "runner" that runs the interface spec.

engine/graph/autogroup/autogroup_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -31,8 +31,7 @@ import (
 	"github.com/purpleidea/mgmt/engine/traits"
 	"github.com/purpleidea/mgmt/pgraph"
 	"github.com/purpleidea/mgmt/util"
-
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 func init() {

engine/graph/autogroup/base.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/autogroup/nonreachability.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -19,8 +19,7 @@ package autogroup
 
 import (
 	"github.com/purpleidea/mgmt/pgraph"
-
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 // NonReachabilityGrouper is the most straight-forward algorithm for grouping.

engine/graph/autogroup/util.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -19,8 +19,7 @@ package autogroup
 
 import (
 	"github.com/purpleidea/mgmt/pgraph"
-
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 // VertexMerge merges v2 into v1 by reattaching the edges where appropriate,

engine/graph/engine.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -25,12 +25,9 @@ import (
 
 	"github.com/purpleidea/mgmt/converger"
 	"github.com/purpleidea/mgmt/engine"
-	"github.com/purpleidea/mgmt/engine/event"
 	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/util/errwrap"
 	"github.com/purpleidea/mgmt/util/semaphore"
-
-	multierr "github.com/hashicorp/go-multierror"
-	errwrap "github.com/pkg/errors"
 )
 
 // Engine encapsulates a generic graph and manages its operations.
@@ -42,7 +39,7 @@ type Engine struct {
 	// Prefix is a unique directory prefix which can be used. It should be
 	// created if needed.
 	Prefix    string
-	Converger converger.Converger
+	Converger *converger.Coordinator
 
 	Debug bool
 	Logf  func(format string, v ...interface{})
@@ -50,13 +47,14 @@ type Engine struct {
 	graph     *pgraph.Graph
 	nextGraph *pgraph.Graph
 	state     map[pgraph.Vertex]*State
-	waits     map[pgraph.Vertex]*sync.WaitGroup
+	waits     map[pgraph.Vertex]*sync.WaitGroup // wg for the Worker func
 
 	slock *sync.Mutex // semaphore lock
 	semas map[string]*semaphore.Semaphore
 
-	wg *sync.WaitGroup
+	wg *sync.WaitGroup // wg for the whole engine (only used for close)
 
+	paused    bool // are we paused?
 	fastPause bool
 }
 
@@ -64,6 +62,13 @@ type Engine struct {
 // If the struct does not validate, or it cannot initialize, then this errors.
 // Initially it will contain an empty graph.
 func (obj *Engine) Init() error {
+	if obj.Program == "" {
+		return fmt.Errorf("the Program is empty")
+	}
+	if obj.Hostname == "" {
+		return fmt.Errorf("the Hostname is empty")
+	}
+
 	var err error
 	if obj.graph, err = pgraph.NewGraph("graph"); err != nil {
 		return err
@@ -84,6 +89,8 @@ func (obj *Engine) Init() error {
 
 	obj.wg = &sync.WaitGroup{}
 
+	obj.paused = true // start off true, so we can Resume after first Commit
+
 	return nil
 }
 
@@ -137,6 +144,7 @@ func (obj *Engine) Apply(fn func(*pgraph.Graph) error) error {
 func (obj *Engine) Commit() error {
 	// TODO: Does this hurt performance or graph changes ?
 
+	start := []func() error{} // functions to run after graphsync to start...
 	vertexAddFn := func(vertex pgraph.Vertex) error {
 		// some of these validation steps happen before this Commit step
 		// in Validate() to avoid erroring here. These are redundant.
@@ -192,12 +200,36 @@ func (obj *Engine) Commit() error {
 		if err := obj.state[vertex].Init(); err != nil {
 			return errwrap.Wrapf(err, "the Res did not Init")
 		}
+
+		fn := func() error {
+			// start the Worker
+			obj.wg.Add(1)
+			obj.waits[vertex].Add(1)
+			go func(v pgraph.Vertex) {
+				defer obj.wg.Done()
+				defer obj.waits[v].Done()
+
+				obj.Logf("Worker(%s)", v)
+				// contains the Watch and CheckApply loops
+				err := obj.Worker(v)
+				obj.Logf("Worker(%s): Exited(%+v)", v, err)
+				obj.state[v].workerErr = err // store the error
+				// If the Rewatch metaparam is true, then this will get
+				// restarted if we do a graph cmp swap. This is why the
+				// graph cmp function runs the removes before the adds.
+				// XXX: This should feed into an $error var in the lang.
+			}(vertex)
+			return nil
+		}
+		start = append(start, fn) // do this at the end, if it's needed
 		return nil
 	}
+
 	free := []func() error{} // functions to run after graphsync to reset...
 	vertexRemoveFn := func(vertex pgraph.Vertex) error {
 		// wait for exit before starting new graph!
-		obj.state[vertex].Event(event.Exit) // signal an exit
+		close(obj.state[vertex].removeDone) // causes doneChan to close
+		obj.state[vertex].Resume()          // unblock from resume
 		obj.waits[vertex].Wait()            // sync
 
 		// close the state and resource
@@ -216,15 +248,58 @@ func (obj *Engine) Commit() error {
 		return nil
 	}
 
+	// add the Worker swap (reload) on error decision into this vertexCmpFn
+	vertexCmpFn := func(v1, v2 pgraph.Vertex) (bool, error) {
+		r1, ok1 := v1.(engine.Res)
+		r2, ok2 := v2.(engine.Res)
+		if !ok1 || !ok2 { // should not happen, previously validated
+			return false, fmt.Errorf("not a Res")
+		}
+		m1 := r1.MetaParams()
+		m2 := r2.MetaParams()
+		swap1, swap2 := true, true // assume default of true
+		if m1 != nil {
+			swap1 = m1.Rewatch
+		}
+		if m2 != nil {
+			swap2 = m2.Rewatch
+		}
+
+		s1, ok1 := obj.state[v1]
+		s2, ok2 := obj.state[v2]
+		x1, x2 := false, false
+		if ok1 {
+			x1 = s1.workerErr != nil && swap1
+		}
+		if ok2 {
+			x2 = s2.workerErr != nil && swap2
+		}
+
+		if x1 || x2 {
+			// We swap, even if they're the same, so that we reload!
+			// This causes an add and remove of the "same" vertex...
+			return false, nil
+		}
+
+		return engine.VertexCmpFn(v1, v2) // do the normal cmp otherwise
+	}
+
 	// If GraphSync succeeds, it updates the receiver graph accordingly...
 	// Running the shutdown in vertexRemoveFn does not need to happen in a
 	// topologically sorted order because it already paused in that order.
 	obj.Logf("graph sync...")
-	if err := obj.graph.GraphSync(obj.nextGraph, engine.VertexCmpFn, vertexAddFn, vertexRemoveFn, engine.EdgeCmpFn); err != nil {
+	if err := obj.graph.GraphSync(obj.nextGraph, vertexCmpFn, vertexAddFn, vertexRemoveFn, engine.EdgeCmpFn); err != nil {
 		return errwrap.Wrapf(err, "error running graph sync")
 	}
-	// we run these afterwards, so that the state structs (that might get
-	// referenced) aren't destroyed while someone might poke or use one.
+	// We run these afterwards, so that we don't unnecessarily start anyone
+	// if GraphSync failed in some way. Otherwise we'd have to do clean up!
+	for _, fn := range start {
+		if err := fn(); err != nil {
+			return errwrap.Wrapf(err, "error running start fn")
+		}
+	}
+	// We run these afterwards, so that the state structs (that might get
+	// referenced) are not destroyed while someone might poke or use one.
 	for _, fn := range free {
 		if err := fn(); err != nil {
 			return errwrap.Wrapf(err, "error running free fn")
@@ -248,50 +323,28 @@ func (obj *Engine) Commit() error {
 	return nil
 }
 
-// Start runs the currently active graph. It also un-pauses the graph if it was
-// paused.
-func (obj *Engine) Start() error {
+// Resume runs the currently active graph. It also un-pauses the graph if it was
+// paused. Very little that is interesting should happen here. It all happens in
+// the Commit method. After Commit, new things are already started, but we still
+// need to Resume any pre-existing resources.
+func (obj *Engine) Resume() error {
+	if !obj.paused {
+		return fmt.Errorf("already resumed")
+	}
+
 	topoSort, err := obj.graph.TopologicalSort()
 	if err != nil {
 		return err
 	}
-	indegree := obj.graph.InDegree() // compute all of the indegree's
+	//indegree := obj.graph.InDegree() // compute all of the indegree's
 	reversed := pgraph.Reverse(topoSort)
 
 	for _, vertex := range reversed {
-		state := obj.state[vertex]
-		state.starter = (indegree[vertex] == 0)
-		var unpause = true // assume true
-
-		if !state.working { // if not running...
-			state.working = true
-			unpause = false // doesn't need unpausing if starting
-			obj.wg.Add(1)
-			obj.waits[vertex].Add(1)
-			go func(v pgraph.Vertex) {
-				defer obj.wg.Done()
-				defer obj.waits[vertex].Done()
-				defer func() {
-					obj.state[v].working = false
-				}()
-
-				obj.Logf("Worker(%s)", v)
-				// contains the Watch and CheckApply loops
-				err := obj.Worker(v)
-				obj.Logf("Worker(%s): Exited(%+v)", v, err)
-			}(vertex)
-		}
-
-		select {
-		case <-state.started:
-		case <-state.stopped: // we failed on Watch start
-		}
-
-		if unpause { // unpause (if needed)
-			obj.state[vertex].Event(event.Start)
-		}
+		//obj.state[vertex].starter = (indegree[vertex] == 0)
+		obj.state[vertex].Resume() // doesn't error
 	}
 	// we wait for everyone to start before exiting!
+	obj.paused = false
 	return nil
 }
 
@@ -302,40 +355,46 @@ func (obj *Engine) Start() error {
 // This is because once you've started a fast pause, some dependencies might
 // have been skipped when fast pausing, and future resources might have missed a
 // poke. In general this is only called when you're trying to hurry up the exit.
+// XXX: Not implemented
 func (obj *Engine) SetFastPause() {
 	obj.fastPause = true
 }
 
-// Pause the active, running graph. At the moment this cannot error.
-func (obj *Engine) Pause(fastPause bool) {
+// Pause the active, running graph.
+func (obj *Engine) Pause(fastPause bool) error {
+	if obj.paused {
+		return fmt.Errorf("already paused")
+	}
+
 	obj.fastPause = fastPause
 	topoSort, _ := obj.graph.TopologicalSort()
 	for _, vertex := range topoSort { // squeeze out the events...
 		// The Event is sent to an unbuffered channel, so this event is
 		// synchronous, and as a result it blocks until it is received.
-		obj.state[vertex].Event(event.Pause)
+		if err := obj.state[vertex].Pause(); err != nil && err != engine.ErrClosed {
+			return err
+		}
 	}
 
+	obj.paused = true
+
 	// we are now completely paused...
 	obj.fastPause = false // reset
+	return nil
 }
 
 // Close triggers a shutdown. Engine must be already paused before this is run.
 func (obj *Engine) Close() error {
-	var reterr error
-
-	emptyGraph, err := pgraph.NewGraph("empty")
-	if err != nil {
-		reterr = multierr.Append(reterr, err) // list of errors
-	}
+	emptyGraph, reterr := pgraph.NewGraph("empty")
 
 	// this is a graph switch (graph sync) that switches to an empty graph!
 	if err := obj.Load(emptyGraph); err != nil { // copy in empty graph
-		reterr = multierr.Append(reterr, err)
+		reterr = errwrap.Append(reterr, err)
 	}
+	// FIXME: Do we want to run commit if Load failed? Does this even work?
 	// the commit will cause the graph sync to shut things down cleverly...
 	if err := obj.Commit(); err != nil {
-		reterr = multierr.Append(reterr, err)
+		reterr = errwrap.Append(reterr, err)
 	}
 
 	obj.wg.Wait() // for now, this doesn't need to be a separate Wait() method

engine/graph/graph_test.go

@@ -0,0 +1,37 @@
+// Mgmt
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// +build !root
+
+package graph
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/purpleidea/mgmt/util/errwrap"
+)
+
+func TestMultiErr(t *testing.T) {
+	var err error
+	e := fmt.Errorf("some error")
+	err = errwrap.Append(err, e) // build an error from a nil base
+	// ensure that this lib allows us to append to a nil
+	if err == nil {
+		t.Errorf("missing error")
+	}
+}

engine/graph/refresh.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/semaphore.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -23,9 +23,8 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/purpleidea/mgmt/util/errwrap"
 	"github.com/purpleidea/mgmt/util/semaphore"
-
-	multierr "github.com/hashicorp/go-multierror"
 )
 
 // SemaSep is the trailing separator to split the semaphore id from the size.
@@ -46,9 +45,8 @@ func (obj *Engine) semaLock(semas []string) error {
 		}
 		obj.slock.Unlock()
 
-		if err := sema.P(1); err != nil { // lock!
-			reterr = multierr.Append(reterr, err) // list of errors
-		}
+		err := sema.P(1)                     // lock!
+		reterr = errwrap.Append(reterr, err) // list of errors
 	}
 	return reterr
 }
@@ -65,9 +63,8 @@ func (obj *Engine) semaUnlock(semas []string) error {
 			panic(fmt.Sprintf("graph: sema: %s does not exist", id))
 		}
 
-		if err := sema.V(1); err != nil { // unlock!
-			reterr = multierr.Append(reterr, err) // list of errors
-		}
+		err := sema.V(1)                     // unlock!
+		reterr = errwrap.Append(reterr, err) // list of errors
 	}
 	return reterr
 }

engine/graph/semaphore_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/sendrecv.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -22,9 +22,8 @@ import (
 	"reflect"
 
 	"github.com/purpleidea/mgmt/engine"
-
-	multierr "github.com/hashicorp/go-multierror"
-	errwrap "github.com/pkg/errors"
+	engineUtil "github.com/purpleidea/mgmt/engine/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 // SendRecv pulls in the sent values into the receive slots. It is called by the
@@ -47,16 +46,51 @@ func (obj *Engine) SendRecv(res engine.RecvableRes) (map[string]bool, error) {
 			st = v.Res.Sent()
 		}
 
+		if st == nil {
+			e := fmt.Errorf("received nil value from: %s", v.Res)
+			err = errwrap.Append(err, e) // list of errors
+			continue
+		}
+
+		if e := engineUtil.StructFieldCompat(st, v.Key, res, k); e != nil {
+			err = errwrap.Append(err, e) // list of errors
+			continue
+		}
+
 		// send
+		m1, e := engineUtil.StructTagToFieldName(st)
+		if e != nil {
+			err = errwrap.Append(err, e) // list of errors
+			continue
+		}
+		key1, exists := m1[v.Key]
+		if !exists {
+			e := fmt.Errorf("requested key of `%s` not found in send struct", v.Key)
+			err = errwrap.Append(err, e) // list of errors
+			continue
+		}
+
 		obj1 := reflect.Indirect(reflect.ValueOf(st))
 		type1 := obj1.Type()
-		value1 := obj1.FieldByName(v.Key)
+		value1 := obj1.FieldByName(key1)
 		kind1 := value1.Kind()
 
 		// recv
+		m2, e := engineUtil.StructTagToFieldName(res)
+		if e != nil {
+			err = errwrap.Append(err, e) // list of errors
+			continue
+		}
+		key2, exists := m2[k]
+		if !exists {
+			e := fmt.Errorf("requested key of `%s` not found in recv struct", k)
+			err = errwrap.Append(err, e) // list of errors
+			continue
+		}
+
 		obj2 := reflect.Indirect(reflect.ValueOf(res)) // pass in full struct
 		type2 := obj2.Type()
-		value2 := obj2.FieldByName(k)
+		value2 := obj2.FieldByName(key2)
 		kind2 := value2.Kind()
 
 		if obj.Debug {
@@ -67,7 +101,7 @@ func (obj *Engine) SendRecv(res engine.RecvableRes) (map[string]bool, error) {
 		// i think we probably want the same kind, at least for now...
 		if kind1 != kind2 {
 			e := fmt.Errorf("kind mismatch between %s: %s and %s: %s", v.Res, kind1, res, kind2)
-			err = multierr.Append(err, e) // list of errors
+			err = errwrap.Append(err, e) // list of errors
 			continue
 		}
 
@@ -75,21 +109,21 @@ func (obj *Engine) SendRecv(res engine.RecvableRes) (map[string]bool, error) {
 		// FIXME: do we want to relax this for string -> *string ?
 		if e := TypeCmp(value1, value2); e != nil {
 			e := errwrap.Wrapf(e, "type mismatch between %s and %s", v.Res, res)
-			err = multierr.Append(err, e) // list of errors
+			err = errwrap.Append(err, e) // list of errors
 			continue
 		}
 
 		// if we can't set, then well this is pointless!
 		if !value2.CanSet() {
 			e := fmt.Errorf("can't set %s.%s", res, k)
-			err = multierr.Append(err, e) // list of errors
+			err = errwrap.Append(err, e) // list of errors
 			continue
 		}
 
 		// if we can't interface, we can't compare...
 		if !value1.CanInterface() || !value2.CanInterface() {
 			e := fmt.Errorf("can't interface %s.%s", res, k)
-			err = multierr.Append(err, e) // list of errors
+			err = errwrap.Append(err, e) // list of errors
 			continue
 		}
 

engine/graph/state.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -19,18 +19,14 @@ package graph
 
 import (
 	"fmt"
-	"os"
-	"path"
 	"sync"
 	"time"
 
 	"github.com/purpleidea/mgmt/converger"
 	"github.com/purpleidea/mgmt/engine"
-	"github.com/purpleidea/mgmt/engine/event"
 	"github.com/purpleidea/mgmt/pgraph"
 	"github.com/purpleidea/mgmt/util"
-
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 // State stores some state about the resource it is mapped to.
@@ -51,7 +47,7 @@ type State struct {
 	// created if needed.
 	Prefix string
 
-	//Converger converger.Converger
+	//Converger *converger.Coordinator
 
 	// Debug turns on additional output and behaviours.
 	Debug bool
@@ -61,49 +57,62 @@ type State struct {
 
 	timestamp int64 // last updated timestamp
 	isStateOK bool  // is state OK or do we need to run CheckApply ?
+	workerErr error // did the Worker error?
 
-	// events is a channel of incoming events which is read by the Watch
-	// loop for that resource. It receives events like pause, start, and
-	// poke. The channel shuts down to signal for Watch to exit.
-	eventsChan chan *event.Msg // incoming to resource
-	eventsLock *sync.Mutex     // lock around sending and closing of events channel
-	eventsDone bool            // is channel closed?
+	// doneChan closes when Watch should shut down. When any of the
+	// following channels close, it causes this to close.
+	doneChan chan struct{}
 
-	// outputChan is the channel that the engine listens on for events from
+	// processDone is closed when the Process/CheckApply function fails
+	// permanently, and wants to cause Watch to exit.
+	processDone chan struct{}
+	// watchDone is closed when the Watch function fails permanently, and we
+	// close this to signal we should definitely exit. (Often redundant.)
+	watchDone chan struct{} // could be shared with limitDone
+	// limitDone is closed when the Watch function fails permanently, and we
+	// close this to signal we should definitely exit. This happens inside
+	// of the limit loop of the Process section of Worker.
+	limitDone chan struct{} // could be shared with watchDone
+	// removeDone is closed when the vertexRemoveFn method asks for an exit.
+	// This happens when we're switching graphs. The switch to an "empty" is
+	// the equivalent of asking for a final shutdown.
+	removeDone chan struct{}
+	// eventsDone is closed when we shutdown the Process loop because we
+	// closed without error. In theory this shouldn't happen, but it could
+	// if Watch returns without error for some reason.
+	eventsDone chan struct{}
+
+	// eventsChan is the channel that the engine listens on for events from
 	// the Watch loop for that resource. The event is nil normally, except
 	// when events are sent on this channel from the engine. This only
 	// happens as a signaling mechanism when Watch has shutdown and we want
 	// to notify the Process loop which reads from this.
-	outputChan chan error // outgoing from resource
+	eventsChan chan error // outgoing from resource
 
-	wg   *sync.WaitGroup
-	exit *util.EasyExit
+	// pokeChan is a separate channel that the Process loop listens on to
+	// know when we might need to run Process. It never closes, and is safe
+	// to send on since it is buffered.
+	pokeChan chan struct{} // outgoing from resource
 
-	started chan struct{} // closes when it's started
-	stopped chan struct{} // closes when it's stopped
+	// paused represents if this particular res is paused or not.
+	paused bool
+	// pauseSignal closes to request a pause of this resource.
+	pauseSignal chan struct{}
+	// resumeSignal closes to request a resume of this resource.
+	resumeSignal chan struct{}
+	// pausedAck is used to send an ack message saying that we've paused.
+	pausedAck *util.EasyAck
 
-	starter bool // do we have an indegree of 0 ?
-	working bool // is the Main() loop running ?
+	wg *sync.WaitGroup // used for all vertex specific processes
 
-	cuid converger.UID // primary converger
-	tuid converger.UID // secondary converger
+	cuid *converger.UID // primary converger
+	tuid *converger.UID // secondary converger
 
 	init *engine.Init // a copy of the init struct passed to res Init
 }
 
 // Init initializes structures like channels.
 func (obj *State) Init() error {
-	obj.eventsChan = make(chan *event.Msg)
-	obj.eventsLock = &sync.Mutex{}
-
-	obj.outputChan = make(chan error)
-
-	obj.wg = &sync.WaitGroup{}
-	obj.exit = util.NewEasyExit()
-
-	obj.started = make(chan struct{})
-	obj.stopped = make(chan struct{})
-
 	res, isRes := obj.Vertex.(engine.Res)
 	if !isRes {
 		return fmt.Errorf("vertex is not a Res")
@@ -121,6 +130,25 @@ func (obj *State) Init() error {
 		return fmt.Errorf("the Logf function is missing")
 	}
 
+	obj.doneChan = make(chan struct{})
+
+	obj.processDone = make(chan struct{})
+	obj.watchDone = make(chan struct{})
+	obj.limitDone = make(chan struct{})
+	obj.removeDone = make(chan struct{})
+	obj.eventsDone = make(chan struct{})
+
+	obj.eventsChan = make(chan error)
+
+	obj.pokeChan = make(chan struct{}, 1) // must be buffered
+
+	//obj.paused = false // starts off as started
+	obj.pauseSignal = make(chan struct{})
+	//obj.resumeSignal = make(chan struct{}) // happens on pause
+	//obj.pausedAck = util.NewEasyAck() // happens on pause
+
+	obj.wg = &sync.WaitGroup{}
+
 	//obj.cuid = obj.Converger.Register() // gets registered in Worker()
 	//obj.tuid = obj.Converger.Register() // gets registered in Worker()
 
@@ -129,24 +157,9 @@ func (obj *State) Init() error {
 		Hostname: obj.Hostname,
 
 		// Watch:
-		Running: func() error {
-			obj.tuid.StopTimer()
-			close(obj.started)    // this is reset in the reset func
-			obj.isStateOK = false // assume we're initially dirty
-			// optimization: skip the initial send if not a starter
-			// because we'll get poked from a starter soon anyways!
-			if !obj.starter {
-				return nil
-			}
-			return obj.event()
-		},
-		Event:  obj.event,
-		Events: obj.eventsChan,
-		Read:   obj.read,
-		Dirty: func() { // TODO: should we rename this SetDirty?
-			obj.tuid.StopTimer()
-			obj.isStateOK = false
-		},
+		Running: obj.event,
+		Event:   obj.event,
+		Done:    obj.doneChan,
 
 		// CheckApply:
 		Refresh: func() bool {
@@ -231,187 +244,91 @@ func (obj *State) Close() error {
 	return err
 }
 
-// reset is run to reset the state so that Watch can run a second time. Thus is
-// needed for the Watch retry in particular.
-func (obj *State) reset() {
-	obj.started = make(chan struct{})
-	obj.stopped = make(chan struct{})
-}
-
-// Poke sends a nil message on the outputChan. This channel is used by the
-// resource to signal a possible change. This will cause the Process loop to
-// run if it can.
+// Poke sends a notification on the poke channel. This channel is used to notify
+// the Worker to run the Process/CheckApply when it can. This is used when there
+// is a need to schedule or reschedule some work which got postponed or dropped.
+// This doesn't contain any internal synchronization primitives or wait groups,
+// callers are expected to make sure that they don't leave any of these running
+// by the time the Worker() shuts down.
 func (obj *State) Poke() {
-	// add a wait group on the vertex we're poking!
-	obj.wg.Add(1)
-	defer obj.wg.Done()
-
-	// now that we've added to the wait group, obj.outputChan won't close...
-	// so see if there's an exit signal before we release the wait group!
-	// XXX: i don't think this is necessarily happening, but maybe it is?
-	// XXX: re-write some of the engine to ensure that: "the sender closes"!
-	select {
-	case <-obj.exit.Signal():
-		return // skip sending the poke b/c we're closing
-	default:
-	}
+	// redundant
+	//if len(obj.pokeChan) > 0 {
+	//	return
+	//}
 
 	select {
-	case obj.outputChan <- nil:
-
-	case <-obj.exit.Signal():
+	case obj.pokeChan <- struct{}{}:
+	default: // if chan is now full because more than one poke happened...
 	}
 }
 
-// Event sends a Pause or Start event to the resource. It can also be used to
-// send Poke events, but it's much more efficient to send them directly instead
-// of passing them through the resource.
-func (obj *State) Event(msg *event.Msg) {
-	// TODO: should these happen after the lock?
-	obj.wg.Add(1)
-	defer obj.wg.Done()
+// Pause pauses this resource. It should not be called on any already paused
+// resource. It will block until the resource pauses with an acknowledgment, or
+// until an exit for that resource is seen. If the latter happens it will error.
+// It is NOT thread-safe with the Resume() method so only call either one at a
+// time.
+func (obj *State) Pause() error {
+	if obj.paused {
+		return fmt.Errorf("already paused")
+	}
 
-	obj.eventsLock.Lock()
-	defer obj.eventsLock.Unlock()
+	obj.pausedAck = util.NewEasyAck()
+	obj.resumeSignal = make(chan struct{}) // build the resume signal
+	close(obj.pauseSignal)
+	obj.Poke() // unblock and notice the pause if necessary
 
-	if obj.eventsDone { // closing, skip events...
+	// wait for ack (or exit signal)
+	select {
+	case <-obj.pausedAck.Wait(): // we got it!
+		// we're paused
+	case <-obj.doneChan:
+		return engine.ErrClosed
+	}
+	obj.paused = true
+
+	return nil
+}
+
+// Resume unpauses this resource. It can be safely called on a brand-new
+// resource that has just started running without incident. It is NOT
+// thread-safe with the Pause() method, so only call either one at a time.
+func (obj *State) Resume() {
+	// TODO: do we need a mutex around Resume?
+	if !obj.paused { // no need to unpause brand-new resources
 		return
 	}
 
-	if msg.Kind == event.KindExit { // set this so future events don't deadlock
-		obj.Logf("exit event...")
-		obj.eventsDone = true
-		close(obj.eventsChan) // causes resource Watch loop to close
-		obj.exit.Done(nil)    // trigger exit signal to unblock some cases
-		return
-	}
+	obj.pauseSignal = make(chan struct{}) // rebuild for next pause
+	close(obj.resumeSignal)
+	//obj.Poke() // not needed, we're already waiting for resume
+
+	obj.paused = false
+
+	// no need to wait for it to resume
+	//return // implied
+}
+
+// event is a helper function to send an event to the CheckApply process loop.
+// It can be used for the initial `running` event, or any regular event. You
+// should instead use Poke() to "schedule" a new Process/CheckApply loop when
+// one might be needed. This method will block until we're unpaused and ready to
+// receive on the events channel.
+func (obj *State) event() {
+	obj.setDirty() // assume we're initially dirty
 
 	select {
-	case obj.eventsChan <- msg:
-
-	case <-obj.exit.Signal():
+	case obj.eventsChan <- nil:
+		// send!
 	}
+
+	//return // implied
 }
 
-// read is a helper function used inside the main select statement of resources.
-// If it returns an error, then this is a signal for the resource to exit.
-func (obj *State) read(msg *event.Msg) error {
-	switch msg.Kind {
-	case event.KindPoke:
-		return obj.event() // a poke needs to cause an event...
-	case event.KindStart:
-		return fmt.Errorf("unexpected start")
-	case event.KindPause:
-		// pass
-	case event.KindExit:
-		return engine.ErrSignalExit
-
-	default:
-		return fmt.Errorf("unhandled event: %+v", msg.Kind)
-	}
-
-	// we're paused now
-	select {
-	case msg, ok := <-obj.eventsChan:
-		if !ok {
-			return engine.ErrWatchExit
-		}
-		switch msg.Kind {
-		case event.KindPoke:
-			return fmt.Errorf("unexpected poke")
-		case event.KindPause:
-			return fmt.Errorf("unexpected pause")
-		case event.KindStart:
-			// resumed
-			return nil
-		case event.KindExit:
-			return engine.ErrSignalExit
-
-		default:
-			return fmt.Errorf("unhandled event: %+v", msg.Kind)
-		}
-	}
-}
-
-// event is a helper function to send an event from the resource Watch loop. It
-// can be used for the initial `running` event, or any regular event. If it
-// returns an error, then the Watch loop must return this error and shutdown.
-func (obj *State) event() error {
-	// loop until we sent on obj.outputChan or exit with error
-	for {
-		select {
-		// send "activity" event
-		case obj.outputChan <- nil:
-			return nil // sent event!
-
-		// make sure to keep handling incoming
-		case msg, ok := <-obj.eventsChan:
-			if !ok {
-				return engine.ErrWatchExit
-			}
-			switch msg.Kind {
-			case event.KindPoke:
-				// we're trying to send an event, so swallow the
-				// poke: it's what we wanted to have happen here
-				continue
-			case event.KindStart:
-				return fmt.Errorf("unexpected start")
-			case event.KindPause:
-				// pass
-			case event.KindExit:
-				return engine.ErrSignalExit
-
-			default:
-				return fmt.Errorf("unhandled event: %+v", msg.Kind)
-			}
-		}
-
-		// we're paused now
-		select {
-		case msg, ok := <-obj.eventsChan:
-			if !ok {
-				return engine.ErrWatchExit
-			}
-			switch msg.Kind {
-			case event.KindPoke:
-				return fmt.Errorf("unexpected poke")
-			case event.KindPause:
-				return fmt.Errorf("unexpected pause")
-			case event.KindStart:
-				// resumed
-			case event.KindExit:
-				return engine.ErrSignalExit
-
-			default:
-				return fmt.Errorf("unhandled event: %+v", msg.Kind)
-			}
-		}
-	}
-}
-
-// varDir returns the path to a working directory for the resource. It will try
-// and create the directory first, and return an error if this failed. The dir
-// should be cleaned up by the resource on Close if it wishes to discard the
-// contents. If it does not, then a future resource with the same kind and name
-// may see those contents in that directory. The resource should clean up the
-// contents before use if it is important that nothing exist. It is always
-// possible that contents could remain after an abrupt crash, so do not store
-// overly sensitive data unless you're aware of the risks.
-func (obj *State) varDir(extra string) (string, error) {
-	// Using extra adds additional dirs onto our namespace. An empty extra
-	// adds no additional directories.
-	if obj.Prefix == "" { // safety
-		return "", fmt.Errorf("the VarDir prefix is empty")
-	}
-
-	// an empty string at the end has no effect
-	p := fmt.Sprintf("%s/", path.Join(obj.Prefix, extra))
-	if err := os.MkdirAll(p, 0770); err != nil {
-		return "", errwrap.Wrapf(err, "can't create prefix in: %s", p)
-	}
-
-	// returns with a trailing slash as per the mgmt file res convention
-	return p, nil
+// setDirty marks the resource state as dirty. This signals to the engine that
+// CheckApply will have some work to do in order to converge it.
+func (obj *State) setDirty() {
+	obj.tuid.StopTimer()
+	obj.isStateOK = false
 }
 
 // poll is a replacement for Watch when the Poll metaparameter is used.
@@ -420,34 +337,17 @@ func (obj *State) poll(interval uint32) error {
 	ticker := time.NewTicker(time.Duration(interval) * time.Second)
 	defer ticker.Stop()
 
-	// notify engine that we're running
-	if err := obj.init.Running(); err != nil {
-		return err // exit if requested
-	}
+	obj.init.Running() // when started, notify engine that we're running
 
-	var send = false // send event?
 	for {
 		select {
 		case <-ticker.C: // received the timer event
 			obj.init.Logf("polling...")
-			send = true
-			obj.init.Dirty() // dirty
 
-		case event, ok := <-obj.init.Events:
-			if !ok {
-				return nil
-			}
-			if err := obj.init.Read(event); err != nil {
-				return err
-			}
+		case <-obj.init.Done: // signal for shutdown request
+			return nil
 		}
 
-		// do all our event sending all together to avoid duplicate msgs
-		if send {
-			send = false
-			if err := obj.init.Event(); err != nil {
-				return err // exit if requested
-			}
-		}
+		obj.init.Event() // notify engine of an event (this can block)
 	}
 }

engine/graph/vardir.go

@@ -0,0 +1,51 @@
+// Mgmt
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package graph
+
+import (
+	"fmt"
+	"os"
+	"path"
+
+	"github.com/purpleidea/mgmt/util/errwrap"
+)
+
+// varDir returns the path to a working directory for the resource. It will try
+// and create the directory first, and return an error if this failed. The dir
+// should be cleaned up by the resource on Close if it wishes to discard the
+// contents. If it does not, then a future resource with the same kind and name
+// may see those contents in that directory. The resource should clean up the
+// contents before use if it is important that nothing exist. It is always
+// possible that contents could remain after an abrupt crash, so do not store
+// overly sensitive data unless you're aware of the risks.
+func (obj *State) varDir(extra string) (string, error) {
+	// Using extra adds additional dirs onto our namespace. An empty extra
+	// adds no additional directories.
+	if obj.Prefix == "" { // safety
+		return "", fmt.Errorf("the VarDir prefix is empty")
+	}
+
+	// an empty string at the end has no effect
+	p := fmt.Sprintf("%s/", path.Join(obj.Prefix, extra))
+	if err := os.MkdirAll(p, 0770); err != nil {
+		return "", errwrap.Wrapf(err, "can't create prefix in: %s", p)
+	}
+
+	// returns with a trailing slash as per the mgmt file res convention
+	return p, nil
+}

engine/metaparams.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -22,8 +22,8 @@ import (
 	"strconv"
 
 	"github.com/purpleidea/mgmt/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
 
-	errwrap "github.com/pkg/errors"
 	"golang.org/x/time/rate"
 )
 
@@ -37,6 +37,8 @@ var DefaultMetaParams = &MetaParams{
 	Limit: rate.Inf, // defaults to no limit
 	Burst: 0,        // no burst needed on an infinite rate
 	//Sema:  []string{},
+	Rewatch: true,
+	Realize: false, // true would be more awesome, but unexpected for users
 }
 
 // MetaRes is the interface a resource must implement to support meta params.
@@ -81,6 +83,24 @@ type MetaParams struct {
 	// has a count equal to 1, is different from a sema named `foo:1` which
 	// also has a count equal to 1, but is a different semaphore.
 	Sema []string `yaml:"sema"`
+
+	// Rewatch specifies whether we re-run the Watch worker during a swap
+	// if it has errored. When doing a GraphCmp to swap the graphs, if this
+	// is true, and this particular worker has errored, then we'll remove it
+	// and add it back as a new vertex, thus causing it to run again. This
+	// is different from the Retry metaparam which applies during the normal
+	// execution. It is only when this is exhausted that we're in permanent
+	// worker failure, and only then can we rely on this metaparam.
+	Rewatch bool `yaml:"rewatch"`
+
+	// Realize ensures that the resource is guaranteed to converge at least
+	// once before a potential graph swap removes or changes it. This
+	// guarantee is useful for fast changing graphs, to ensure that the
+	// brief creation of a resource is seen. This guarantee does not prevent
+	// against the engine quitting normally, and it can't guarantee it if
+	// the resource is blocked because of a failed pre-requisite resource.
+	// XXX: Not implemented!
+	Realize bool `yaml:"realize"`
 }
 
 // Cmp compares two AutoGroupMeta structs and determines if they're equivalent.
@@ -118,6 +138,13 @@ func (obj *MetaParams) Cmp(meta *MetaParams) error {
 		return errwrap.Wrapf(err, "values for Sema are different")
 	}
 
+	if obj.Rewatch != meta.Rewatch {
+		return fmt.Errorf("values for Rewatch are different")
+	}
+	if obj.Realize != meta.Realize {
+		return fmt.Errorf("values for Realize are different")
+	}
+
 	return nil
 }
 
@@ -147,13 +174,15 @@ func (obj *MetaParams) Copy() *MetaParams {
 		copy(sema, obj.Sema)
 	}
 	return &MetaParams{
-		Noop:  obj.Noop,
-		Retry: obj.Retry,
-		Delay: obj.Delay,
-		Poll:  obj.Poll,
-		Limit: obj.Limit, // FIXME: can we copy this type like this? test me!
-		Burst: obj.Burst,
-		Sema:  sema,
+		Noop:    obj.Noop,
+		Retry:   obj.Retry,
+		Delay:   obj.Delay,
+		Poll:    obj.Poll,
+		Limit:   obj.Limit, // FIXME: can we copy this type like this? test me!
+		Burst:   obj.Burst,
+		Sema:    sema,
+		Rewatch: obj.Rewatch,
+		Realize: obj.Realize,
 	}
 }
 

engine/metaparams_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/refresh.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/resources.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -21,9 +21,8 @@ import (
 	"encoding/gob"
 	"fmt"
 
-	"github.com/purpleidea/mgmt/engine/event"
+	"github.com/purpleidea/mgmt/util/errwrap"
 
-	errwrap "github.com/pkg/errors"
 	"gopkg.in/yaml.v2"
 )
 
@@ -93,22 +92,14 @@ type Init struct {
 	// Called from within Watch:
 
 	// Running must be called after your watches are all started and ready.
-	Running func() error
+	Running func()
 
 	// Event sends an event notifying the engine of a possible state change.
-	Event func() error
+	Event func()
 
-	// Events returns a channel that we must watch for messages from the
-	// engine. When it closes, this is a signal to shutdown.
-	Events chan *event.Msg
-
-	// Read processes messages that come in from the Events channel. It is a
-	// helper method that knows how to handle the pause mechanism correctly.
-	Read func(*event.Msg) error
-
-	// Dirty marks the resource state as dirty. This signals to the engine
-	// that CheckApply will have some work to do in order to converge it.
-	Dirty func()
+	// Done returns a channel that will close to signal to us that it's time
+	// for us to shutdown.
+	Done chan struct{}
 
 	// Called from within CheckApply:
 

engine/resources/augeas.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -27,8 +27,8 @@ import (
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/engine/traits"
 	"github.com/purpleidea/mgmt/recwatch"
+	"github.com/purpleidea/mgmt/util/errwrap"
 
-	errwrap "github.com/pkg/errors"
 	// FIXME: we vendor go/augeas because master requires augeas 1.6.0
 	// and libaugeas-dev-1.6.0 is not yet available in a PPA.
 	"honnef.co/go/augeas"
@@ -135,10 +135,7 @@ func (obj *AugeasRes) Watch() error {
 	}
 	defer obj.recWatcher.Close()
 
-	// notify engine that we're running
-	if err := obj.init.Running(); err != nil {
-		return err // exit if requested
-	}
+	obj.init.Running() // when started, notify engine that we're running
 
 	var send = false // send event?
 	for {
@@ -158,23 +155,15 @@ func (obj *AugeasRes) Watch() error {
 				obj.init.Logf("Event(%s): %v", event.Body.Name, event.Body.Op)
 			}
 			send = true
-			obj.init.Dirty() // dirty
 
-		case event, ok := <-obj.init.Events:
-			if !ok {
-				return nil
-			}
-			if err := obj.init.Read(event); err != nil {
-				return err
-			}
+		case <-obj.init.Done: // closed by the engine to signal shutdown
+			return nil
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
 			send = false
-			if err := obj.init.Event(); err != nil {
-				return err // exit if requested
-			}
+			obj.init.Event() // notify engine of an event (this can block)
 		}
 	}
 }

engine/resources/aws_ec2.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -34,6 +34,7 @@ import (
 
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/engine/traits"
+	"github.com/purpleidea/mgmt/util/errwrap"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
@@ -42,8 +43,6 @@ import (
 	cwe "github.com/aws/aws-sdk-go/service/cloudwatchevents"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/sns"
-	multierr "github.com/hashicorp/go-multierror"
-	errwrap "github.com/pkg/errors"
 )
 
 func init() {
@@ -393,17 +392,14 @@ func (obj *AwsEc2Res) Close() error {
 	// clean up sns objects created by Init/snsWatch
 	if obj.snsClient != nil {
 		// delete the topic and associated subscriptions
-		if err := obj.snsDeleteTopic(obj.snsTopicArn); err != nil {
-			errList = multierr.Append(errList, err)
-		}
+		e1 := obj.snsDeleteTopic(obj.snsTopicArn)
+		errList = errwrap.Append(errList, e1)
 		// remove the target
-		if err := obj.cweRemoveTarget(CweTargetID, CweRuleName); err != nil {
-			errList = multierr.Append(errList, err)
-		}
+		e2 := obj.cweRemoveTarget(CweTargetID, CweRuleName)
+		errList = errwrap.Append(errList, e2)
 		// delete the cloudwatch rule
-		if err := obj.cweDeleteRule(CweRuleName); err != nil {
-			errList = multierr.Append(errList, err)
-		}
+		e3 := obj.cweDeleteRule(CweRuleName)
+		errList = errwrap.Append(errList, e3)
 	}
 
 	return errList
@@ -423,9 +419,7 @@ func (obj *AwsEc2Res) longpollWatch() error {
 
 	// We tell the engine that we're running right away. This is not correct,
 	// but the api doesn't have a way to signal when the waiters are ready.
-	if err := obj.init.Running(); err != nil {
-		return err // exit if requested
-	}
+	obj.init.Running() // when started, notify engine that we're running
 
 	// cancellable context used for exiting cleanly
 	ctx, cancel := context.WithCancel(context.TODO())
@@ -488,14 +482,6 @@ func (obj *AwsEc2Res) longpollWatch() error {
 	// process events from the goroutine
 	for {
 		select {
-		case event, ok := <-obj.init.Events:
-			if !ok {
-				return nil
-			}
-			if err := obj.init.Read(event); err != nil {
-				return err
-			}
-
 		case msg, ok := <-obj.awsChan:
 			if !ok {
 				return nil
@@ -509,15 +495,16 @@ func (obj *AwsEc2Res) longpollWatch() error {
 				continue
 			default:
 				obj.init.Logf("State: %v", msg.state)
-				obj.init.Dirty() // dirty
 				send = true
 			}
+
+		case <-obj.init.Done: // closed by the engine to signal shutdown
+			return nil
 		}
+
 		if send {
 			send = false
-			if err := obj.init.Event(); err != nil {
-				return err // exit if requested
-			}
+			obj.init.Event() // notify engine of an event (this can block)
 		}
 	}
 }
@@ -587,14 +574,6 @@ func (obj *AwsEc2Res) snsWatch() error {
 	// process events
 	for {
 		select {
-		case event, ok := <-obj.init.Events:
-			if !ok {
-				return nil
-			}
-			if err := obj.init.Read(event); err != nil {
-				return err
-			}
-
 		case msg, ok := <-obj.awsChan:
 			if !ok {
 				return nil
@@ -607,26 +586,25 @@ func (obj *AwsEc2Res) snsWatch() error {
 			// is confirmed, we are ready to receive events, so we
 			// can notify the engine that we're running.
 			if msg.event == awsEc2EventWatchReady {
-				if err := obj.init.Running(); err != nil {
-					return err // exit if requested
-				}
+				obj.init.Running() // when started, notify engine that we're running
 				continue
 			}
 			obj.init.Logf("State: %v", msg.event)
-			obj.init.Dirty() // dirty
 			send = true
+
+		case <-obj.init.Done: // closed by the engine to signal shutdown
+			return nil
 		}
+
 		if send {
 			send = false
-			if err := obj.init.Event(); err != nil {
-				return err // exit if requested
-			}
+			obj.init.Event() // notify engine of an event (this can block)
 		}
 	}
 }
 
 // CheckApply method for AwsEc2 resource.
-func (obj *AwsEc2Res) CheckApply(apply bool) (checkOK bool, err error) {
+func (obj *AwsEc2Res) CheckApply(apply bool) (bool, error) {
 	obj.init.Logf("CheckApply(%t)", apply)
 
 	// find the instance we need to check

engine/resources/config_etcd.go

@@ -0,0 +1,250 @@
+// Mgmt
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/purpleidea/mgmt/engine"
+	"github.com/purpleidea/mgmt/engine/traits"
+	"github.com/purpleidea/mgmt/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
+)
+
+func init() {
+	engine.RegisterResource("config:etcd", func() engine.Res { return &ConfigEtcdRes{} })
+}
+
+const (
+	sizeCheckApplyTimeout = 5 * time.Second
+)
+
+// ConfigEtcdRes is a resource that sets mgmt's etcd configuration.
+type ConfigEtcdRes struct {
+	traits.Base // add the base methods without re-implementation
+
+	init *engine.Init
+
+	// IdealClusterSize is the requested minimum size of the cluster. If you
+	// set this to zero, it will cause a cluster wide shutdown if
+	// AllowSizeShutdown is true. If it's not true, then it will cause a
+	// validation error.
+	IdealClusterSize uint16 `lang:"idealclustersize"`
+	// AllowSizeShutdown is a required safety flag that you must set to true
+	// if you want to allow causing a cluster shutdown by setting
+	// IdealClusterSize to zero.
+	AllowSizeShutdown bool `lang:"allow_size_shutdown"`
+
+	// sizeFlag determines whether sizeCheckApply already ran or not.
+	sizeFlag bool
+
+	interruptChan chan struct{}
+	wg            *sync.WaitGroup
+}
+
+// Default returns some sensible defaults for this resource.
+func (obj *ConfigEtcdRes) Default() engine.Res {
+	return &ConfigEtcdRes{}
+}
+
+// Validate if the params passed in are valid data.
+func (obj *ConfigEtcdRes) Validate() error {
+	if obj.IdealClusterSize < 0 {
+		return fmt.Errorf("the IdealClusterSize param must be positive")
+	}
+
+	if obj.IdealClusterSize == 0 && !obj.AllowSizeShutdown {
+		return fmt.Errorf("the IdealClusterSize can't be zero if AllowSizeShutdown is false")
+	}
+
+	return nil
+}
+
+// Init runs some startup code for this resource.
+func (obj *ConfigEtcdRes) Init(init *engine.Init) error {
+	obj.init = init // save for later
+
+	obj.interruptChan = make(chan struct{})
+	obj.wg = &sync.WaitGroup{}
+
+	return nil
+}
+
+// Close is run by the engine to clean up after the resource is done.
+func (obj *ConfigEtcdRes) Close() error {
+	obj.wg.Wait() // bonus
+	return nil
+}
+
+// Watch is the primary listener for this resource and it outputs events.
+func (obj *ConfigEtcdRes) Watch() error {
+	obj.wg.Add(1)
+	defer obj.wg.Done()
+	// FIXME: add timeout to context
+	// The obj.init.Done channel is closed by the engine to signal shutdown.
+	ctx, cancel := util.ContextWithCloser(context.Background(), obj.init.Done)
+	defer cancel()
+	ch, err := obj.init.World.IdealClusterSizeWatch(util.CtxWithWg(ctx, obj.wg))
+	if err != nil {
+		return errwrap.Wrapf(err, "could not watch ideal cluster size")
+	}
+
+	obj.init.Running() // when started, notify engine that we're running
+
+Loop:
+	for {
+		select {
+		case event, ok := <-ch:
+			if !ok {
+				break Loop
+			}
+			if obj.init.Debug {
+				obj.init.Logf("event: %+v", event)
+			}
+			// pass through and send an event
+
+		case <-obj.init.Done: // closed by the engine to signal shutdown
+		}
+
+		obj.init.Event() // notify engine of an event (this can block)
+	}
+
+	return nil
+}
+
+// sizeCheckApply sets the IdealClusterSize parameter. If it sees a value change
+// to zero, then it *won't* try and change it away from zero, because it assumes
+// that someone has requested a shutdown. If the value is seen on first startup,
+// then it will change it, because it might be a zero from the previous cluster.
+func (obj *ConfigEtcdRes) sizeCheckApply(apply bool) (bool, error) {
+	wg := &sync.WaitGroup{}
+	defer wg.Wait() // this must be above the defer cancel() call
+	ctx, cancel := context.WithTimeout(context.Background(), sizeCheckApplyTimeout)
+	defer cancel()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		select {
+		case <-obj.interruptChan:
+			cancel()
+		case <-ctx.Done():
+			// let this exit
+		}
+	}()
+
+	val, err := obj.init.World.IdealClusterSizeGet(ctx)
+	if err != nil {
+		return false, errwrap.Wrapf(err, "could not get ideal cluster size")
+	}
+
+	// if we got a value of zero, and we've already run before, then it's ok
+	if obj.IdealClusterSize != 0 && val == 0 && obj.sizeFlag {
+		obj.init.Logf("impending cluster shutdown, not setting ideal cluster size")
+		return true, nil // impending shutdown, don't try and cancel it.
+	}
+	obj.sizeFlag = true
+
+	// must be done after setting the above flag
+	if obj.IdealClusterSize == val { // state is correct
+		return true, nil
+	}
+
+	if !apply {
+		return false, nil
+	}
+
+	// set!
+	// This is run as a transaction so we detect if we needed to change it.
+	changed, err := obj.init.World.IdealClusterSizeSet(ctx, obj.IdealClusterSize)
+	if err != nil {
+		return false, errwrap.Wrapf(err, "could not set ideal cluster size")
+	}
+	if !changed {
+		return true, nil // we lost a race, which means no change needed
+	}
+	obj.init.Logf("set dynamic cluster size to: %d", obj.IdealClusterSize)
+
+	return false, nil
+}
+
+// CheckApply method for Noop resource. Does nothing, returns happy!
+func (obj *ConfigEtcdRes) CheckApply(apply bool) (bool, error) {
+	checkOK := true
+
+	if c, err := obj.sizeCheckApply(apply); err != nil {
+		return false, err
+	} else if !c {
+		checkOK = false
+	}
+
+	// TODO: add more config settings management here...
+	//if c, err := obj.TODOCheckApply(apply); err != nil {
+	//	return false, err
+	//} else if !c {
+	//	checkOK = false
+	//}
+
+	return checkOK, nil // w00t
+}
+
+// Cmp compares two resources and returns an error if they are not equivalent.
+func (obj *ConfigEtcdRes) Cmp(r engine.Res) error {
+	// we can only compare ConfigEtcdRes to others of the same resource kind
+	res, ok := r.(*ConfigEtcdRes)
+	if !ok {
+		return fmt.Errorf("not a %s", obj.Kind())
+	}
+
+	if obj.IdealClusterSize != res.IdealClusterSize {
+		return fmt.Errorf("the IdealClusterSize param differs")
+	}
+	if obj.AllowSizeShutdown != res.AllowSizeShutdown {
+		return fmt.Errorf("the AllowSizeShutdown param differs")
+	}
+
+	return nil
+}
+
+// Interrupt is called to ask the execution of this resource to end early.
+func (obj *ConfigEtcdRes) Interrupt() error {
+	close(obj.interruptChan)
+	return nil
+}
+
+// UnmarshalYAML is the custom unmarshal handler for this struct.
+// It is primarily useful for setting the defaults.
+func (obj *ConfigEtcdRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type rawRes ConfigEtcdRes // indirection to avoid infinite recursion
+
+	def := obj.Default()            // get the default
+	res, ok := def.(*ConfigEtcdRes) // put in the right format
+	if !ok {
+		return fmt.Errorf("could not convert to ConfigEtcdRes")
+	}
+	raw := rawRes(*res) // convert; the defaults go here
+
+	if err := unmarshal(&raw); err != nil {
+		return err
+	}
+
+	*obj = ConfigEtcdRes(raw) // restore from indirection with type conversion!
+	return nil
+}

engine/resources/cron.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -31,12 +31,12 @@ import (
 	engineUtil "github.com/purpleidea/mgmt/engine/util"
 	"github.com/purpleidea/mgmt/recwatch"
 	"github.com/purpleidea/mgmt/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
 
 	sdbus "github.com/coreos/go-systemd/dbus"
 	"github.com/coreos/go-systemd/unit"
 	systemdUtil "github.com/coreos/go-systemd/util"
 	"github.com/godbus/dbus"
-	errwrap "github.com/pkg/errors"
 )
 
 const (
@@ -271,10 +271,7 @@ func (obj *CronRes) Watch() error {
 	}
 	defer obj.recWatcher.Close()
 
-	// notify engine that we're running
-	if err := obj.init.Running(); err != nil {
-		return err // exit if requested
-	}
+	obj.init.Running() // when started, notify engine that we're running
 
 	var send = false // send event?
 	for {
@@ -285,7 +282,7 @@ func (obj *CronRes) Watch() error {
 				obj.init.Logf("%+v", event)
 			}
 			send = true
-			obj.init.Dirty() // dirty
+
 		case event, ok := <-obj.recWatcher.Events():
 			// process unit file recwatch events
 			if !ok { // channel shutdown
@@ -298,21 +295,14 @@ func (obj *CronRes) Watch() error {
 				obj.init.Logf("Event(%s): %v", event.Body.Name, event.Body.Op)
 			}
 			send = true
-			obj.init.Dirty() // dirty
-		case event, ok := <-obj.init.Events:
-			if !ok {
-				return nil
-			}
-			if err := obj.init.Read(event); err != nil {
-				return err
-			}
+
+		case <-obj.init.Done: // closed by the engine to signal shutdown
+			return nil
 		}
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
 			send = false
-			if err := obj.init.Event(); err != nil {
-				return err // exit if requested
-			}
+			obj.init.Event() // notify engine of an event (this can block)
 		}
 	}
 }
@@ -320,28 +310,29 @@ func (obj *CronRes) Watch() error {
 // CheckApply is run to check the state and, if apply is true, to apply the
 // necessary changes to reach the desired state. This is run before Watch and
 // again if Watch finds a change occurring to the state.
-func (obj *CronRes) CheckApply(apply bool) (checkOK bool, err error) {
-	ok := true
+func (obj *CronRes) CheckApply(apply bool) (bool, error) {
+	checkOK := true
 	// use the embedded file resource to apply the correct state
 	if c, err := obj.file.CheckApply(apply); err != nil {
 		return false, errwrap.Wrapf(err, "nested file failed")
 	} else if !c {
-		ok = false
+		checkOK = false
 	}
 	// check timer state and apply the defined state if needed
 	if c, err := obj.unitCheckApply(apply); err != nil {
 		return false, errwrap.Wrapf(err, "unitCheckApply error")
 	} else if !c {
-		ok = false
+		checkOK = false
 	}
-	return ok, nil
+	return checkOK, nil
 }
 
 // unitCheckApply checks the state of the systemd-timer unit and, if apply is
 // true, applies the defined state.
-func (obj *CronRes) unitCheckApply(apply bool) (checkOK bool, err error) {
+func (obj *CronRes) unitCheckApply(apply bool) (bool, error) {
 	var conn *sdbus.Conn
 	var godbusConn *dbus.Conn
+	var err error
 
 	// this resource depends on systemd to ensure that it's running
 	if !systemdUtil.IsRunningSystemd() {

engine/resources/docker_container.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -30,13 +30,13 @@ import (
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/engine/traits"
 	"github.com/purpleidea/mgmt/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
 	"github.com/docker/go-connections/nat"
-	errwrap "github.com/pkg/errors"
 )
 
 const (
@@ -168,10 +168,7 @@ func (obj *DockerContainerRes) Watch() error {
 
 	eventChan, errChan := obj.client.Events(ctx, types.EventsOptions{})
 
-	// notify engine that we're running
-	if err := obj.init.Running(); err != nil {
-		return err // exit if requested
-	}
+	obj.init.Running() // when started, notify engine that we're running
 
 	var send = false // send event?
 	for {
@@ -184,33 +181,27 @@ func (obj *DockerContainerRes) Watch() error {
 				obj.init.Logf("%+v", event)
 			}
 			send = true
-			obj.init.Dirty() // dirty
+
 		case err, ok := <-errChan:
 			if !ok {
 				return nil
 			}
 			return err
-		case event, ok := <-obj.init.Events:
-			if !ok {
-				return nil
-			}
-			if err := obj.init.Read(event); err != nil {
-				return err
-			}
+
+		case <-obj.init.Done: // closed by the engine to signal shutdown
+			return nil
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
 			send = false
-			if err := obj.init.Event(); err != nil {
-				return err // exit if requested
-			}
+			obj.init.Event() // notify engine of an event (this can block)
 		}
 	}
 }
 
 // CheckApply method for Docker resource.
-func (obj *DockerContainerRes) CheckApply(apply bool) (checkOK bool, err error) {
+func (obj *DockerContainerRes) CheckApply(apply bool) (bool, error) {
 	var id string
 	var destroy bool
 

engine/resources/docker_container_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/resources/exec.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -20,19 +20,19 @@ package resources
 import (
 	"bufio"
 	"bytes"
+	"context"
 	"fmt"
 	"os/exec"
 	"os/user"
 	"strings"
 	"sync"
 	"syscall"
+	"time"
 
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/engine/traits"
 	engineUtil "github.com/purpleidea/mgmt/engine/util"
-	"github.com/purpleidea/mgmt/util"
-
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 func init() {
@@ -43,23 +43,60 @@ func init() {
 type ExecRes struct {
 	traits.Base // add the base methods without re-implementation
 	traits.Edgeable
+	traits.Sendable
 
 	init *engine.Init
 
-	Cmd        string  `yaml:"cmd"`        // the command to run
-	Shell      string  `yaml:"shell"`      // the (optional) shell to use to run the cmd
-	Timeout    int     `yaml:"timeout"`    // the cmd timeout in seconds
-	WatchCmd   string  `yaml:"watchcmd"`   // the watch command to run
-	WatchShell string  `yaml:"watchshell"` // the (optional) shell to use to run the watch cmd
-	IfCmd      string  `yaml:"ifcmd"`      // the if command to run
-	IfShell    string  `yaml:"ifshell"`    // the (optional) shell to use to run the if cmd
-	User       string  `yaml:"user"`       // the (optional) user to use to execute the command
-	Group      string  `yaml:"group"`      // the (optional) group to use to execute the command
-	Output     *string // all cmd output, read only, do not set!
-	Stdout     *string // the cmd stdout, read only, do not set!
-	Stderr     *string // the cmd stderr, read only, do not set!
+	// Cmd is the command to run. If this is not specified, we use the name.
+	Cmd string `yaml:"cmd"`
+	// Args is a list of args to pass to Cmd. This can be used *instead* of
+	// passing the full command and args as a single string to Cmd. It can
+	// only be used when a Shell is *not* specified. The advantage of this
+	// is that you don't have to worry about escape characters.
+	Args []string `yaml:"args"`
+	// Cmd is the dir to run the command in. If empty, then this will use
+	// the working directory of the calling process. (This process is mgmt,
+	// not the process being run here.)
+	Cwd string `yaml:"cwd"`
+	// Shell is the (optional) shell to use to run the cmd. If you specify
+	// this, then you can't use the Args parameter.
+	Shell string `yaml:"shell"`
+	// Timeout is the number of seconds to wait before sending a Kill to the
+	// running command. If the Kill is received before the process exits,
+	// then this be treated as an error.
+	Timeout uint64 `yaml:"timeout"`
 
-	wg *sync.WaitGroup
+	// Watch is the command to run to detect event changes. Each line of
+	// output from this command is treated as an event.
+	WatchCmd string `yaml:"watchcmd"`
+	// WatchCwd is the Cwd for the WatchCmd. See the docs for Cwd.
+	WatchCwd string `yaml:"watchcwd"`
+	// WatchShell is the Shell for the WatchCmd. See the docs for Shell.
+	WatchShell string `yaml:"watchshell"`
+
+	// IfCmd is the command that runs to guard against running the Cmd. If
+	// this command succeeds, then Cmd *will* be run. If this command
+	// returns a non-zero result, then the Cmd will not be run. Any error
+	// scenario or timeout will cause the resource to error.
+	IfCmd string `yaml:"ifcmd"`
+	// IfCwd is the Cwd for the IfCmd. See the docs for Cwd.
+	IfCwd string `yaml:"ifcwd"`
+	// IfShell is the Shell for the IfCmd. See the docs for Shell.
+	IfShell string `yaml:"ifshell"`
+
+	// User is the (optional) user to use to execute the command. It is used
+	// for any command being run.
+	User string `yaml:"user"`
+	// Group is the (optional) group to use to execute the command. It is
+	// used for any command being run.
+	Group string `yaml:"group"`
+
+	output *string // all cmd output, read only, do not set!
+	stdout *string // the cmd stdout, read only, do not set!
+	stderr *string // the cmd stderr, read only, do not set!
+
+	interruptChan chan struct{}
+	wg            *sync.WaitGroup
 }
 
 // Default returns some sensible defaults for this resource.
@@ -67,10 +104,27 @@ func (obj *ExecRes) Default() engine.Res {
 	return &ExecRes{}
 }
 
+// getCmd returns the actual command to run. When Cmd is not specified, we use
+// the Name.
+func (obj *ExecRes) getCmd() string {
+	if obj.Cmd != "" {
+		return obj.Cmd
+	}
+	return obj.Name()
+}
+
 // Validate if the params passed in are valid data.
 func (obj *ExecRes) Validate() error {
-	if obj.Cmd == "" { // this is the only thing that is really required
-		return fmt.Errorf("command can't be empty")
+	if obj.getCmd() == "" { // this is the only thing that is really required
+		return fmt.Errorf("the Cmd can't be empty")
+	}
+
+	split := strings.Fields(obj.getCmd())
+	if len(obj.Args) > 0 && obj.Shell != "" {
+		return fmt.Errorf("the Args param can't be used with a Shell")
+	}
+	if len(obj.Args) > 0 && len(split) > 1 {
+		return fmt.Errorf("the Args param can't be used when Cmd has args")
 	}
 
 	// check that, if an user or a group is set, we're running as root
@@ -80,7 +134,7 @@ func (obj *ExecRes) Validate() error {
 			return errwrap.Wrapf(err, "error looking up current user")
 		}
 		if currentUser.Uid != "0" {
-			return errwrap.Errorf("running as root is required if you want to use exec with a different user/group")
+			return fmt.Errorf("running as root is required if you want to use exec with a different user/group")
 		}
 	}
 
@@ -91,6 +145,7 @@ func (obj *ExecRes) Validate() error {
 func (obj *ExecRes) Init(init *engine.Init) error {
 	obj.init = init // save for later
 
+	obj.interruptChan = make(chan struct{})
 	obj.wg = &sync.WaitGroup{}
 
 	return nil
@@ -103,7 +158,7 @@ func (obj *ExecRes) Close() error {
 
 // Watch is the primary listener for this resource and it outputs events.
 func (obj *ExecRes) Watch() error {
-	ioChan := make(chan *bufioOutput)
+	ioChan := make(chan *cmdOutput)
 	defer obj.wg.Wait()
 
 	if obj.WatchCmd != "" {
@@ -121,8 +176,11 @@ func (obj *ExecRes) Watch() error {
 			cmdName = obj.WatchShell // usually bash, or sh
 			cmdArgs = []string{"-c", obj.WatchCmd}
 		}
-		cmd := exec.Command(cmdName, cmdArgs...)
-		//cmd.Dir = "" // look for program in pwd ?
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		cmd := exec.CommandContext(ctx, cmdName, cmdArgs...)
+		cmd.Dir = obj.WatchCwd // run program in pwd if ""
 		// ignore signals sent to parent process (we're in our own group)
 		cmd.SysProcAttr = &syscall.SysProcAttr{
 			Setpgid: true,
@@ -135,29 +193,12 @@ func (obj *ExecRes) Watch() error {
 			return errwrap.Wrapf(err, "error while setting credential")
 		}
 
-		cmdReader, err := cmd.StdoutPipe()
-		if err != nil {
-			return errwrap.Wrapf(err, "error creating StdoutPipe for Cmd")
+		if ioChan, err = obj.cmdOutputRunner(ctx, cmd); err != nil {
+			return errwrap.Wrapf(err, "error starting WatchCmd")
 		}
-		scanner := bufio.NewScanner(cmdReader)
-
-		defer cmd.Wait() // wait for the command to exit before return!
-		defer func() {
-			// FIXME: without wrapping this in this func it panic's
-			// when running certain graphs... why?
-			cmd.Process.Kill() // shutdown the Watch command on exit
-		}()
-		if err := cmd.Start(); err != nil {
-			return errwrap.Wrapf(err, "error starting Cmd")
-		}
-
-		ioChan = obj.bufioChanScanner(scanner)
 	}
 
-	// notify engine that we're running
-	if err := obj.init.Running(); err != nil {
-		return err // exit if requested
-	}
+	obj.init.Running() // when started, notify engine that we're running
 
 	var send = false // send event?
 	for {
@@ -169,32 +210,44 @@ func (obj *ExecRes) Watch() error {
 				return fmt.Errorf("reached EOF")
 			}
 			if err := data.err; err != nil {
-				// error reading input?
-				return errwrap.Wrapf(err, "unknown error")
+				// error reading input or cmd failure
+				exitErr, ok := err.(*exec.ExitError) // embeds an os.ProcessState
+				if !ok {
+					// command failed in some bad way
+					return errwrap.Wrapf(err, "unknown error")
+				}
+				pStateSys := exitErr.Sys() // (*os.ProcessState) Sys
+				wStatus, ok := pStateSys.(syscall.WaitStatus)
+				if !ok {
+					return errwrap.Wrapf(err, "error running cmd")
+				}
+				exitStatus := wStatus.ExitStatus()
+				obj.init.Logf("watchcmd exited with: %d", exitStatus)
+				if exitStatus != 0 {
+					return errwrap.Wrapf(err, "unexpected exit status of zero")
+				}
+				return err // i'm not sure if this could happen
 			}
 
 			// each time we get a line of output, we loop!
-			obj.init.Logf("watch output: %s", data.text)
+			if s := data.text; s == "" {
+				obj.init.Logf("watch output is empty!")
+			} else {
+				obj.init.Logf("watch output is:")
+				obj.init.Logf(s)
+			}
 			if data.text != "" {
 				send = true
-				obj.init.Dirty() // dirty
 			}
 
-		case event, ok := <-obj.init.Events:
-			if !ok {
-				return nil
-			}
-			if err := obj.init.Read(event); err != nil {
-				return err
-			}
+		case <-obj.init.Done: // closed by the engine to signal shutdown
+			return nil
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
 			send = false
-			if err := obj.init.Event(); err != nil {
-				return err // exit if requested
-			}
+			obj.init.Event() // notify engine of an event (this can block)
 		}
 	}
 }
@@ -208,7 +261,6 @@ func (obj *ExecRes) CheckApply(apply bool) (bool, error) {
 	// have a chance to execute, and all without the check of obj.Refresh()!
 
 	if obj.IfCmd != "" { // if there is no onlyif check, we should just run
-
 		var cmdName string
 		var cmdArgs []string
 		if obj.IfShell == "" {
@@ -224,6 +276,7 @@ func (obj *ExecRes) CheckApply(apply bool) (bool, error) {
 			cmdArgs = []string{"-c", obj.IfCmd}
 		}
 		cmd := exec.Command(cmdName, cmdArgs...)
+		cmd.Dir = obj.IfCwd // run program in pwd if ""
 		// ignore signals sent to parent process (we're in our own group)
 		cmd.SysProcAttr = &syscall.SysProcAttr{
 			Setpgid: true,
@@ -236,11 +289,42 @@ func (obj *ExecRes) CheckApply(apply bool) (bool, error) {
 			return false, errwrap.Wrapf(err, "error while setting credential")
 		}
 
+		var out splitWriter
+		out.Init()
+		cmd.Stdout = out.Stdout
+		cmd.Stderr = out.Stderr
+
 		if err := cmd.Run(); err != nil {
-			// TODO: check exit value
+			exitErr, ok := err.(*exec.ExitError) // embeds an os.ProcessState
+			if !ok {
+				// command failed in some bad way
+				return false, err
+			}
+			pStateSys := exitErr.Sys() // (*os.ProcessState) Sys
+			wStatus, ok := pStateSys.(syscall.WaitStatus)
+			if !ok {
+				return false, errwrap.Wrapf(err, "error running cmd")
+			}
+			exitStatus := wStatus.ExitStatus()
+			if exitStatus == 0 {
+				return false, fmt.Errorf("unexpected exit status of zero")
+			}
+
+			obj.init.Logf("ifcmd exited with: %d", exitStatus)
+			if s := out.String(); s == "" {
+				obj.init.Logf("ifcmd output is empty!")
+			} else {
+				obj.init.Logf("ifcmd output is:")
+				obj.init.Logf(s)
+			}
 			return true, nil // don't run
 		}
-
+		if s := out.String(); s == "" {
+			obj.init.Logf("ifcmd output is empty!")
+		} else {
+			obj.init.Logf("ifcmd output is:")
+			obj.init.Logf(s)
+		}
 	}
 
 	// state is not okay, no work done, exit, but without error
@@ -256,17 +340,34 @@ func (obj *ExecRes) CheckApply(apply bool) (bool, error) {
 		// call without a shell
 		// FIXME: are there still whitespace splitting issues?
 		// TODO: we could make the split character user selectable...!
-		split := strings.Fields(obj.Cmd)
+		split := strings.Fields(obj.getCmd())
 		cmdName = split[0]
 		//d, _ := os.Getwd() // TODO: how does this ever error ?
 		//cmdName = path.Join(d, cmdName)
 		cmdArgs = split[1:]
+		if len(obj.Args) > 0 {
+			if len(split) != 1 { // should not happen
+				return false, fmt.Errorf("validation error")
+			}
+			cmdArgs = obj.Args
+		}
 	} else {
 		cmdName = obj.Shell // usually bash, or sh
-		cmdArgs = []string{"-c", obj.Cmd}
+		cmdArgs = []string{"-c", obj.getCmd()}
 	}
-	cmd := exec.Command(cmdName, cmdArgs...)
-	//cmd.Dir = "" // look for program in pwd ?
+
+	wg := &sync.WaitGroup{}
+	defer wg.Wait() // this must be above the defer cancel() call
+	var ctx context.Context
+	var cancel context.CancelFunc
+	if obj.Timeout > 0 { // cmd.Process.Kill() is called on timeout
+		ctx, cancel = context.WithTimeout(context.Background(), time.Duration(obj.Timeout)*time.Second)
+	} else { // zero timeout means no timer
+		ctx, cancel = context.WithCancel(context.Background())
+	}
+	defer cancel()
+	cmd := exec.CommandContext(ctx, cmdName, cmdArgs...)
+	cmd.Dir = obj.Cwd // run program in pwd if ""
 	// ignore signals sent to parent process (we're in our own group)
 	cmd.SysProcAttr = &syscall.SysProcAttr{
 		Setpgid: true,
@@ -290,35 +391,32 @@ func (obj *ExecRes) CheckApply(apply bool) (bool, error) {
 		return false, errwrap.Wrapf(err, "error starting cmd")
 	}
 
-	timeout := obj.Timeout
-	if timeout == 0 { // zero timeout means no timer, so disable it
-		timeout = -1
-	}
-	done := make(chan error)
-	go func() { done <- cmd.Wait() }()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		select {
+		case <-obj.interruptChan:
+			cancel()
+		case <-ctx.Done():
+			// let this exit
+		}
+	}()
 
-	select {
-	case e := <-done:
-		err = e // store
-
-	case <-util.TimeAfterOrBlock(timeout):
-		cmd.Process.Kill() // TODO: check error?
-		return false, fmt.Errorf("timeout for cmd")
-	}
+	err = cmd.Wait() // we can unblock this with the timeout
 
 	// save in memory for send/recv
 	// we use pointers to strings to indicate if used or not
 	if out.Stdout.Activity || out.Stderr.Activity {
 		str := out.String()
-		obj.Output = &str
+		obj.output = &str
 	}
 	if out.Stdout.Activity {
 		str := out.Stdout.String()
-		obj.Stdout = &str
+		obj.stdout = &str
 	}
 	if out.Stderr.Activity {
 		str := out.Stderr.String()
-		obj.Stderr = &str
+		obj.stderr = &str
 	}
 
 	// process the err result from cmd, we process non-zero exits here too!
@@ -329,7 +427,18 @@ func (obj *ExecRes) CheckApply(apply bool) (bool, error) {
 		if !ok {
 			return false, errwrap.Wrapf(err, "error running cmd")
 		}
-		return false, fmt.Errorf("cmd error, exit status: %d", wStatus.ExitStatus())
+		exitStatus := wStatus.ExitStatus()
+		if !wStatus.Signaled() { // not a timeout or cancel (no signal)
+			return false, errwrap.Wrapf(err, "cmd error, exit status: %d", exitStatus)
+		}
+		sig := wStatus.Signal()
+
+		// we get this on timeout, because ctx calls cmd.Process.Kill()
+		if sig == syscall.SIGKILL {
+			return false, errwrap.Wrapf(err, "cmd timeout, exit status: %d", exitStatus)
+		}
+
+		return false, fmt.Errorf("unknown cmd error, signal: %s, exit status: %d", sig, exitStatus)
 
 	} else if err != nil {
 		return false, errwrap.Wrapf(err, "general cmd error")
@@ -339,11 +448,18 @@ func (obj *ExecRes) CheckApply(apply bool) (bool, error) {
 	// would be nice, but it would require terminal log output that doesn't
 	// interleave all the parallel parts which would mix it all up...
 	if s := out.String(); s == "" {
-		obj.init.Logf("Command output is empty!")
-
+		obj.init.Logf("command output is empty!")
 	} else {
-		obj.init.Logf("Command output is:")
-		obj.init.Logf(out.String())
+		obj.init.Logf("command output is:")
+		obj.init.Logf(s)
+	}
+
+	if err := obj.init.Send(&ExecSends{
+		Output: obj.output,
+		Stdout: obj.stdout,
+		Stderr: obj.stderr,
+	}); err != nil {
+		return false, err
 	}
 
 	// The state tracking is for exec resources that can't "detect" their
@@ -356,49 +472,67 @@ func (obj *ExecRes) CheckApply(apply bool) (bool, error) {
 
 // Cmp compares two resources and returns an error if they are not equivalent.
 func (obj *ExecRes) Cmp(r engine.Res) error {
-	if !obj.Compare(r) {
-		return fmt.Errorf("did not compare")
-	}
-	return nil
-}
-
-// Compare two resources and return if they are equivalent.
-func (obj *ExecRes) Compare(r engine.Res) bool {
 	// we can only compare ExecRes to others of the same resource kind
 	res, ok := r.(*ExecRes)
 	if !ok {
-		return false
+		return fmt.Errorf("not a %s", obj.Kind())
 	}
 
 	if obj.Cmd != res.Cmd {
-		return false
+		return fmt.Errorf("the Cmd differs")
+	}
+	if len(obj.Args) != len(res.Args) {
+		return fmt.Errorf("the Args differ")
+	}
+	for i, a := range obj.Args {
+		if a != res.Args[i] {
+			return fmt.Errorf("the Args differ at index: %d", i)
+		}
+	}
+	if obj.Cwd != res.Cwd {
+		return fmt.Errorf("the Cwd differs")
 	}
 	if obj.Shell != res.Shell {
-		return false
+		return fmt.Errorf("the Shell differs")
 	}
 	if obj.Timeout != res.Timeout {
-		return false
-	}
-	if obj.WatchCmd != res.WatchCmd {
-		return false
-	}
-	if obj.WatchShell != res.WatchShell {
-		return false
-	}
-	if obj.IfCmd != res.IfCmd {
-		return false
-	}
-	if obj.IfShell != res.IfShell {
-		return false
-	}
-	if obj.User != res.User {
-		return false
-	}
-	if obj.Group != res.Group {
-		return false
+		return fmt.Errorf("the Timeout differs")
 	}
 
-	return true
+	if obj.WatchCmd != res.WatchCmd {
+		return fmt.Errorf("the WatchCmd differs")
+	}
+	if obj.WatchCwd != res.WatchCwd {
+		return fmt.Errorf("the WatchCwd differs")
+	}
+	if obj.WatchShell != res.WatchShell {
+		return fmt.Errorf("the WatchShell differs")
+	}
+
+	if obj.IfCmd != res.IfCmd {
+		return fmt.Errorf("the IfCmd differs")
+	}
+	if obj.IfCwd != res.IfCwd {
+		return fmt.Errorf("the IfCwd differs")
+	}
+	if obj.IfShell != res.IfShell {
+		return fmt.Errorf("the IfShell differs")
+	}
+
+	if obj.User != res.User {
+		return fmt.Errorf("the User differs")
+	}
+	if obj.Group != res.Group {
+		return fmt.Errorf("the Group differs")
+	}
+
+	return nil
+}
+
+// Interrupt is called to ask the execution of this resource to end early.
+func (obj *ExecRes) Interrupt() error {
+	close(obj.interruptChan)
+	return nil
 }
 
 // ExecUID is the UID struct for ExecRes.
@@ -449,13 +583,32 @@ func (obj *ExecRes) AutoEdges() (engine.AutoEdge, error) {
 func (obj *ExecRes) UIDs() []engine.ResUID {
 	x := &ExecUID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
-		Cmd:     obj.Cmd,
+		Cmd:     obj.getCmd(),
 		IfCmd:   obj.IfCmd,
 		// TODO: add more params here
 	}
 	return []engine.ResUID{x}
 }
 
+// ExecSends is the struct of data which is sent after a successful Apply.
+type ExecSends struct {
+	// Output is the combined stdout and stderr of the command.
+	Output *string `lang:"output"`
+	// Stdout is the stdout of the command.
+	Stdout *string `lang:"stdout"`
+	// Stderr is the stderr of the command.
+	Stderr *string `lang:"stderr"`
+}
+
+// Sends represents the default struct of values we can send using Send/Recv.
+func (obj *ExecRes) Sends() interface{} {
+	return &ExecSends{
+		Output: nil,
+		Stdout: nil,
+		Stderr: nil,
+	}
+}
+
 // UnmarshalYAML is the custom unmarshal handler for this struct.
 // It is primarily useful for setting the defaults.
 func (obj *ExecRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
@@ -512,7 +665,7 @@ func (obj *ExecRes) cmdFiles() []string {
 	var paths []string
 	if obj.Shell != "" {
 		paths = append(paths, obj.Shell)
-	} else if cmdSplit := strings.Fields(obj.Cmd); len(cmdSplit) > 0 {
+	} else if cmdSplit := strings.Fields(obj.getCmd()); len(cmdSplit) > 0 {
 		paths = append(paths, cmdSplit[0])
 	}
 	if obj.WatchShell != "" {
@@ -528,28 +681,56 @@ func (obj *ExecRes) cmdFiles() []string {
 	return paths
 }
 
-// bufioOutput is the output struct of the bufioChanScanner channel output.
-type bufioOutput struct {
+// cmdOutput is the output struct of the cmdOutputRunner channel output. You
+// should always check the error first. If it's nil, then you can assume the
+// text data is good to use.
+type cmdOutput struct {
 	text string
 	err  error
 }
 
-// bufioChanScanner wraps the scanner output in a channel.
-func (obj *ExecRes) bufioChanScanner(scanner *bufio.Scanner) chan *bufioOutput {
-	ch := make(chan *bufioOutput)
+// cmdOutputRunner wraps the Cmd in with a StdoutPipe scanner and reads for
+// errors. It runs Start and Wait, and errors runtime things in the channel.
+// If it can't start up the command, it will fail early. Once it's running, it
+// will return the channel which can be used for the duration of the process.
+// Cancelling the context merely unblocks the sending on the output channel, it
+// does not Kill the cmd process. For that you must do it yourself elsewhere.
+func (obj *ExecRes) cmdOutputRunner(ctx context.Context, cmd *exec.Cmd) (chan *cmdOutput, error) {
+	cmdReader, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "error creating StdoutPipe for Cmd")
+	}
+	scanner := bufio.NewScanner(cmdReader)
+	if err := cmd.Start(); err != nil {
+		return nil, errwrap.Wrapf(err, "error starting Cmd")
+	}
+
+	ch := make(chan *cmdOutput)
 	obj.wg.Add(1)
 	go func() {
 		defer obj.wg.Done()
 		defer close(ch)
 		for scanner.Scan() {
-			ch <- &bufioOutput{text: scanner.Text()} // blocks here ?
+			select {
+			case ch <- &cmdOutput{text: scanner.Text()}: // blocks here ?
+			case <-ctx.Done():
+				return
+			}
 		}
+
 		// on EOF, scanner.Err() will be nil
-		if err := scanner.Err(); err != nil {
-			ch <- &bufioOutput{err: err} // send any misc errors we encounter
+		reterr := scanner.Err()
+		reterr = errwrap.Append(reterr, cmd.Wait()) // always run Wait()
+		// send any misc errors we encounter on the channel
+		if reterr != nil {
+			select {
+			case ch <- &cmdOutput{err: reterr}:
+			case <-ctx.Done():
+				return
+			}
 		}
 	}()
-	return ch
+	return ch, nil
 }
 
 // splitWriter mimics what the ssh.CombinedOutput command does, but stores the

engine/resources/exec_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -20,23 +20,34 @@
 package resources
 
 import (
+	"context"
+	"fmt"
+	"os/exec"
+	"syscall"
 	"testing"
+	"time"
 
 	"github.com/purpleidea/mgmt/engine"
 )
 
-func fakeInit(t *testing.T) *engine.Init {
+func fakeExecInit(t *testing.T) (*engine.Init, *ExecSends) {
 	debug := testing.Verbose() // set via the -test.v flag to `go test`
 	logf := func(format string, v ...interface{}) {
 		t.Logf("test: "+format, v...)
 	}
+	execSends := &ExecSends{}
 	return &engine.Init{
-		Running: func() error {
+		Send: func(st interface{}) error {
+			x, ok := st.(*ExecSends)
+			if !ok {
+				return fmt.Errorf("unable to send")
+			}
+			*execSends = *x // set
 			return nil
 		},
 		Debug: debug,
 		Logf:  logf,
-	}
+	}, execSends
 }
 
 func TestExecSendRecv1(t *testing.T) {
@@ -53,7 +64,8 @@ func TestExecSendRecv1(t *testing.T) {
 			t.Errorf("close failed with: %v", err)
 		}
 	}()
-	if err := r1.Init(fakeInit(t)); err != nil {
+	init, execSends := fakeExecInit(t)
+	if err := r1.Init(init); err != nil {
 		t.Errorf("init failed with: %v", err)
 	}
 	// run artificially without the entire engine
@@ -61,23 +73,23 @@ func TestExecSendRecv1(t *testing.T) {
 		t.Errorf("checkapply failed with: %v", err)
 	}
 
-	t.Logf("output is: %v", r1.Output)
-	if r1.Output != nil {
-		t.Logf("output is: %v", *r1.Output)
+	t.Logf("output is: %v", execSends.Output)
+	if execSends.Output != nil {
+		t.Logf("output is: %v", *execSends.Output)
 	}
-	t.Logf("stdout is: %v", r1.Stdout)
-	if r1.Stdout != nil {
-		t.Logf("stdout is: %v", *r1.Stdout)
+	t.Logf("stdout is: %v", execSends.Stdout)
+	if execSends.Stdout != nil {
+		t.Logf("stdout is: %v", *execSends.Stdout)
 	}
-	t.Logf("stderr is: %v", r1.Stderr)
-	if r1.Stderr != nil {
-		t.Logf("stderr is: %v", *r1.Stderr)
+	t.Logf("stderr is: %v", execSends.Stderr)
+	if execSends.Stderr != nil {
+		t.Logf("stderr is: %v", *execSends.Stderr)
 	}
 
-	if r1.Stdout == nil {
+	if execSends.Stdout == nil {
 		t.Errorf("stdout is nil")
 	} else {
-		if out := *r1.Stdout; out != "hello world\n" {
+		if out := *execSends.Stdout; out != "hello world\n" {
 			t.Errorf("got wrong stdout(%d): %s", len(out), out)
 		}
 	}
@@ -97,7 +109,8 @@ func TestExecSendRecv2(t *testing.T) {
 			t.Errorf("close failed with: %v", err)
 		}
 	}()
-	if err := r1.Init(fakeInit(t)); err != nil {
+	init, execSends := fakeExecInit(t)
+	if err := r1.Init(init); err != nil {
 		t.Errorf("init failed with: %v", err)
 	}
 	// run artificially without the entire engine
@@ -105,23 +118,23 @@ func TestExecSendRecv2(t *testing.T) {
 		t.Errorf("checkapply failed with: %v", err)
 	}
 
-	t.Logf("output is: %v", r1.Output)
-	if r1.Output != nil {
-		t.Logf("output is: %v", *r1.Output)
+	t.Logf("output is: %v", execSends.Output)
+	if execSends.Output != nil {
+		t.Logf("output is: %v", *execSends.Output)
 	}
-	t.Logf("stdout is: %v", r1.Stdout)
-	if r1.Stdout != nil {
-		t.Logf("stdout is: %v", *r1.Stdout)
+	t.Logf("stdout is: %v", execSends.Stdout)
+	if execSends.Stdout != nil {
+		t.Logf("stdout is: %v", *execSends.Stdout)
 	}
-	t.Logf("stderr is: %v", r1.Stderr)
-	if r1.Stderr != nil {
-		t.Logf("stderr is: %v", *r1.Stderr)
+	t.Logf("stderr is: %v", execSends.Stderr)
+	if execSends.Stderr != nil {
+		t.Logf("stderr is: %v", *execSends.Stderr)
 	}
 
-	if r1.Stderr == nil {
+	if execSends.Stderr == nil {
 		t.Errorf("stderr is nil")
 	} else {
-		if out := *r1.Stderr; out != "hello world\n" {
+		if out := *execSends.Stderr; out != "hello world\n" {
 			t.Errorf("got wrong stderr(%d): %s", len(out), out)
 		}
 	}
@@ -141,7 +154,8 @@ func TestExecSendRecv3(t *testing.T) {
 			t.Errorf("close failed with: %v", err)
 		}
 	}()
-	if err := r1.Init(fakeInit(t)); err != nil {
+	init, execSends := fakeExecInit(t)
+	if err := r1.Init(init); err != nil {
 		t.Errorf("init failed with: %v", err)
 	}
 	// run artificially without the entire engine
@@ -149,42 +163,97 @@ func TestExecSendRecv3(t *testing.T) {
 		t.Errorf("checkapply failed with: %v", err)
 	}
 
-	t.Logf("output is: %v", r1.Output)
-	if r1.Output != nil {
-		t.Logf("output is: %v", *r1.Output)
+	t.Logf("output is: %v", execSends.Output)
+	if execSends.Output != nil {
+		t.Logf("output is: %v", *execSends.Output)
 	}
-	t.Logf("stdout is: %v", r1.Stdout)
-	if r1.Stdout != nil {
-		t.Logf("stdout is: %v", *r1.Stdout)
+	t.Logf("stdout is: %v", execSends.Stdout)
+	if execSends.Stdout != nil {
+		t.Logf("stdout is: %v", *execSends.Stdout)
 	}
-	t.Logf("stderr is: %v", r1.Stderr)
-	if r1.Stderr != nil {
-		t.Logf("stderr is: %v", *r1.Stderr)
+	t.Logf("stderr is: %v", execSends.Stderr)
+	if execSends.Stderr != nil {
+		t.Logf("stderr is: %v", *execSends.Stderr)
 	}
 
-	if r1.Output == nil {
+	if execSends.Output == nil {
 		t.Errorf("output is nil")
 	} else {
 		// it looks like bash or golang race to the write, so whichever
 		// order they come out in is ok, as long as they come out whole
-		if out := *r1.Output; out != "hello world\ngoodbye world\n" && out != "goodbye world\nhello world\n" {
+		if out := *execSends.Output; out != "hello world\ngoodbye world\n" && out != "goodbye world\nhello world\n" {
 			t.Errorf("got wrong output(%d): %s", len(out), out)
 		}
 	}
 
-	if r1.Stdout == nil {
+	if execSends.Stdout == nil {
 		t.Errorf("stdout is nil")
 	} else {
-		if out := *r1.Stdout; out != "hello world\n" {
+		if out := *execSends.Stdout; out != "hello world\n" {
 			t.Errorf("got wrong stdout(%d): %s", len(out), out)
 		}
 	}
 
-	if r1.Stderr == nil {
+	if execSends.Stderr == nil {
 		t.Errorf("stderr is nil")
 	} else {
-		if out := *r1.Stderr; out != "goodbye world\n" {
+		if out := *execSends.Stderr; out != "goodbye world\n" {
 			t.Errorf("got wrong stderr(%d): %s", len(out), out)
 		}
 	}
 }
+
+func TestExecTimeoutBehaviour(t *testing.T) {
+	// cmd.Process.Kill() is called on timeout
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+	cmdName := "/bin/sleep"    // it's /usr/bin/sleep on modern distros
+	cmdArgs := []string{"300"} // 5 min in seconds
+	cmd := exec.CommandContext(ctx, cmdName, cmdArgs...)
+	// ignore signals sent to parent process (we're in our own group)
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setpgid: true,
+		Pgid:    0,
+	}
+
+	if err := cmd.Start(); err != nil {
+		t.Errorf("error starting cmd: %+v", err)
+		return
+	}
+
+	err := cmd.Wait() // we can unblock this with the timeout
+
+	if err == nil {
+		t.Errorf("expected error, got nil")
+		return
+	}
+
+	exitErr, ok := err.(*exec.ExitError) // embeds an os.ProcessState
+	if err != nil && ok {
+		pStateSys := exitErr.Sys() // (*os.ProcessState) Sys
+		wStatus, ok := pStateSys.(syscall.WaitStatus)
+		if !ok {
+			t.Errorf("error running cmd")
+			return
+		}
+		if !wStatus.Signaled() {
+			t.Errorf("did not get signal, exit status: %d", wStatus.ExitStatus())
+			return
+		}
+
+		// we get this on timeout, because ctx calls cmd.Process.Kill()
+		if sig := wStatus.Signal(); sig != syscall.SIGKILL {
+			t.Errorf("got wrong signal: %+v, exit status: %d", sig, wStatus.ExitStatus())
+			return
+		}
+
+		t.Logf("exit status: %d", wStatus.ExitStatus())
+		return
+
+	} else if err != nil {
+		t.Errorf("general cmd error")
+		return
+	}
+
+	// no error
+}

engine/resources/file.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -36,8 +36,7 @@ import (
 	engineUtil "github.com/purpleidea/mgmt/engine/util"
 	"github.com/purpleidea/mgmt/recwatch"
 	"github.com/purpleidea/mgmt/util"
-
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 func init() {
@@ -54,23 +53,30 @@ type FileRes struct {
 
 	init *engine.Init
 
-	// Path variable, which usually defaults to the name, represents the
+	// Path, which defaults to the name if not specified, represents the
 	// destination path for the file or directory being managed. It must be
 	// an absolute path, and as a result must start with a slash.
-	Path     string  `yaml:"path"`
-	Dirname  string  `yaml:"dirname"`  // override the path dirname
-	Basename string  `yaml:"basename"` // override the path basename
-	Content  *string `yaml:"content"`  // nil to mark as undefined
-	Source   string  `yaml:"source"`   // file path for source content
-	State    string  `yaml:"state"`    // state: exists/present?, absent, (undefined?)
-	Owner    string  `yaml:"owner"`
-	Group    string  `yaml:"group"`
-	Mode     string  `yaml:"mode"`
-	Recurse  bool    `yaml:"recurse"`
-	Force    bool    `yaml:"force"`
+	Path     string `lang:"path" yaml:"path"`
+	Dirname  string `lang:"dirname" yaml:"dirname"`   // override the path dirname
+	Basename string `lang:"basename" yaml:"basename"` // override the path basename
+
+	// Content specifies the file contents to use. If this is nil, they are
+	// left undefined. It cannot be combined with Source.
+	Content *string `lang:"content" yaml:"content"`
+	// Source specifies the source contents for the file resource. It cannot
+	// be combined with the Content parameter.
+	Source string `lang:"source" yaml:"source"`
+	// State specifies the desired state of the file. It can be either
+	// `exists` or `absent`. If you do not specify this, it will be
+	// undefined, and determined based on the other parameters.
+	State string `lang:"state" yaml:"state"`
+
+	Owner   string `lang:"owner" yaml:"owner"`
+	Group   string `lang:"group" yaml:"group"`
+	Mode    string `lang:"mode" yaml:"mode"`
+	Recurse bool   `lang:"recurse" yaml:"recurse"`
+	Force   bool   `lang:"force" yaml:"force"`
 
-	path       string // computed path
-	isDir      bool   // computed isDir
 	sha256sum  string
 	recWatcher *recwatch.RecWatcher
 }
@@ -84,7 +90,7 @@ func (obj *FileRes) Default() engine.Res {
 
 // Validate reports any problems with the struct definition.
 func (obj *FileRes) Validate() error {
-	if obj.GetPath() == "" {
+	if obj.getPath() == "" {
 		return fmt.Errorf("path is empty")
 	}
 
@@ -96,7 +102,7 @@ func (obj *FileRes) Validate() error {
 		return fmt.Errorf("basename must not start with a slash")
 	}
 
-	if !strings.HasPrefix(obj.GetPath(), "/") {
+	if !strings.HasPrefix(obj.getPath(), "/") {
 		return fmt.Errorf("resultant path must be absolute")
 	}
 
@@ -104,7 +110,7 @@ func (obj *FileRes) Validate() error {
 		return fmt.Errorf("can't specify both Content and Source")
 	}
 
-	if obj.isDir && obj.Content != nil { // makes no sense
+	if obj.isDir() && obj.Content != nil { // makes no sense
 		return fmt.Errorf("can't specify Content when creating a Dir")
 	}
 
@@ -114,6 +120,16 @@ func (obj *FileRes) Validate() error {
 		}
 	}
 
+	if obj.Owner != "" || obj.Group != "" {
+		fileInfo, err := os.Stat("/") // pick root just to do this test
+		if err != nil {
+			return fmt.Errorf("can't stat root to get system information")
+		}
+		_, ok := fileInfo.Sys().(*syscall.Stat_t)
+		if !ok {
+			return fmt.Errorf("can't set Owner or Group on this platform")
+		}
+	}
 	if _, err := engineUtil.GetUID(obj.Owner); obj.Owner != "" && err != nil {
 		return err
 	}
@@ -123,7 +139,7 @@ func (obj *FileRes) Validate() error {
 	}
 
 	// XXX: should this specify that we create an empty directory instead?
-	//if obj.Source == "" && obj.isDir {
+	//if obj.Source == "" && obj.isDir() {
 	//	return fmt.Errorf("Can't specify an empty source when creating a Dir.")
 	//}
 
@@ -136,7 +152,7 @@ func (obj *FileRes) Validate() error {
 func (obj *FileRes) mode() (os.FileMode, error) {
 	m, err := strconv.ParseInt(obj.Mode, 8, 32)
 	if err != nil {
-		return os.FileMode(0), errwrap.Wrapf(err, "Mode should be an octal number (%s)", obj.Mode)
+		return os.FileMode(0), errwrap.Wrapf(err, "mode should be an octal number (%s)", obj.Mode)
 	}
 	return os.FileMode(m), nil
 }
@@ -146,8 +162,6 @@ func (obj *FileRes) Init(init *engine.Init) error {
 	obj.init = init // save for later
 
 	obj.sha256sum = ""
-	obj.path = obj.GetPath()                     // compute once
-	obj.isDir = strings.HasSuffix(obj.path, "/") // dirs have trailing slashes
 
 	return nil
 }
@@ -157,9 +171,10 @@ func (obj *FileRes) Close() error {
 	return nil
 }
 
-// GetPath returns the actual path to use for this resource. It computes this
+// getPath returns the actual path to use for this resource. It computes this
 // after analysis of the Path, Dirname and Basename values. Dirs end with slash.
-func (obj *FileRes) GetPath() string {
+// TODO: memoize the result if this seems important.
+func (obj *FileRes) getPath() string {
 	p := obj.Path
 	if obj.Path == "" { // use the name as the path default if missing
 		p = obj.Name()
@@ -180,6 +195,11 @@ func (obj *FileRes) GetPath() string {
 	return obj.Dirname + obj.Basename
 }
 
+// isDir is a helper function to specify whether the path should be a dir.
+func (obj *FileRes) isDir() bool {
+	return strings.HasSuffix(obj.getPath(), "/") // dirs have trailing slashes
+}
+
 // Watch is the primary listener for this resource and it outputs events.
 // This one is a file watcher for files and directories.
 // Modify with caution, it is probably important to write some test cases first!
@@ -188,21 +208,18 @@ func (obj *FileRes) GetPath() string {
 // FIXME: Also watch the source directory when using obj.Source !!!
 func (obj *FileRes) Watch() error {
 	var err error
-	obj.recWatcher, err = recwatch.NewRecWatcher(obj.path, obj.Recurse)
+	obj.recWatcher, err = recwatch.NewRecWatcher(obj.getPath(), obj.Recurse)
 	if err != nil {
 		return err
 	}
 	defer obj.recWatcher.Close()
 
-	// notify engine that we're running
-	if err := obj.init.Running(); err != nil {
-		return err // exit if requested
-	}
+	obj.init.Running() // when started, notify engine that we're running
 
 	var send = false // send event?
 	for {
 		if obj.init.Debug {
-			obj.init.Logf("Watching: %s", obj.path) // attempting to watch...
+			obj.init.Logf("watching: %s", obj.getPath()) // attempting to watch...
 		}
 
 		select {
@@ -214,26 +231,18 @@ func (obj *FileRes) Watch() error {
 				return errwrap.Wrapf(err, "unknown %s watcher error", obj)
 			}
 			if obj.init.Debug { // don't access event.Body if event.Error isn't nil
-				obj.init.Logf("Event(%s): %v", event.Body.Name, event.Body.Op)
+				obj.init.Logf("event(%s): %v", event.Body.Name, event.Body.Op)
 			}
 			send = true
-			obj.init.Dirty() // dirty
 
-		case event, ok := <-obj.init.Events:
-			if !ok {
-				return nil
-			}
-			if err := obj.init.Read(event); err != nil {
-				return err
-			}
+		case <-obj.init.Done: // closed by the engine to signal shutdown
+			return nil
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
 			send = false
-			if err := obj.init.Event(); err != nil {
-				return err // exit if requested
-			}
+			obj.init.Event() // notify engine of an event (this can block)
 		}
 	}
 }
@@ -248,7 +257,7 @@ func (obj *FileRes) fileCheckApply(apply bool, src io.ReadSeeker, dst string, sh
 	// TODO: does it make sense to switch dst to an io.Writer ?
 	// TODO: use obj.Force when dealing with symlinks and other file types!
 	if obj.init.Debug {
-		obj.init.Logf("fileCheckApply: %s -> %s", src, dst)
+		obj.init.Logf("fileCheckApply: %v -> %s", src, dst)
 	}
 
 	srcFile, isFile := src.(*os.File)
@@ -299,7 +308,7 @@ func (obj *FileRes) fileCheckApply(apply bool, src io.ReadSeeker, dst string, sh
 		}
 		// FIXME: respect obj.Recurse here...
 		// there is a dir here, where we want a file...
-		obj.init.Logf("fileCheckApply: Removing (force): %s", cleanDst)
+		obj.init.Logf("fileCheckApply: removing (force): %s", cleanDst)
 		if err := os.RemoveAll(cleanDst); err != nil { // dangerous ;)
 			return "", false, err
 		}
@@ -345,7 +354,7 @@ func (obj *FileRes) fileCheckApply(apply bool, src io.ReadSeeker, dst string, sh
 		return sha256sum, false, nil
 	}
 	if obj.init.Debug {
-		obj.init.Logf("fileCheckApply: Apply: %s -> %s", src, dst)
+		obj.init.Logf("fileCheckApply: apply: %v -> %s", src, dst)
 	}
 
 	dstClose() // unlock file usage so we can write to it
@@ -366,12 +375,12 @@ func (obj *FileRes) fileCheckApply(apply bool, src io.ReadSeeker, dst string, sh
 
 	// TODO: should we offer a way to cancel the copy on ^C ?
 	if obj.init.Debug {
-		obj.init.Logf("fileCheckApply: Copy: %s -> %s", src, dst)
+		obj.init.Logf("fileCheckApply: copy: %v -> %s", src, dst)
 	}
 	if n, err := io.Copy(dstFile, src); err != nil {
 		return sha256sum, false, err
 	} else if obj.init.Debug {
-		obj.init.Logf("fileCheckApply: Copied: %v", n)
+		obj.init.Logf("fileCheckApply: copied: %v", n)
 	}
 	return sha256sum, false, dstFile.Sync()
 }
@@ -379,16 +388,16 @@ func (obj *FileRes) fileCheckApply(apply bool, src io.ReadSeeker, dst string, sh
 // dirCheckApply is the CheckApply operation for an empty directory.
 func (obj *FileRes) dirCheckApply(apply bool) (bool, error) {
 	// check if the path exists and is a directory
-	st, err := os.Stat(obj.path)
+	fileInfo, err := os.Stat(obj.getPath())
 	if err != nil && !os.IsNotExist(err) {
 		return false, errwrap.Wrapf(err, "error checking file resource existence")
 	}
 
-	if err == nil && st.IsDir() {
+	if err == nil && fileInfo.IsDir() {
 		return true, nil // already a directory, nothing to do
 	}
-	if err == nil && !st.IsDir() && !obj.Force {
-		return false, fmt.Errorf("can't force file into dir: %s", obj.path)
+	if err == nil && !fileInfo.IsDir() && !obj.Force {
+		return false, fmt.Errorf("can't force file into dir: %s", obj.getPath())
 	}
 
 	if !apply {
@@ -397,9 +406,9 @@ func (obj *FileRes) dirCheckApply(apply bool) (bool, error) {
 
 	// the path exists and is not a directory
 	// delete the file if force is given
-	if err == nil && !st.IsDir() {
-		obj.init.Logf("dirCheckApply: Removing (force): %s", obj.path)
-		if err := os.Remove(obj.path); err != nil {
+	if err == nil && !fileInfo.IsDir() {
+		obj.init.Logf("dirCheckApply: removing (force): %s", obj.getPath())
+		if err := os.Remove(obj.getPath()); err != nil {
 			return false, err
 		}
 	}
@@ -418,10 +427,10 @@ func (obj *FileRes) dirCheckApply(apply bool) (bool, error) {
 	if obj.Force {
 		// FIXME: respect obj.Recurse here...
 		// TODO: add recurse limit here
-		return false, os.MkdirAll(obj.path, mode)
+		return false, os.MkdirAll(obj.getPath(), mode)
 	}
 
-	return false, os.Mkdir(obj.path, mode)
+	return false, os.Mkdir(obj.getPath(), mode)
 }
 
 // syncCheckApply is the CheckApply operation for a source and destination dir.
@@ -435,7 +444,7 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 		return false, fmt.Errorf("the src and dst must not be empty")
 	}
 
-	var checkOK = true
+	checkOK := true
 	// TODO: handle ./ cases or ../ cases that need cleaning ?
 
 	srcIsDir := strings.HasSuffix(src, "/")
@@ -452,7 +461,7 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 		fin, err := os.Open(src)
 		if err != nil {
 			if obj.init.Debug && os.IsNotExist(err) { // if we get passed an empty src
-				obj.init.Logf("syncCheckApply: Missing src: %s", src)
+				obj.init.Logf("syncCheckApply: missing src: %s", src)
 			}
 			return false, err
 		}
@@ -500,7 +509,7 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 					if absCleanDst == "" || absCleanDst == "/" {
 						return false, fmt.Errorf("don't want to remove root") // safety
 					}
-					obj.init.Logf("syncCheckApply: Removing (force): %s", absCleanDst)
+					obj.init.Logf("syncCheckApply: removing (force): %s", absCleanDst)
 					if err := os.Remove(absCleanDst); err != nil {
 						return false, err
 					}
@@ -519,11 +528,11 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 		}
 
 		if obj.init.Debug {
-			obj.init.Logf("syncCheckApply: Recurse: %s -> %s", absSrc, absDst)
+			obj.init.Logf("syncCheckApply: recurse: %s -> %s", absSrc, absDst)
 		}
 		if obj.Recurse {
 			if c, err := obj.syncCheckApply(apply, absSrc, absDst); err != nil { // recurse
-				return false, errwrap.Wrapf(err, "syncCheckApply: Recurse failed")
+				return false, errwrap.Wrapf(err, "syncCheckApply: recurse failed")
 			} else if !c { // don't let subsequent passes make this true
 				checkOK = false
 			}
@@ -552,7 +561,7 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 		// think the symmetry is more elegant and correct here for now
 		// Avoiding this is also useful if we had a recurse limit arg!
 		if true { // switch
-			obj.init.Logf("syncCheckApply: Removing: %s", absCleanDst)
+			obj.init.Logf("syncCheckApply: removing: %s", absCleanDst)
 			if apply {
 				if err := os.RemoveAll(absCleanDst); err != nil { // dangerous ;)
 					return false, err
@@ -580,12 +589,58 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 	return checkOK, nil
 }
 
+// state performs a CheckApply of the file state to create an empty file.
+func (obj *FileRes) stateCheckApply(apply bool) (bool, error) {
+	if obj.State == "" { // state is not specified
+		return true, nil
+	}
+
+	_, err := os.Stat(obj.getPath())
+
+	if err != nil && !os.IsNotExist(err) {
+		return false, errwrap.Wrapf(err, "could not stat file")
+	}
+
+	if obj.State == "absent" && os.IsNotExist(err) {
+		return true, nil
+	}
+
+	if obj.State == "exists" && err == nil {
+		return true, nil
+	}
+
+	// state is not okay, no work done, exit, but without error
+	if !apply {
+		return false, nil
+	}
+
+	if obj.State == "absent" {
+		return false, nil // defer the work to contentCheckApply
+	}
+
+	if obj.Content == nil && !obj.isDir() {
+		// Create an empty file to ensure one exists. Don't O_TRUNC it,
+		// in case one is magically created right after our exists test.
+		// The chmod used is what is used by the os.Create function.
+		// TODO: is using O_EXCL okay?
+		f, err := os.OpenFile(obj.getPath(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666)
+		if err != nil {
+			return false, errwrap.Wrapf(err, "problem creating empty file")
+		}
+		if err := f.Close(); err != nil {
+			return false, errwrap.Wrapf(err, "problem closing empty file")
+		}
+	}
+
+	return false, nil // defer the Content != nil and isDir work to later...
+}
+
 // contentCheckApply performs a CheckApply for the file existence and content.
-func (obj *FileRes) contentCheckApply(apply bool) (checkOK bool, _ error) {
+func (obj *FileRes) contentCheckApply(apply bool) (bool, error) {
 	obj.init.Logf("contentCheckApply(%t)", apply)
 
 	if obj.State == "absent" {
-		if _, err := os.Stat(obj.path); os.IsNotExist(err) {
+		if _, err := os.Stat(obj.getPath()); os.IsNotExist(err) {
 			// no such file or directory, but
 			// file should be missing, phew :)
 			return true, nil
@@ -600,17 +655,17 @@ func (obj *FileRes) contentCheckApply(apply bool) (checkOK bool, _ error) {
 		}
 
 		// apply portion
-		if obj.path == "" || obj.path == "/" {
+		if obj.getPath() == "" || obj.getPath() == "/" {
 			return false, fmt.Errorf("don't want to remove root") // safety
 		}
-		obj.init.Logf("contentCheckApply: Removing: %s", obj.path)
+		obj.init.Logf("contentCheckApply: removing: %s", obj.getPath())
 		// FIXME: respect obj.Recurse here...
 		// TODO: add recurse limit here
-		err := os.RemoveAll(obj.path) // dangerous ;)
-		return false, err             // either nil or not
+		err := os.RemoveAll(obj.getPath()) // dangerous ;)
+		return false, err                  // either nil or not
 	}
 
-	if obj.isDir && obj.Source == "" {
+	if obj.isDir() && obj.Source == "" {
 		return obj.dirCheckApply(apply)
 	}
 
@@ -621,7 +676,7 @@ func (obj *FileRes) contentCheckApply(apply bool) (checkOK bool, _ error) {
 
 	if obj.Source == "" { // do the obj.Content checks first...
 		bufferSrc := bytes.NewReader([]byte(*obj.Content))
-		sha256sum, checkOK, err := obj.fileCheckApply(apply, bufferSrc, obj.path, obj.sha256sum)
+		sha256sum, checkOK, err := obj.fileCheckApply(apply, bufferSrc, obj.getPath(), obj.sha256sum)
 		if sha256sum != "" { // empty values mean errored or didn't hash
 			// this can be valid even when the whole function errors
 			obj.sha256sum = sha256sum // cache value
@@ -633,7 +688,7 @@ func (obj *FileRes) contentCheckApply(apply bool) (checkOK bool, _ error) {
 		return checkOK, nil // success
 	}
 
-	checkOK, err := obj.syncCheckApply(apply, obj.Source, obj.path)
+	checkOK, err := obj.syncCheckApply(apply, obj.Source, obj.getPath())
 	if err != nil {
 		obj.init.Logf("syncCheckApply: Error: %v", err)
 		return false, err
@@ -643,16 +698,16 @@ func (obj *FileRes) contentCheckApply(apply bool) (checkOK bool, _ error) {
 }
 
 // chmodCheckApply performs a CheckApply for the file permissions.
-func (obj *FileRes) chmodCheckApply(apply bool) (checkOK bool, _ error) {
+func (obj *FileRes) chmodCheckApply(apply bool) (bool, error) {
 	obj.init.Logf("chmodCheckApply(%t)", apply)
 
 	if obj.State == "absent" {
-		// File is absent
+		// file is absent
 		return true, nil
 	}
 
 	if obj.Mode == "" {
-		// No mode specified, everything is ok
+		// no mode specified, everything is ok
 		return true, nil
 	}
 
@@ -668,36 +723,36 @@ func (obj *FileRes) chmodCheckApply(apply bool) (checkOK bool, _ error) {
 		return false, err
 	}
 
-	st, err := os.Stat(obj.path)
+	fileInfo, err := os.Stat(obj.getPath())
 	if err != nil {
 		return false, err
 	}
 
-	// Nothing to do
-	if st.Mode() == mode {
+	// nothing to do
+	if fileInfo.Mode() == mode {
 		return true, nil
 	}
 
-	// Not clean but don't apply
+	// not clean but don't apply
 	if !apply {
 		return false, nil
 	}
 
-	err = os.Chmod(obj.path, mode)
+	err = os.Chmod(obj.getPath(), mode)
 	return false, err
 }
 
 // chownCheckApply performs a CheckApply for the file ownership.
-func (obj *FileRes) chownCheckApply(apply bool) (checkOK bool, _ error) {
+func (obj *FileRes) chownCheckApply(apply bool) (bool, error) {
 	var expectedUID, expectedGID int
 	obj.init.Logf("chownCheckApply(%t)", apply)
 
 	if obj.State == "absent" {
-		// File is absent or no owner specified
+		// file is absent or no owner specified
 		return true, nil
 	}
 
-	st, err := os.Stat(obj.path)
+	fileInfo, err := os.Stat(obj.getPath())
 
 	// If the file does not exist and we are in
 	// noop mode, do not throw an error.
@@ -709,10 +764,10 @@ func (obj *FileRes) chownCheckApply(apply bool) (checkOK bool, _ error) {
 		return false, err
 	}
 
-	stUnix, ok := st.Sys().(*syscall.Stat_t)
-	if !ok {
-		// Not unix
-		panic("No support for your platform")
+	stUnix, ok := fileInfo.Sys().(*syscall.Stat_t)
+	if !ok { // this check is done in Validate, but it's done here again...
+		// not unix
+		return false, fmt.Errorf("can't set Owner or Group on this platform")
 	}
 
 	if obj.Owner != "" {
@@ -721,7 +776,7 @@ func (obj *FileRes) chownCheckApply(apply bool) (checkOK bool, _ error) {
 			return false, err
 		}
 	} else {
-		// Nothing specified, no changes to be made, expect same as actual
+		// nothing specified, no changes to be made, expect same as actual
 		expectedUID = int(stUnix.Uid)
 	}
 
@@ -731,27 +786,26 @@ func (obj *FileRes) chownCheckApply(apply bool) (checkOK bool, _ error) {
 			return false, err
 		}
 	} else {
-		// Nothing specified, no changes to be made, expect same as actual
+		// nothing specified, no changes to be made, expect same as actual
 		expectedGID = int(stUnix.Gid)
 	}
 
-	// Nothing to do
+	// nothing to do
 	if int(stUnix.Uid) == expectedUID && int(stUnix.Gid) == expectedGID {
 		return true, nil
 	}
 
-	// Not clean, but don't apply
+	// not clean, but don't apply
 	if !apply {
 		return false, nil
 	}
 
-	err = os.Chown(obj.path, expectedUID, expectedGID)
-	return false, err
+	return false, os.Chown(obj.getPath(), expectedUID, expectedGID)
 }
 
 // CheckApply checks the resource state and applies the resource if the bool
 // input is true. It returns error info and if the state check passed or not.
-func (obj *FileRes) CheckApply(apply bool) (checkOK bool, _ error) {
+func (obj *FileRes) CheckApply(apply bool) (bool, error) {
 	// NOTE: all send/recv change notifications *must* be processed before
 	// there is a possibility of failure in CheckApply. This is because if
 	// we fail (and possibly run again) the subsequent send->recv transfer
@@ -760,12 +814,18 @@ func (obj *FileRes) CheckApply(apply bool) (checkOK bool, _ error) {
 	// promptly, if they must not be lost, such as for cache invalidation.
 	if val, exists := obj.init.Recv()["Content"]; exists && val.Changed {
 		// if we received on Content, and it changed, invalidate the cache!
-		obj.init.Logf("contentCheckApply: Invalidating sha256sum of `Content`")
+		obj.init.Logf("contentCheckApply: invalidating sha256sum of `Content`")
 		obj.sha256sum = "" // invalidate!!
 	}
 
-	checkOK = true
+	checkOK := true
 
+	// always run stateCheckApply before contentCheckApply, they go together
+	if c, err := obj.stateCheckApply(apply); err != nil {
+		return false, err
+	} else if !c {
+		checkOK = false
+	}
 	if c, err := obj.contentCheckApply(apply); err != nil {
 		return false, err
 	} else if !c {
@@ -789,45 +849,52 @@ func (obj *FileRes) CheckApply(apply bool) (checkOK bool, _ error) {
 
 // Cmp compares two resources and returns an error if they are not equivalent.
 func (obj *FileRes) Cmp(r engine.Res) error {
-	if !obj.Compare(r) {
-		return fmt.Errorf("did not compare")
-	}
-	return nil
-}
-
-// Compare two resources and return if they are equivalent.
-func (obj *FileRes) Compare(r engine.Res) bool {
 	// we can only compare FileRes to others of the same resource kind
 	res, ok := r.(*FileRes)
 	if !ok {
-		return false
+		return fmt.Errorf("not a %s", obj.Kind())
 	}
 
-	if obj.path != res.path {
-		return false
+	// We don't need to compare Path, Dirname or Basename-- we only care if
+	// the resultant path is different or not.
+	if obj.getPath() != res.getPath() {
+		return fmt.Errorf("the Path differs")
 	}
 	if (obj.Content == nil) != (res.Content == nil) { // xor
-		return false
+		return fmt.Errorf("the Content differs")
 	}
 	if obj.Content != nil && res.Content != nil {
 		if *obj.Content != *res.Content { // compare the strings
-			return false
+			return fmt.Errorf("the contents of Content differ")
 		}
 	}
 	if obj.Source != res.Source {
-		return false
+		return fmt.Errorf("the Source differs")
 	}
 	if obj.State != res.State {
-		return false
-	}
-	if obj.Recurse != res.Recurse {
-		return false
-	}
-	if obj.Force != res.Force {
-		return false
+		return fmt.Errorf("the State differs")
 	}
 
-	return true
+	if obj.Owner != res.Owner {
+		return fmt.Errorf("the Owner differs")
+	}
+	if obj.Group != res.Group {
+		return fmt.Errorf("the Group differs")
+	}
+	// TODO: when we start to allow alternate representations for the mode,
+	// ensure that we compare in the same format. Eg: `ug=rw` == `0660`.
+	if obj.Mode != res.Mode {
+		return fmt.Errorf("the Mode differs")
+	}
+
+	if obj.Recurse != res.Recurse {
+		return fmt.Errorf("the Recurse option differs")
+	}
+	if obj.Force != res.Force {
+		return fmt.Errorf("the Force option differs")
+	}
+
+	return nil
 }
 
 // FileUID is the UID struct for FileRes.
@@ -888,8 +955,8 @@ func (obj *FileResAutoEdges) Test(input []bool) bool {
 // the bottom up!
 func (obj *FileRes) AutoEdges() (engine.AutoEdge, error) {
 	var data []engine.ResUID // store linear result chain here...
-	// build it, but don't use obj.path because this gets called before Init
-	values := util.PathSplitFullReversed(obj.GetPath())
+	// don't use any memoization run in Init (this gets called before Init)
+	values := util.PathSplitFullReversed(obj.getPath())
 	_, values = values[0], values[1:] // get rid of first value which is me!
 	for _, x := range values {
 		var reversed = true // cheat by passing a pointer
@@ -914,7 +981,7 @@ func (obj *FileRes) AutoEdges() (engine.AutoEdge, error) {
 func (obj *FileRes) UIDs() []engine.ResUID {
 	x := &FileUID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
-		path:    obj.GetPath(), // not obj.path b/c we didn't init yet!
+		path:    obj.getPath(),
 	}
 	return []engine.ResUID{x}
 }

engine/resources/file_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -119,6 +119,12 @@ func TestMiscEncodeDecode2(t *testing.T) {
 		t.Errorf("Can't create: %v", err)
 		return
 	}
+	// NOTE: Do not add this bit of code, because it would cause the path to
+	// get taken from the actual Path parameter, instead of using the name,
+	// and if we use the name, the Cmp function will detect if the name is
+	// stored properly or not.
+	//fileRes := input.(*FileRes) // must not panic
+	//fileRes.Path = "/tmp/whatever"
 
 	b64, err := engineUtil.ResToB64(input)
 	if err != nil {
@@ -142,11 +148,53 @@ func TestMiscEncodeDecode2(t *testing.T) {
 		t.Errorf("Output %v is not a Res", res2)
 		return
 	}
+	// this uses the standalone file cmp function
 	if err := res1.Cmp(res2); err != nil {
 		t.Errorf("The input and output Res values do not match: %+v", err)
 	}
 }
 
+func TestMiscEncodeDecode3(t *testing.T) {
+	var err error
+
+	// encode
+	input, err := engine.NewNamedResource("file", "file1")
+	if err != nil {
+		t.Errorf("Can't create: %v", err)
+		return
+	}
+	fileRes := input.(*FileRes) // must not panic
+	fileRes.Path = "/tmp/whatever"
+	// TODO: add other params/traits/etc here!
+
+	b64, err := engineUtil.ResToB64(input)
+	if err != nil {
+		t.Errorf("Can't encode: %v", err)
+		return
+	}
+
+	output, err := engineUtil.B64ToRes(b64)
+	if err != nil {
+		t.Errorf("Can't decode: %v", err)
+		return
+	}
+
+	res1, ok := input.(engine.Res)
+	if !ok {
+		t.Errorf("Input %v is not a Res", res1)
+		return
+	}
+	res2, ok := output.(engine.Res)
+	if !ok {
+		t.Errorf("Output %v is not a Res", res2)
+		return
+	}
+	// this uses the more complete, engine cmp function
+	if err := engine.ResCmp(res1, res2); err != nil {
+		t.Errorf("The input and output Res values do not match: %+v", err)
+	}
+}
+
 func TestFileAbsolute1(t *testing.T) {
 	// file resource paths should be absolute
 	f1 := &FileRes{

engine/resources/group.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -28,8 +28,7 @@ import (
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/engine/traits"
 	"github.com/purpleidea/mgmt/recwatch"
-
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 func init() {
@@ -85,10 +84,7 @@ func (obj *GroupRes) Watch() error {
 	}
 	defer obj.recWatcher.Close()
 
-	// notify engine that we're running
-	if err := obj.init.Running(); err != nil {
-		return err // exit if requested
-	}
+	obj.init.Running() // when started, notify engine that we're running
 
 	var send = false // send event?
 	for {
@@ -108,29 +104,21 @@ func (obj *GroupRes) Watch() error {
 				obj.init.Logf("Event(%s): %v", event.Body.Name, event.Body.Op)
 			}
 			send = true
-			obj.init.Dirty() // dirty
 
-		case event, ok := <-obj.init.Events:
-			if !ok {
-				return nil
-			}
-			if err := obj.init.Read(event); err != nil {
-				return err
-			}
+		case <-obj.init.Done: // closed by the engine to signal shutdown
+			return nil
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
 			send = false
-			if err := obj.init.Event(); err != nil {
-				return err // exit if requested
-			}
+			obj.init.Event() // notify engine of an event (this can block)
 		}
 	}
 }
 
 // CheckApply method for Group resource.
-func (obj *GroupRes) CheckApply(apply bool) (checkOK bool, err error) {
+func (obj *GroupRes) CheckApply(apply bool) (bool, error) {
 	obj.init.Logf("CheckApply(%t)", apply)
 
 	// check if the group exists

engine/resources/hostname.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -25,9 +25,9 @@ import (
 	"github.com/purpleidea/mgmt/engine/traits"
 	engineUtil "github.com/purpleidea/mgmt/engine/util"
 	"github.com/purpleidea/mgmt/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
 
 	"github.com/godbus/dbus"
-	errwrap "github.com/pkg/errors"
 )
 
 func init() {
@@ -110,7 +110,7 @@ func (obj *HostnameRes) Watch() error {
 	// if we share the bus with others, we will get each others messages!!
 	bus, err := util.SystemBusPrivateUsable() // don't share the bus connection!
 	if err != nil {
-		return errwrap.Wrap(err, "Failed to connect to bus")
+		return errwrap.Wrapf(err, "failed to connect to bus")
 	}
 	defer bus.Close()
 	// watch the PropertiesChanged signal on the hostname1 dbus path
@@ -120,56 +120,45 @@ func (obj *HostnameRes) Watch() error {
 		dbusPropertiesIface,
 	)
 	if call := bus.BusObject().Call(engineUtil.DBusAddMatch, 0, args); call.Err != nil {
-		return errwrap.Wrap(call.Err, "Failed to subscribe to DBus events for hostname1")
+		return errwrap.Wrapf(call.Err, "failed to subscribe to DBus events for hostname1")
 	}
 	defer bus.BusObject().Call(engineUtil.DBusRemoveMatch, 0, args) // ignore the error
 
 	signals := make(chan *dbus.Signal, 10) // closed by dbus package
 	bus.Signal(signals)
 
-	// notify engine that we're running
-	if err := obj.init.Running(); err != nil {
-		return err // exit if requested
-	}
+	obj.init.Running() // when started, notify engine that we're running
 
 	var send = false // send event?
 	for {
 		select {
 		case <-signals:
 			send = true
-			obj.init.Dirty() // dirty
 
-		case event, ok := <-obj.init.Events:
-			if !ok {
-				return nil
-			}
-			if err := obj.init.Read(event); err != nil {
-				return err
-			}
+		case <-obj.init.Done: // closed by the engine to signal shutdown
+			return nil
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
 			send = false
-			if err := obj.init.Event(); err != nil {
-				return err // exit if requested
-			}
+			obj.init.Event() // notify engine of an event (this can block)
 		}
 	}
 }
 
-func (obj *HostnameRes) updateHostnameProperty(object dbus.BusObject, expectedValue, property, setterName string, apply bool) (checkOK bool, err error) {
+func (obj *HostnameRes) updateHostnameProperty(object dbus.BusObject, expectedValue, property, setterName string, apply bool) (bool, error) {
 	propertyObject, err := object.GetProperty("org.freedesktop.hostname1." + property)
 	if err != nil {
 		return false, errwrap.Wrapf(err, "failed to get org.freedesktop.hostname1.%s", property)
 	}
 	if propertyObject.Value() == nil {
-		return false, errwrap.Errorf("Unexpected nil value received when reading property %s", property)
+		return false, fmt.Errorf("unexpected nil value received when reading property %s", property)
 	}
 
 	propertyValue, ok := propertyObject.Value().(string)
 	if !ok {
-		return false, fmt.Errorf("Received unexpected type as %s value, expected string got '%T'", property, propertyValue)
+		return false, fmt.Errorf("received unexpected type as %s value, expected string got '%T'", property, propertyValue)
 	}
 
 	// expected value and actual value match => checkOk
@@ -193,16 +182,16 @@ func (obj *HostnameRes) updateHostnameProperty(object dbus.BusObject, expectedVa
 }
 
 // CheckApply method for Hostname resource.
-func (obj *HostnameRes) CheckApply(apply bool) (checkOK bool, err error) {
+func (obj *HostnameRes) CheckApply(apply bool) (bool, error) {
 	conn, err := util.SystemBusPrivateUsable()
 	if err != nil {
-		return false, errwrap.Wrap(err, "Failed to connect to the private system bus")
+		return false, errwrap.Wrapf(err, "failed to connect to the private system bus")
 	}
 	defer conn.Close()
 
 	hostnameObject := conn.Object(hostname1Iface, hostname1Path)
 
-	checkOK = true
+	checkOK := true
 	if obj.PrettyHostname != "" {
 		propertyCheckOK, err := obj.updateHostnameProperty(hostnameObject, obj.PrettyHostname, "PrettyHostname", "SetPrettyHostname", apply)
 		if err != nil {

engine/resources/kv.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -18,13 +18,16 @@
 package resources
 
 import (
+	"context"
 	"fmt"
 	"strconv"
+	"sync"
+	"time"
 
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/engine/traits"
-
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 func init() {
@@ -40,6 +43,10 @@ const (
 	SkipCmpStyleString
 )
 
+const (
+	kvCheckApplyTimeout = 5 * time.Second
+)
+
 // KVRes is a resource which writes a key/value pair into cluster wide storage.
 // It will ensure that the key is set to the requested value. The one exception
 // is that if you use the SkipLessThan parameter, then it will only replace the
@@ -56,14 +63,32 @@ type KVRes struct {
 
 	init *engine.Init
 
-	// XXX: shouldn't the name be the key?
-	Key          string            `yaml:"key"`          // key to set
-	Value        *string           `yaml:"value"`        // value to set (nil to delete)
-	SkipLessThan bool              `yaml:"skiplessthan"` // skip updates as long as stored value is greater
-	SkipCmpStyle KVResSkipCmpStyle `yaml:"skipcmpstyle"` // how to do the less than cmp
+	// Key represents the key to set. If it is not specified, the Name value
+	// is used instead.
+	Key string `lang:"key" yaml:"key"`
+	// Value represents the string value to set. If this value is nil or,
+	// undefined, then this will delete that key.
+	Value *string `lang:"value" yaml:"value"`
+	// SkipLessThan causes the value to be updated as long as it is greater.
+	SkipLessThan bool `lang:"skiplessthan" yaml:"skiplessthan"`
+	// SkipCmpStyle is the type of compare function used when determining if
+	// the value is greater when using the SkipLessThan parameter.
+	SkipCmpStyle KVResSkipCmpStyle `lang:"skipcmpstyle" yaml:"skipcmpstyle"`
+
+	interruptChan chan struct{}
+
 	// TODO: does it make sense to have different backends here? (eg: local)
 }
 
+// getKey returns the key to be used for this resource. If the Key field is
+// specified, it will use that, otherwise it uses the Name.
+func (obj *KVRes) getKey() string {
+	if obj.Key != "" {
+		return obj.Key
+	}
+	return obj.Name()
+}
+
 // Default returns some sensible defaults for this resource.
 func (obj *KVRes) Default() engine.Res {
 	return &KVRes{}
@@ -71,7 +96,7 @@ func (obj *KVRes) Default() engine.Res {
 
 // Validate if the params passed in are valid data.
 func (obj *KVRes) Validate() error {
-	if obj.Key == "" {
+	if obj.getKey() == "" {
 		return fmt.Errorf("key must not be empty")
 	}
 	if obj.SkipLessThan {
@@ -92,6 +117,8 @@ func (obj *KVRes) Validate() error {
 func (obj *KVRes) Init(init *engine.Init) error {
 	obj.init = init // save for later
 
+	obj.interruptChan = make(chan struct{})
+
 	return nil
 }
 
@@ -102,13 +129,17 @@ func (obj *KVRes) Close() error {
 
 // Watch is the primary listener for this resource and it outputs events.
 func (obj *KVRes) Watch() error {
+	// FIXME: add timeout to context
+	// The obj.init.Done channel is closed by the engine to signal shutdown.
+	ctx, cancel := util.ContextWithCloser(context.Background(), obj.init.Done)
+	defer cancel()
 
-	// notify engine that we're running
-	if err := obj.init.Running(); err != nil {
-		return err // exit if requested
+	ch, err := obj.init.World.StrMapWatch(ctx, obj.getKey()) // get possible events!
+	if err != nil {
+		return err
 	}
 
-	ch := obj.init.World.StrMapWatch(obj.Key) // get possible events!
+	obj.init.Running() // when started, notify engine that we're running
 
 	var send = false // send event?
 	for {
@@ -122,32 +153,24 @@ func (obj *KVRes) Watch() error {
 				return errwrap.Wrapf(err, "unknown %s watcher error", obj)
 			}
 			if obj.init.Debug {
-				obj.init.Logf("Event!")
+				obj.init.Logf("event!")
 			}
 			send = true
-			obj.init.Dirty() // dirty
 
-		case event, ok := <-obj.init.Events:
-			if !ok {
-				return nil
-			}
-			if err := obj.init.Read(event); err != nil {
-				return err
-			}
+		case <-obj.init.Done: // closed by the engine to signal shutdown
+			return nil
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
 			send = false
-			if err := obj.init.Event(); err != nil {
-				return err // exit if requested
-			}
+			obj.init.Event() // notify engine of an event (this can block)
 		}
 	}
 }
 
 // lessThanCheck checks for less than validity.
-func (obj *KVRes) lessThanCheck(value string) (checkOK bool, err error) {
+func (obj *KVRes) lessThanCheck(value string) (bool, error) {
 	v := *obj.Value
 	if value == v { // redundant check for safety
 		return true, nil
@@ -185,16 +208,31 @@ func (obj *KVRes) lessThanCheck(value string) (checkOK bool, err error) {
 }
 
 // CheckApply method for Password resource. Does nothing, returns happy!
-func (obj *KVRes) CheckApply(apply bool) (checkOK bool, err error) {
+func (obj *KVRes) CheckApply(apply bool) (bool, error) {
 	obj.init.Logf("CheckApply(%t)", apply)
 
+	wg := &sync.WaitGroup{}
+	defer wg.Wait() // this must be above the defer cancel() call
+	ctx, cancel := context.WithTimeout(context.Background(), kvCheckApplyTimeout)
+	defer cancel()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		select {
+		case <-obj.interruptChan:
+			cancel()
+		case <-ctx.Done():
+			// let this exit
+		}
+	}()
+
 	if val, exists := obj.init.Recv()["Value"]; exists && val.Changed {
 		// if we received on Value, and it changed, wooo, nothing to do.
 		obj.init.Logf("CheckApply: `Value` was updated!")
 	}
 
 	hostname := obj.init.Hostname // me
-	keyMap, err := obj.init.World.StrMapGet(obj.Key)
+	keyMap, err := obj.init.World.StrMapGet(ctx, obj.getKey())
 	if err != nil {
 		return false, errwrap.Wrapf(err, "check error during StrGet")
 	}
@@ -214,7 +252,7 @@ func (obj *KVRes) CheckApply(apply bool) (checkOK bool, err error) {
 		return true, nil // nothing to delete, we're good!
 
 	} else if ok && obj.Value == nil { // delete
-		err := obj.init.World.StrMapDel(obj.Key)
+		err := obj.init.World.StrMapDel(ctx, obj.getKey())
 		return false, errwrap.Wrapf(err, "apply error during StrDel")
 	}
 
@@ -222,7 +260,7 @@ func (obj *KVRes) CheckApply(apply bool) (checkOK bool, err error) {
 		return false, nil
 	}
 
-	if err := obj.init.World.StrMapSet(obj.Key, *obj.Value); err != nil {
+	if err := obj.init.World.StrMapSet(ctx, obj.getKey(), *obj.Value); err != nil {
 		return false, errwrap.Wrapf(err, "apply error during StrSet")
 	}
 
@@ -231,39 +269,37 @@ func (obj *KVRes) CheckApply(apply bool) (checkOK bool, err error) {
 
 // Cmp compares two resources and returns an error if they are not equivalent.
 func (obj *KVRes) Cmp(r engine.Res) error {
-	if !obj.Compare(r) {
-		return fmt.Errorf("did not compare")
-	}
-	return nil
-}
-
-// Compare two resources and return if they are equivalent.
-func (obj *KVRes) Compare(r engine.Res) bool {
 	// we can only compare KVRes to others of the same resource kind
 	res, ok := r.(*KVRes)
 	if !ok {
-		return false
+		return fmt.Errorf("not a %s", obj.Kind())
 	}
 
-	if obj.Key != res.Key {
-		return false
+	if obj.getKey() != res.getKey() {
+		return fmt.Errorf("the Key differs")
 	}
 	if (obj.Value == nil) != (res.Value == nil) { // xor
-		return false
+		return fmt.Errorf("the Value differs")
 	}
 	if obj.Value != nil && res.Value != nil {
 		if *obj.Value != *res.Value { // compare the strings
-			return false
+			return fmt.Errorf("the contents of Value differs")
 		}
 	}
 	if obj.SkipLessThan != res.SkipLessThan {
-		return false
+		return fmt.Errorf("the SkipLessThan param differs")
 	}
 	if obj.SkipCmpStyle != res.SkipCmpStyle {
-		return false
+		return fmt.Errorf("the SkipCmpStyle param differs")
 	}
 
-	return true
+	return nil
+}
+
+// Interrupt is called to ask the execution of this resource to end early.
+func (obj *KVRes) Interrupt() error {
+	close(obj.interruptChan)
+	return nil
 }
 
 // KVUID is the UID struct for KVRes.

engine/resources/mount.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -33,13 +33,13 @@ import (
 	engineUtil "github.com/purpleidea/mgmt/engine/util"
 	"github.com/purpleidea/mgmt/recwatch"
 	"github.com/purpleidea/mgmt/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
 
 	sdbus "github.com/coreos/go-systemd/dbus"
 	"github.com/coreos/go-systemd/unit"
 	systemdUtil "github.com/coreos/go-systemd/util"
 	fstab "github.com/deniswernert/go-fstab"
 	"github.com/godbus/dbus"
-	errwrap "github.com/pkg/errors"
 	"golang.org/x/sys/unix"
 )
 
@@ -224,10 +224,7 @@ func (obj *MountRes) Watch() error {
 	// close the recwatcher when we're done
 	defer recWatcher.Close()
 
-	// notify engine that we're running
-	if err := obj.init.Running(); err != nil {
-		return err // bubble up a NACK...
-	}
+	obj.init.Running() // when started, notify engine that we're running
 
 	var send bool
 	var done bool
@@ -248,7 +245,6 @@ func (obj *MountRes) Watch() error {
 				obj.init.Logf("event(%s): %v", event.Body.Name, event.Body.Op)
 			}
 
-			obj.init.Dirty()
 			send = true
 
 		case event, ok := <-ch:
@@ -263,31 +259,23 @@ func (obj *MountRes) Watch() error {
 				obj.init.Logf("event: %+v", event)
 			}
 
-			obj.init.Dirty()
 			send = true
 
-		case event, ok := <-obj.init.Events:
-			if !ok {
-				return nil
-			}
-			if err := obj.init.Read(event); err != nil {
-				return err
-			}
+		case <-obj.init.Done: // closed by the engine to signal shutdown
+			return nil
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
 			send = false
-			if err := obj.init.Event(); err != nil {
-				return err // exit if requested
-			}
+			obj.init.Event() // notify engine of an event (this can block)
 		}
 	}
 }
 
 // fstabCheckApply checks /etc/fstab for entries corresponding to the resource
 // definition, and adds or deletes the entry as needed.
-func (obj *MountRes) fstabCheckApply(apply bool) (checkOK bool, err error) {
+func (obj *MountRes) fstabCheckApply(apply bool) (bool, error) {
 	exists, err := fstabEntryExists(fstabPath, obj.mount)
 	if err != nil {
 		return false, errwrap.Wrapf(err, "error checking if fstab entry exists")
@@ -351,8 +339,8 @@ func (obj *MountRes) mountCheckApply(apply bool) (bool, error) {
 // CheckApply is run to check the state and, if apply is true, to apply the
 // necessary changes to reach the desired state. This is run before Watch and
 // again if Watch finds a change occurring to the state.
-func (obj *MountRes) CheckApply(apply bool) (checkOK bool, err error) {
-	checkOK = true
+func (obj *MountRes) CheckApply(apply bool) (bool, error) {
+	checkOK := true
 
 	if c, err := obj.fstabCheckApply(apply); err != nil {
 		return false, err
@@ -596,7 +584,7 @@ func mountReload() error {
 	}
 
 	// systemctl restart remote-fs.target
-	if err := restartUnit(conn, "local-fs.target"); err != nil {
+	if err := restartUnit(conn, "remote-fs.target"); err != nil {
 		return errwrap.Wrapf(err, "error restarting unit")
 	}
 

engine/resources/mount_linux_test.go

@@ -0,0 +1,76 @@
+// Mgmt
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// +build !root !darwin
+
+package resources
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	fstab "github.com/deniswernert/go-fstab"
+)
+
+func TestMountExists(t *testing.T) {
+	const procMock1 = `/tmp/mount0 /mnt/proctest ext4 rw,seclabel,relatime,data=ordered 0 0` + "\n"
+
+	var mountExistsTests = []struct {
+		procMock []byte
+		in       *fstab.Mount
+		out      bool
+	}{
+		{
+			[]byte(procMock1),
+			&fstab.Mount{
+				Spec:    "/tmp/mount0",
+				File:    "/mnt/proctest",
+				VfsType: "ext4",
+				MntOps:  map[string]string{"defaults": ""},
+				Freq:    1,
+				PassNo:  1,
+			},
+			true,
+		},
+	}
+
+	file, err := ioutil.TempFile("", "proc")
+	if err != nil {
+		t.Errorf("error creating temp file: %v", err)
+		return
+	}
+	defer os.Remove(file.Name())
+	for _, test := range mountExistsTests {
+		if err := ioutil.WriteFile(file.Name(), test.procMock, 0664); err != nil {
+			t.Errorf("error writing proc file: %s: %v", file.Name(), err)
+			return
+		}
+		if err := ioutil.WriteFile(test.in.Spec, []byte{}, 0664); err != nil {
+			t.Errorf("error writing fstab file: %s: %v", file.Name(), err)
+			return
+		}
+		result, err := mountExists(file.Name(), test.in)
+		if err != nil {
+			t.Errorf("error checking if fstab entry %s exists: %v", test.in.String(), err)
+			return
+		}
+		if result != test.out {
+			t.Errorf("mountExistsTests test wanted: %t, got: %t", test.out, result)
+		}
+	}
+}

engine/resources/mount_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -29,8 +29,6 @@ import (
 
 const fstabMock1 = `UUID=ef5726f2-615c-4350-b0ab-f106e5fc90ad / ext4 defaults 1 1` + "\n"
 
-const procMock1 = `/tmp/mount0 /mnt/proctest ext4 rw,seclabel,relatime,data=ordered 0 0` + "\n"
-
 var fstabWriteTests = []struct {
 	in fstab.Mounts
 }{
@@ -295,49 +293,3 @@ func TestMountCompare(t *testing.T) {
 		}
 	}
 }
-
-var mountExistsTests = []struct {
-	procMock []byte
-	in       *fstab.Mount
-	out      bool
-}{
-	{
-		[]byte(procMock1),
-		&fstab.Mount{
-			Spec:    "/tmp/mount0",
-			File:    "/mnt/proctest",
-			VfsType: "ext4",
-			MntOps:  map[string]string{"defaults": ""},
-			Freq:    1,
-			PassNo:  1,
-		},
-		true,
-	},
-}
-
-func TestMountExists(t *testing.T) {
-	file, err := ioutil.TempFile("", "proc")
-	if err != nil {
-		t.Errorf("error creating temp file: %v", err)
-		return
-	}
-	defer os.Remove(file.Name())
-	for _, test := range mountExistsTests {
-		if err := ioutil.WriteFile(file.Name(), test.procMock, 0664); err != nil {
-			t.Errorf("error writing proc file: %s: %v", file.Name(), err)
-			return
-		}
-		if err := ioutil.WriteFile(test.in.Spec, []byte{}, 0664); err != nil {
-			t.Errorf("error writing fstab file: %s: %v", file.Name(), err)
-			return
-		}
-		result, err := mountExists(file.Name(), test.in)
-		if err != nil {
-			t.Errorf("error checking if fstab entry %s exists: %v", test.in.String(), err)
-			return
-		}
-		if result != test.out {
-			t.Errorf("mountExistsTests test wanted: %t, got: %t", test.out, result)
-		}
-	}
-}

engine/resources/msg.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -94,30 +94,20 @@ func (obj *MsgRes) Close() error {
 
 // Watch is the primary listener for this resource and it outputs events.
 func (obj *MsgRes) Watch() error {
-	// notify engine that we're running
-	if err := obj.init.Running(); err != nil {
-		return err // exit if requested
-	}
+	obj.init.Running() // when started, notify engine that we're running
 
-	var send = false // send event?
+	//var send = false // send event?
 	for {
 		select {
-		case event, ok := <-obj.init.Events:
-			if !ok {
-				return nil
-			}
-			if err := obj.init.Read(event); err != nil {
-				return err
-			}
+		case <-obj.init.Done: // closed by the engine to signal shutdown
+			return nil
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
-		if send {
-			send = false
-			if err := obj.init.Event(); err != nil {
-				return err // exit if requested
-			}
-		}
+		//if send {
+		//	send = false
+		//	obj.init.Event() // notify engine of an event (this can block)
+		//}
 	}
 }
 
@@ -137,7 +127,7 @@ func (obj *MsgRes) isAllStateOK() bool {
 func (obj *MsgRes) updateStateOK() {
 	// XXX: this resource doesn't entirely make sense to me at the moment.
 	if !obj.isAllStateOK() {
-		obj.init.Dirty()
+		//obj.init.Dirty() // XXX: removed with API cleanup
 	}
 }
 

engine/resources/msg_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/resources/net.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -34,9 +34,9 @@ import (
 	"github.com/purpleidea/mgmt/engine/traits"
 	"github.com/purpleidea/mgmt/recwatch"
 	"github.com/purpleidea/mgmt/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
+	"github.com/purpleidea/mgmt/util/socketset"
 
-	multierr "github.com/hashicorp/go-multierror"
-	errwrap "github.com/pkg/errors"
 	// XXX: Do NOT use subscribe methods from this lib, as they are racey and
 	// do not clean up spawned goroutines. Should be replaced when a suitable
 	// alternative is available.
@@ -178,9 +178,7 @@ func (obj *NetRes) Close() error {
 		return fmt.Errorf("socket file should not be the root path")
 	}
 	if obj.socketFile != "" { // safety
-		if err := os.Remove(obj.socketFile); err != nil {
-			errList = multierr.Append(errList, err)
-		}
+		errList = errwrap.Append(errList, os.Remove(obj.socketFile))
 	}
 
 	return errList
@@ -190,16 +188,20 @@ func (obj *NetRes) Close() error {
 // TODO: currently gets events from ALL interfaces, would be nice to reject
 // events from other interfaces.
 func (obj *NetRes) Watch() error {
-	// waitgroup for netlink receive goroutine
-	wg := &sync.WaitGroup{}
-	defer wg.Wait()
-
 	// create a netlink socket for receiving network interface events
-	conn, err := newSocketSet(rtmGrps, obj.socketFile)
+	conn, err := socketset.NewSocketSet(rtmGrps, obj.socketFile, unix.NETLINK_ROUTE)
 	if err != nil {
 		return errwrap.Wrapf(err, "error creating socket set")
 	}
-	defer conn.shutdown() // close the netlink socket and unblock conn.receive()
+
+	// waitgroup for netlink receive goroutine
+	wg := &sync.WaitGroup{}
+	defer conn.Close()
+	// We must wait for the Shutdown() AND the select inside of SocketSet to
+	// complete before we Close, since the unblocking in SocketSet is not a
+	// synchronous operation.
+	defer wg.Wait()
+	defer conn.Shutdown() // close the netlink socket and unblock conn.receive()
 
 	// watch the systemd-networkd configuration file
 	recWatcher, err := recwatch.NewRecWatcher(obj.unitFilePath, false)
@@ -219,11 +221,10 @@ func (obj *NetRes) Watch() error {
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		defer conn.close() // close the pipe when we're done with it
 		defer close(nlChan)
 		for {
 			// receive messages from the socket set
-			msgs, err := conn.receive()
+			msgs, err := conn.ReceiveNetlinkMessages()
 			if err != nil {
 				select {
 				case nlChan <- &nlChanStruct{
@@ -243,10 +244,7 @@ func (obj *NetRes) Watch() error {
 		}
 	}()
 
-	// notify engine that we're running
-	if err := obj.init.Running(); err != nil {
-		return err // exit if requested
-	}
+	obj.init.Running() // when started, notify engine that we're running
 
 	var send = false // send event?
 	var done bool
@@ -268,7 +266,6 @@ func (obj *NetRes) Watch() error {
 			}
 
 			send = true
-			obj.init.Dirty() // dirty
 
 		case event, ok := <-recWatcher.Events():
 			if !ok {
@@ -286,23 +283,15 @@ func (obj *NetRes) Watch() error {
 			}
 
 			send = true
-			obj.init.Dirty() // dirty
 
-		case event, ok := <-obj.init.Events:
-			if !ok {
-				return nil
-			}
-			if err := obj.init.Read(event); err != nil {
-				return err
-			}
+		case <-obj.init.Done: // closed by the engine to signal shutdown
+			return nil
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
 			send = false
-			if err := obj.init.Event(); err != nil {
-				return err // exit if requested
-			}
+			obj.init.Event() // notify engine of an event (this can block)
 		}
 	}
 }
@@ -471,8 +460,8 @@ func (obj *NetRes) fileCheckApply(apply bool) (bool, error) {
 // CheckApply is run to check the state and, if apply is true, to apply the
 // necessary changes to reach the desired state. This is run before Watch and
 // again if Watch finds a change occurring to the state.
-func (obj *NetRes) CheckApply(apply bool) (checkOK bool, err error) {
-	checkOK = true
+func (obj *NetRes) CheckApply(apply bool) (bool, error) {
+	checkOK := true
 
 	// check the network device
 	if c, err := obj.ifaceCheckApply(apply); err != nil {
@@ -768,122 +757,3 @@ func (obj *iface) addrApplyAdd(objAddrs []string) error {
 	}
 	return nil
 }
-
-// socketSet is used to receive events from a socket and shut it down cleanly
-// when asked. It contains a socket for events and a pipe socket to unblock
-// receive on shutdown.
-type socketSet struct {
-	fdEvents int
-	fdPipe   int
-	pipeFile string
-}
-
-// newSocketSet returns a socketSet, initialized with the given parameters.
-func newSocketSet(groups uint32, file string) (*socketSet, error) {
-	// make a netlink socket file descriptor
-	fdEvents, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_RAW, unix.NETLINK_ROUTE)
-	if err != nil {
-		return nil, errwrap.Wrapf(err, "error creating netlink socket")
-	}
-	// bind to the socket and add add the netlink groups we need to get events
-	if err := unix.Bind(fdEvents, &unix.SockaddrNetlink{
-		Family: unix.AF_NETLINK,
-		Groups: groups,
-	}); err != nil {
-		return nil, errwrap.Wrapf(err, "error binding netlink socket")
-	}
-
-	// create a pipe socket to unblock unix.Select when we close
-	fdPipe, err := unix.Socket(unix.AF_UNIX, unix.SOCK_RAW, unix.PROT_NONE)
-	if err != nil {
-		return nil, errwrap.Wrapf(err, "error creating pipe socket")
-	}
-	// bind the pipe to a file
-	if err = unix.Bind(fdPipe, &unix.SockaddrUnix{
-		Name: file,
-	}); err != nil {
-		return nil, errwrap.Wrapf(err, "error binding pipe socket")
-	}
-	return &socketSet{
-		fdEvents: fdEvents,
-		fdPipe:   fdPipe,
-		pipeFile: file,
-	}, nil
-}
-
-// shutdown closes the event file descriptor and unblocks receive by sending
-// a message to the pipe file descriptor. It must be called before close, and
-// should only be called once.
-func (obj *socketSet) shutdown() error {
-	// close the event socket so no more events are produced
-	if err := unix.Close(obj.fdEvents); err != nil {
-		return err
-	}
-	// send a message to the pipe to unblock select
-	return unix.Sendto(obj.fdPipe, nil, 0, &unix.SockaddrUnix{
-		Name: path.Join(obj.pipeFile),
-	})
-}
-
-// close closes the pipe file descriptor. It must only be called after
-// shutdown has closed fdEvents, and unblocked receive. It should only be
-// called once.
-func (obj *socketSet) close() error {
-	return unix.Close(obj.fdPipe)
-}
-
-// receive waits for bytes from fdEvents and parses them into a slice of
-// netlink messages. It will block until an event is produced, or shutdown
-// is called.
-func (obj *socketSet) receive() ([]syscall.NetlinkMessage, error) {
-	// Select will return when any fd in fdSet (fdEvents and fdPipe) is ready
-	// to read.
-	_, err := unix.Select(obj.nfd(), obj.fdSet(), nil, nil, nil)
-	if err != nil {
-		// if a system interrupt is caught
-		if err == unix.EINTR { // signal interrupt
-			return nil, nil
-		}
-		return nil, errwrap.Wrapf(err, "error selecting on fd")
-	}
-	// receive the message from the netlink socket into b
-	b := make([]byte, os.Getpagesize())
-	n, _, err := unix.Recvfrom(obj.fdEvents, b, unix.MSG_DONTWAIT) // non-blocking receive
-	if err != nil {
-		// if fdEvents is closed
-		if err == unix.EBADF { // bad file descriptor
-			return nil, nil
-		}
-		return nil, errwrap.Wrapf(err, "error receiving messages")
-	}
-	// if we didn't get enough bytes for a header, something went wrong
-	if n < unix.NLMSG_HDRLEN {
-		return nil, fmt.Errorf("received short header")
-	}
-	b = b[:n] // truncate b to message length
-	// use syscall to parse, as func does not exist in x/sys/unix
-	return syscall.ParseNetlinkMessage(b)
-}
-
-// nfd returns one more than the highest fd value in the struct, for use as as
-// the nfds parameter in select. It represents the file descriptor set maximum
-// size. See man select for more info.
-func (obj *socketSet) nfd() int {
-	if obj.fdEvents > obj.fdPipe {
-		return obj.fdEvents + 1
-	}
-	return obj.fdPipe + 1
-}
-
-// fdSet returns a bitmask representation of the integer values of fdEvents
-// and fdPipe. See man select for more info.
-func (obj *socketSet) fdSet() *unix.FdSet {
-	fdSet := &unix.FdSet{}
-	// Generate the bitmask representing the file descriptors in the socketSet.
-	// The rightmost bit corresponds to file descriptor zero, and each bit to
-	// the left represents the next file descriptor number in the sequence of
-	// all real numbers. E.g. the FdSet containing containing 0 and 4 is 10001.
-	fdSet.Bits[obj.fdEvents/64] |= 1 << uint(obj.fdEvents)
-	fdSet.Bits[obj.fdPipe/64] |= 1 << uint(obj.fdPipe)
-	return fdSet
-}

engine/resources/net_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -15,14 +15,14 @@
 // You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+// +build !darwin
+
 package resources
 
 import (
 	"bytes"
 	"strings"
 	"testing"
-
-	"golang.org/x/sys/unix"
 )
 
 // test cases for NetRes.unitFileContents()
@@ -82,85 +82,3 @@ func TestUnitFileContents(t *testing.T) {
 		}
 	}
 }
-
-// test cases for socketSet.fdSet()
-var fdSetTests = []struct {
-	in  *socketSet
-	out *unix.FdSet
-}{
-	{
-		&socketSet{
-			fdEvents: 3,
-			fdPipe:   4,
-		},
-		&unix.FdSet{
-			Bits: [16]int64{0x18}, // 11000
-		},
-	},
-	{
-		&socketSet{
-			fdEvents: 12,
-			fdPipe:   8,
-		},
-		&unix.FdSet{
-			Bits: [16]int64{0x1100}, // 1000100000000
-		},
-	},
-	{
-		&socketSet{
-			fdEvents: 9,
-			fdPipe:   21,
-		},
-		&unix.FdSet{
-			Bits: [16]int64{0x200200}, // 1000000000001000000000
-		},
-	},
-}
-
-// test socketSet.fdSet()
-func TestFdSet(t *testing.T) {
-	for _, test := range fdSetTests {
-		result := test.in.fdSet()
-		if *result != *test.out {
-			t.Errorf("fdSet test wanted: %b, got: %b", *test.out, *result)
-		}
-	}
-}
-
-// test cases for socketSet.nfd()
-var nfdTests = []struct {
-	in  *socketSet
-	out int
-}{
-	{
-		&socketSet{
-			fdEvents: 3,
-			fdPipe:   4,
-		},
-		5,
-	},
-	{
-		&socketSet{
-			fdEvents: 8,
-			fdPipe:   4,
-		},
-		9,
-	},
-	{
-		&socketSet{
-			fdEvents: 90,
-			fdPipe:   900,
-		},
-		901,
-	},
-}
-
-// test socketSet.nfd()
-func TestNfd(t *testing.T) {
-	for _, test := range nfdTests {
-		result := test.in.nfd()
-		if result != test.out {
-			t.Errorf("nfd test wanted: %d, got: %d", test.out, result)
-		}
-	}
-}

engine/resources/noop.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -63,35 +63,19 @@ func (obj *NoopRes) Close() error {
 
 // Watch is the primary listener for this resource and it outputs events.
 func (obj *NoopRes) Watch() error {
-	// notify engine that we're running
-	if err := obj.init.Running(); err != nil {
-		return err // exit if requested
+	obj.init.Running() // when started, notify engine that we're running
+
+	select {
+	case <-obj.init.Done: // closed by the engine to signal shutdown
 	}
 
-	var send = false // send event?
-	for {
-		select {
-		case event, ok := <-obj.init.Events:
-			if !ok {
-				return nil
-			}
-			if err := obj.init.Read(event); err != nil {
-				return err
-			}
-		}
+	//obj.init.Event() // notify engine of an event (this can block)
 
-		// do all our event sending all together to avoid duplicate msgs
-		if send {
-			send = false
-			if err := obj.init.Event(); err != nil {
-				return err // exit if requested
-			}
-		}
-	}
+	return nil
 }
 
 // CheckApply method for Noop resource. Does nothing, returns happy!
-func (obj *NoopRes) CheckApply(apply bool) (checkOK bool, err error) {
+func (obj *NoopRes) CheckApply(apply bool) (bool, error) {
 	if obj.init.Refresh() {
 		obj.init.Logf("received a notification!")
 	}

engine/resources/noop_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/resources/nspawn.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -27,12 +27,12 @@ import (
 	"github.com/purpleidea/mgmt/engine/traits"
 	engineUtil "github.com/purpleidea/mgmt/engine/util"
 	"github.com/purpleidea/mgmt/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
 
 	systemdDbus "github.com/coreos/go-systemd/dbus"
 	machined "github.com/coreos/go-systemd/machine1"
 	systemdUtil "github.com/coreos/go-systemd/util"
 	"github.com/godbus/dbus"
-	errwrap "github.com/pkg/errors"
 )
 
 const (
@@ -167,10 +167,7 @@ func (obj *NspawnRes) Watch() error {
 	bus.Signal(busChan)
 	defer bus.RemoveSignal(busChan) // not needed here, but nice for symmetry
 
-	// notify engine that we're running
-	if err := obj.init.Running(); err != nil {
-		return err // exit if requested
-	}
+	obj.init.Running() // when started, notify engine that we're running
 
 	var send = false // send event?
 	for {
@@ -187,24 +184,16 @@ func (obj *NspawnRes) Watch() error {
 					return fmt.Errorf("unknown event: %s", event.Name)
 				}
 				send = true
-				obj.init.Dirty() // dirty
 			}
 
-		case event, ok := <-obj.init.Events:
-			if !ok {
-				return nil
-			}
-			if err := obj.init.Read(event); err != nil {
-				return err
-			}
+		case <-obj.init.Done: // closed by the engine to signal shutdown
+			return nil
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
 			send = false
-			if err := obj.init.Event(); err != nil {
-				return err // exit if requested
-			}
+			obj.init.Event() // notify engine of an event (this can block)
 		}
 	}
 }
@@ -212,7 +201,7 @@ func (obj *NspawnRes) Watch() error {
 // CheckApply is run to check the state and, if apply is true, to apply the
 // necessary changes to reach the desired state. This is run before Watch and
 // again if Watch finds a change occurring to the state.
-func (obj *NspawnRes) CheckApply(apply bool) (checkOK bool, err error) {
+func (obj *NspawnRes) CheckApply(apply bool) (bool, error) {
 	// this resource depends on systemd to ensure that it's running
 	if !systemdUtil.IsRunningSystemd() {
 		return false, errors.New("systemd is not running")

engine/resources/packagekit/packagekit.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -27,10 +27,9 @@ import (
 
 	engineUtil "github.com/purpleidea/mgmt/engine/util"
 	"github.com/purpleidea/mgmt/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
 
 	"github.com/godbus/dbus"
-	multierr "github.com/hashicorp/go-multierror"
-	errwrap "github.com/pkg/errors"
 )
 
 // global tweaks of verbosity and code path
@@ -198,9 +197,8 @@ func (obj *Conn) matchSignal(ch chan *dbus.Signal, path dbus.ObjectPath, iface s
 	removeSignals := func() error {
 		var errList error
 		for i := len(argsList) - 1; i >= 0; i-- { // last in first out
-			if call := bus.Call(engineUtil.DBusRemoveMatch, 0, argsList[i]); call.Err != nil {
-				errList = multierr.Append(errList, call.Err)
-			}
+			call := bus.Call(engineUtil.DBusRemoveMatch, 0, argsList[i])
+			errList = errwrap.Append(errList, call.Err)
 		}
 		return errList
 	}

engine/resources/password.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -29,8 +29,7 @@ import (
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/engine/traits"
 	"github.com/purpleidea/mgmt/recwatch"
-
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 func init() {
@@ -182,10 +181,7 @@ func (obj *PasswordRes) Watch() error {
 	}
 	defer obj.recWatcher.Close()
 
-	// notify engine that we're running
-	if err := obj.init.Running(); err != nil {
-		return err // exit if requested
-	}
+	obj.init.Running() // when started, notify engine that we're running
 
 	var send = false // send event?
 	for {
@@ -199,29 +195,21 @@ func (obj *PasswordRes) Watch() error {
 				return errwrap.Wrapf(err, "unknown %s watcher error", obj)
 			}
 			send = true
-			obj.init.Dirty() // dirty
 
-		case event, ok := <-obj.init.Events:
-			if !ok {
-				return nil
-			}
-			if err := obj.init.Read(event); err != nil {
-				return err
-			}
+		case <-obj.init.Done: // closed by the engine to signal shutdown
+			return nil
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
 			send = false
-			if err := obj.init.Event(); err != nil {
-				return err // exit if requested
-			}
+			obj.init.Event() // notify engine of an event (this can block)
 		}
 	}
 }
 
 // CheckApply method for Password resource. Does nothing, returns happy!
-func (obj *PasswordRes) CheckApply(apply bool) (checkOK bool, err error) {
+func (obj *PasswordRes) CheckApply(apply bool) (bool, error) {
 	var refresh = obj.init.Refresh() // do we have a pending reload to apply?
 	var exists = true                // does the file (aka the token) exist?
 	var generate bool                // do we need to generate a new password?
@@ -355,7 +343,7 @@ func (obj *PasswordRes) UIDs() []engine.ResUID {
 // PasswordSends is the struct of data which is sent after a successful Apply.
 type PasswordSends struct {
 	// Password is the generated password being sent.
-	Password *string
+	Password *string `lang:"password"`
 	// Hashing is the algorithm used for this password. Empty is plain text.
 	Hashing string // TODO: implement me
 }

engine/resources/pkg.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -26,8 +26,7 @@ import (
 	"github.com/purpleidea/mgmt/engine/resources/packagekit"
 	"github.com/purpleidea/mgmt/engine/traits"
 	"github.com/purpleidea/mgmt/util"
-
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 func init() {
@@ -67,7 +66,7 @@ type PkgRes struct {
 // Default returns some sensible defaults for this resource.
 func (obj *PkgRes) Default() engine.Res {
 	return &PkgRes{
-		State: PkgStateInstalled, // i think this is preferable to "latest"
+		State: PkgStateInstalled, // this *is* preferable to "newest"
 	}
 }
 
@@ -76,6 +75,9 @@ func (obj *PkgRes) Validate() error {
 	if obj.State == "" {
 		return fmt.Errorf("state cannot be empty")
 	}
+	if obj.State == "latest" {
+		return fmt.Errorf("state is invalid, did you mean `newest` ?")
+	}
 
 	return nil
 }
@@ -118,10 +120,7 @@ func (obj *PkgRes) Watch() error {
 		return errwrap.Wrapf(err, "error adding signal match")
 	}
 
-	// notify engine that we're running
-	if err := obj.init.Running(); err != nil {
-		return err // exit if requested
-	}
+	obj.init.Running() // when started, notify engine that we're running
 
 	var send = false // send event?
 	for {
@@ -143,20 +142,15 @@ func (obj *PkgRes) Watch() error {
 			}
 
 			send = true
-			obj.init.Dirty() // dirty
 
-		case event := <-obj.init.Events:
-			if err := obj.init.Read(event); err != nil {
-				return err
-			}
+		case <-obj.init.Done: // closed by the engine to signal shutdown
+			return nil
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
 			send = false
-			if err := obj.init.Event(); err != nil {
-				return err // exit if requested
-			}
+			obj.init.Event() // notify engine of an event (this can block)
 		}
 	}
 }
@@ -203,7 +197,7 @@ func (obj *PkgRes) pkgMappingHelper(bus *packagekit.Conn) (map[string]*packageki
 	packageMap[obj.Name()] = obj.State     // key is pkg name, value is pkg state
 	var filter uint64                      // initializes at the "zero" value of 0
 	filter += packagekit.PkFilterEnumArch  // always search in our arch (optional!)
-	// we're requesting latest version, or to narrow down install choices!
+	// we're requesting newest version, or to narrow down install choices!
 	if obj.State == PkgStateNewest || obj.State == PkgStateInstalled {
 		// if we add this, we'll still see older packages if installed
 		// this is an optimization, and is *optional*, this logic is
@@ -218,7 +212,7 @@ func (obj *PkgRes) pkgMappingHelper(bus *packagekit.Conn) (map[string]*packageki
 	}
 	result, err := bus.PackagesToPackageIDs(packageMap, filter)
 	if err != nil {
-		return nil, errwrap.Wrapf(err, "Can't run PackagesToPackageIDs")
+		return nil, errwrap.Wrapf(err, "can't run PackagesToPackageIDs")
 	}
 	return result, nil
 }
@@ -249,6 +243,10 @@ func (obj *PkgRes) populateFileList() error {
 	if !ok || !data.Found {
 		return fmt.Errorf("can't find package named '%s'", obj.Name())
 	}
+	if data.PackageID == "" {
+		// this can happen if you specify a bad version like "latest"
+		return fmt.Errorf("empty PackageID found for '%s'", obj.Name())
+	}
 
 	packageIDs := []string{data.PackageID} // just one for now
 	filesMap, err := bus.GetFilesByPackageID(packageIDs)
@@ -264,7 +262,7 @@ func (obj *PkgRes) populateFileList() error {
 
 // CheckApply checks the resource state and applies the resource if the bool
 // input is true. It returns error info and if the state check passed or not.
-func (obj *PkgRes) CheckApply(apply bool) (checkOK bool, err error) {
+func (obj *PkgRes) CheckApply(apply bool) (bool, error) {
 	obj.init.Logf("Check: %s", obj.fmtNames(obj.getNames()))
 
 	bus := packagekit.NewBus()

engine/resources/pkg_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/resources/print.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -40,6 +40,10 @@ type PrintRes struct {
 	init *engine.Init
 
 	Msg string `lang:"msg" yaml:"msg"` // the message to display
+	// RefreshOnly is an option that causes the message to be printed only
+	// when notified by another resource. When set to true, this resource
+	// cannot be autogrouped.
+	RefreshOnly bool `lang:"refresh_only" yaml:"refresh_only"`
 }
 
 // Default returns some sensible defaults for this resource.
@@ -66,52 +70,44 @@ func (obj *PrintRes) Close() error {
 
 // Watch is the primary listener for this resource and it outputs events.
 func (obj *PrintRes) Watch() error {
-	// notify engine that we're running
-	if err := obj.init.Running(); err != nil {
-		return err // exit if requested
+	obj.init.Running() // when started, notify engine that we're running
+
+	select {
+	case <-obj.init.Done: // closed by the engine to signal shutdown
 	}
 
-	var send = false // send event?
-	for {
-		select {
-		case event, ok := <-obj.init.Events:
-			if !ok {
-				return nil
-			}
-			if err := obj.init.Read(event); err != nil {
-				return err
-			}
-		}
+	//obj.init.Event() // notify engine of an event (this can block)
 
-		// do all our event sending all together to avoid duplicate msgs
-		if send {
-			send = false
-			if err := obj.init.Event(); err != nil {
-				return err // exit if requested
-			}
-		}
-	}
+	return nil
 }
 
 // CheckApply method for Print resource. Does nothing, returns happy!
-func (obj *PrintRes) CheckApply(apply bool) (checkOK bool, err error) {
+func (obj *PrintRes) CheckApply(apply bool) (bool, error) {
 	obj.init.Logf("CheckApply: %t", apply)
 	if val, exists := obj.init.Recv()["Msg"]; exists && val.Changed {
 		// if we received on Msg, and it changed, log message
 		obj.init.Logf("CheckApply: Received `Msg` of: %s", obj.Msg)
 	}
 
-	if obj.init.Refresh() {
+	var refresh = obj.init.Refresh()
+	// we output if not RefreshOnly, or we are in refresh mode and RefreshOnly
+	var display = refresh || !obj.RefreshOnly
+
+	if refresh {
 		obj.init.Logf("Received a notification!")
 	}
-	obj.init.Logf("Msg: %s", obj.Msg)
+	if display {
+		obj.init.Logf("Msg: %s", obj.Msg)
+	}
 	if g := obj.GetGroup(); len(g) > 0 { // add any grouped elements
 		for _, x := range g {
 			print, ok := x.(*PrintRes) // convert from Res
 			if !ok {
 				panic(fmt.Sprintf("grouped member %v is not a %s", x, obj.Kind()))
 			}
-			obj.init.Logf("%s: Msg: %s", print, print.Msg)
+			if display {
+				obj.init.Logf("%s: Msg: %s", print, print.Msg)
+			}
 		}
 	}
 	return true, nil // state is always okay
@@ -157,10 +153,14 @@ func (obj *PrintRes) UIDs() []engine.ResUID {
 
 // GroupCmp returns whether two resources can be grouped together or not.
 func (obj *PrintRes) GroupCmp(r engine.GroupableRes) error {
-	_, ok := r.(*PrintRes)
+	res, ok := r.(*PrintRes)
 	if !ok {
 		return fmt.Errorf("resource is not the same kind")
 	}
+	// we don't group if it's RefreshOnly: only the notifier may trigger
+	if obj.RefreshOnly || res.RefreshOnly {
+		return fmt.Errorf("resource uses RefreshOnly, it cannot be merged")
+	}
 	return nil // grouped together if we were asked to
 }
 

engine/resources/resources_test.go

@@ -0,0 +1,550 @@
+// Mgmt
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// +build !root
+
+package resources
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/purpleidea/mgmt/engine"
+	"github.com/purpleidea/mgmt/util"
+)
+
+// TODO: consider providing this as a lib so that we can add tests into the
+// specific _test.go file of each resource.
+
+// makeRes is a helper function to build a res. It should only be called in
+// tests, because it panics if something goes wrong.
+func makeRes(kind, name string) engine.Res {
+	res, err := engine.NewNamedResource(kind, name)
+	if err != nil {
+		panic(fmt.Sprintf("could not create resource: %+v", err))
+	}
+	return res
+}
+
+// Step is used for the timeline in tests.
+type Step interface {
+	Action() error
+	Expect() error
+}
+
+type manualStep struct {
+	action func() error
+	expect func() error
+}
+
+func (obj *manualStep) Action() error {
+	return obj.action()
+}
+func (obj *manualStep) Expect() error {
+	return obj.expect()
+}
+
+// NewManualStep creates a new manual step with an action and an expect test.
+func NewManualStep(action, expect func() error) Step {
+	return &manualStep{
+		action: action,
+		expect: expect,
+	}
+}
+
+type startupStep struct {
+	ms uint
+	ch chan struct{} // set by test harness
+}
+
+func (obj *startupStep) Action() error {
+	select {
+	case <-obj.ch: // called by Running() in Watch
+	case <-time.After(time.Duration(obj.ms) * time.Millisecond):
+		return fmt.Errorf("took too long to startup")
+	}
+	return nil
+}
+func (obj *startupStep) Expect() error { return nil }
+
+// NewStartupStep waits up to this many ms for the Watch function to startup.
+func NewStartupStep(ms uint) Step {
+	return &startupStep{
+		ms: ms,
+	}
+}
+
+type changedStep struct {
+	ms     uint
+	expect bool      // what checkOK value we're expecting
+	ch     chan bool // set by test harness, filled with checkOK values
+}
+
+func (obj *changedStep) Action() error {
+	select {
+	case checkOK, ok := <-obj.ch: // from CheckApply() in test Process loop
+		if !ok {
+			return fmt.Errorf("channel closed unexpectedly")
+		}
+		if checkOK != obj.expect {
+			return fmt.Errorf("got unexpected checkOK value of: %t", checkOK)
+		}
+	case <-time.After(time.Duration(obj.ms) * time.Millisecond):
+		return fmt.Errorf("took too long to startup")
+	}
+	return nil
+}
+func (obj *changedStep) Expect() error { return nil }
+
+// NewChangedStep waits up to this many ms for a CheckApply action to occur. Watch function to startup.
+func NewChangedStep(ms uint, expect bool) Step {
+	return &changedStep{
+		ms:     ms,
+		expect: expect,
+	}
+}
+
+type clearChangedStep struct {
+	ms uint
+	ch chan bool // set by test harness, filled with checkOK values
+}
+
+func (obj *clearChangedStep) Action() error {
+	// read all pending events...
+	for {
+		select {
+		case _, ok := <-obj.ch: // from CheckApply() in test Process loop
+			if !ok {
+				return fmt.Errorf("channel closed unexpectedly")
+			}
+		case <-time.After(time.Duration(obj.ms) * time.Millisecond):
+			return nil // done waiting
+		}
+	}
+}
+func (obj *clearChangedStep) Expect() error { return nil }
+
+// NewClearChangedStep waits up to this many ms for additional CheckApply
+// actions to occur, and flushes them all so that a future NewChangedStep won't
+// see unwanted events.
+func NewClearChangedStep(ms uint) Step {
+	return &clearChangedStep{
+		ms: ms,
+	}
+}
+
+func TestResources1(t *testing.T) {
+	type test struct { // an individual test
+		name      string
+		res       engine.Res // a resource
+		fail      bool
+		experr    error        // expected error if fail == true (nil ignores it)
+		experrstr string       // expected error prefix
+		timeline  []Step       // TODO: this could be a generator that keeps pushing out steps until it's done!
+		expect    func() error // function to check for expected state
+		startup   func() error // function to run as startup
+		cleanup   func() error // function to run as cleanup
+	}
+
+	// helpers
+	// TODO: make a series of helps to orchestrate the resources (eg: edit
+	// file, wait for event w/ timeout, run command w/ timeout, etc...)
+	sleep := func(ms uint) Step {
+		return &manualStep{
+			action: func() error {
+				time.Sleep(time.Duration(ms) * time.Millisecond)
+				return nil
+			},
+			expect: func() error { return nil },
+		}
+	}
+	fileExpect := func(p, s string) Step { // path & string
+		return &manualStep{
+			action: func() error { return nil },
+			expect: func() error {
+				content, err := ioutil.ReadFile(p)
+				if err != nil {
+					return err
+				}
+				if string(content) != s {
+					return fmt.Errorf("contents did not match in %s", p)
+				}
+				return nil
+			},
+		}
+	}
+	fileWrite := func(p, s string) Step { // path & string
+		return &manualStep{
+			action: func() error {
+				// TODO: apparently using 0666 is equivalent to respecting the current umask
+				const umask = 0666
+				return ioutil.WriteFile(p, []byte(s), umask)
+			},
+			expect: func() error { return nil },
+		}
+	}
+
+	testCases := []test{}
+	{
+		r := makeRes("file", "r1")
+		res := r.(*FileRes) // if this panics, the test will panic
+		p := "/tmp/whatever"
+		s := "hello, world\n"
+		res.Path = p
+		contents := s
+		res.Content = &contents
+
+		timeline := []Step{
+			NewStartupStep(1000 * 60),          // startup
+			NewChangedStep(1000*60, false),     // did we do something?
+			fileExpect(p, s),                   // check initial state
+			NewClearChangedStep(1000 * 15),     // did we do something?
+			fileWrite(p, "this is whatever\n"), // change state
+			NewChangedStep(1000*60, false),     // did we do something?
+			fileExpect(p, s),                   // check again
+			sleep(1),                           // we can sleep too!
+		}
+
+		testCases = append(testCases, test{
+			name:     "simple file",
+			res:      res,
+			fail:     false,
+			timeline: timeline,
+			expect:   func() error { return nil },
+			startup:  func() error { return nil },
+			cleanup:  func() error { return os.Remove(p) },
+		})
+	}
+	{
+		r := makeRes("exec", "x1")
+		res := r.(*ExecRes) // if this panics, the test will panic
+		s := "hello, world"
+		f := "/tmp/whatever"
+		res.Cmd = fmt.Sprintf("echo '%s' > '%s'", s, f)
+		res.Shell = "/bin/bash"
+		res.IfCmd = "! diff <(cat /tmp/whatever) <(echo hello, world)"
+		res.IfShell = "/bin/bash"
+		res.WatchCmd = fmt.Sprintf("/usr/bin/inotifywait -e modify -m %s", f)
+		//res.WatchShell = "/bin/bash"
+
+		timeline := []Step{
+			NewStartupStep(1000 * 60),        // startup
+			NewChangedStep(1000*60, false),   // did we do something?
+			fileExpect(f, s+"\n"),            // check initial state
+			NewClearChangedStep(1000 * 15),   // did we do something?
+			fileWrite(f, "this is stuff!\n"), // change state
+			NewChangedStep(1000*60, false),   // did we do something?
+			fileExpect(f, s+"\n"),            // check again
+			sleep(1),                         // we can sleep too!
+		}
+
+		testCases = append(testCases, test{
+			name:     "simple exec",
+			res:      res,
+			fail:     false,
+			timeline: timeline,
+			expect:   func() error { return nil },
+			// build file for inotifywait
+			startup: func() error { return ioutil.WriteFile(f, []byte("starting...\n"), 0666) },
+			cleanup: func() error { return os.Remove(f) },
+		})
+	}
+	{
+		r := makeRes("file", "r1")
+		res := r.(*FileRes) // if this panics, the test will panic
+		p := "/tmp/emptyfile"
+		res.Path = p
+		res.State = "exists"
+
+		timeline := []Step{
+			NewStartupStep(1000 * 60),      // startup
+			NewChangedStep(1000*60, false), // did we do something?
+			fileExpect(p, ""),              // check initial state
+			NewClearChangedStep(1000 * 15), // did we do something?
+		}
+
+		testCases = append(testCases, test{
+			name:     "touch file",
+			res:      res,
+			fail:     false,
+			timeline: timeline,
+			expect:   func() error { return nil },
+			startup:  func() error { return nil },
+			cleanup:  func() error { return os.Remove(p) },
+		})
+	}
+	{
+		r := makeRes("file", "r1")
+		res := r.(*FileRes) // if this panics, the test will panic
+		p := "/tmp/existingfile"
+		res.Path = p
+		res.State = "exists"
+		content := "some existing text\n"
+
+		timeline := []Step{
+			NewStartupStep(1000 * 60),     // startup
+			NewChangedStep(1000*60, true), // did we do something?
+			fileExpect(p, content),        // check initial state
+		}
+
+		testCases = append(testCases, test{
+			name:     "existing file",
+			res:      res,
+			fail:     false,
+			timeline: timeline,
+			expect:   func() error { return nil },
+			startup:  func() error { return ioutil.WriteFile(p, []byte(content), 0666) },
+			cleanup:  func() error { return os.Remove(p) },
+		})
+	}
+
+	names := []string{}
+	for index, tc := range testCases { // run all the tests
+		if tc.name == "" {
+			t.Errorf("test #%d: not named", index)
+			continue
+		}
+		if util.StrInList(tc.name, names) {
+			t.Errorf("test #%d: duplicate sub test name of: %s", index, tc.name)
+			continue
+		}
+		names = append(names, tc.name)
+		t.Run(fmt.Sprintf("test #%d (%s)", index, tc.name), func(t *testing.T) {
+			res, fail, experr, experrstr, timeline, expect, startup, cleanup := tc.res, tc.fail, tc.experr, tc.experrstr, tc.timeline, tc.expect, tc.startup, tc.cleanup
+
+			t.Logf("\n\ntest #%d: Res: %+v\n", index, res)
+			defer t.Logf("test #%d: done!", index)
+
+			// run validate!
+			err := res.Validate()
+
+			if !fail && err != nil {
+				t.Errorf("test #%d: FAIL", index)
+				t.Errorf("test #%d: could not validate Res: %+v", index, err)
+				return
+			}
+			if fail && err == nil {
+				t.Errorf("test #%d: FAIL", index)
+				t.Errorf("test #%d: validate passed, expected fail", index)
+				return
+			}
+			if fail && experr != nil && err != experr { // test for specific error!
+				t.Errorf("test #%d: FAIL", index)
+				t.Errorf("test #%d: expected validate fail, got wrong error", index)
+				t.Errorf("test #%d: got error: %+v", index, err)
+				t.Errorf("test #%d: exp error: %+v", index, experr)
+				return
+			}
+			// test for specific error string!
+			if fail && experrstr != "" && !strings.HasPrefix(err.Error(), experrstr) {
+				t.Errorf("test #%d: FAIL", index)
+				t.Errorf("test #%d: expected validate fail, got wrong error", index)
+				t.Errorf("test #%d: got error: %s", index, err.Error())
+				t.Errorf("test #%d: exp error: %s", index, experrstr)
+				return
+			}
+			if fail && err != nil {
+				t.Logf("test #%d: err: %+v", index, err)
+			}
+
+			changedChan := make(chan bool, 1) // buffered!
+			readyChan := make(chan struct{})
+			eventChan := make(chan struct{})
+			doneChan := make(chan struct{})
+			debug := testing.Verbose() // set via the -test.v flag to `go test`
+			logf := func(format string, v ...interface{}) {
+				t.Logf(fmt.Sprintf("test #%d: Res: ", index)+format, v...)
+			}
+			init := &engine.Init{
+				Running: func() {
+					close(readyChan)
+					select { // this always sends one!
+					case eventChan <- struct{}{}:
+
+					}
+				},
+				// Watch runs this to send a changed event.
+				Event: func() {
+					select {
+					case eventChan <- struct{}{}:
+
+					}
+				},
+
+				// Watch listens on this for close/pause events.
+				Done:  doneChan,
+				Debug: debug,
+				Logf:  logf,
+
+				// unused
+				Send: func(st interface{}) error {
+					return nil
+				},
+				Recv: func() map[string]*engine.Send {
+					return map[string]*engine.Send{}
+				},
+			}
+
+			t.Logf("test #%d: running startup()", index)
+			if err := startup(); err != nil {
+				t.Errorf("test #%d: FAIL", index)
+				t.Errorf("test #%d: could not startup: %+v", index, err)
+			}
+			// run init
+			t.Logf("test #%d: running Init", index)
+			err = res.Init(init)
+			defer func() {
+				t.Logf("test #%d: running cleanup()", index)
+				if err := cleanup(); err != nil {
+					t.Errorf("test #%d: FAIL", index)
+					t.Errorf("test #%d: could not cleanup: %+v", index, err)
+				}
+			}()
+			closeFn := func() {
+				// run close (we don't ever expect an error on close!)
+				t.Logf("test #%d: running Close", index)
+				if err := res.Close(); err != nil {
+					t.Errorf("test #%d: FAIL", index)
+					t.Errorf("test #%d: could not close Res: %+v", index, err)
+					//return
+				}
+			}
+
+			if !fail && err != nil {
+				t.Errorf("test #%d: FAIL", index)
+				t.Errorf("test #%d: could not init Res: %+v", index, err)
+				return
+			}
+			if fail && err == nil {
+				closeFn() // close if Init didn't fail
+				t.Errorf("test #%d: FAIL", index)
+				t.Errorf("test #%d: init passed, expected fail", index)
+				return
+			}
+			if fail && experr != nil && err != experr { // test for specific error!
+				t.Errorf("test #%d: FAIL", index)
+				t.Errorf("test #%d: expected init fail, got wrong error", index)
+				t.Errorf("test #%d: got error: %+v", index, err)
+				t.Errorf("test #%d: exp error: %+v", index, experr)
+				return
+			}
+			// test for specific error string!
+			if fail && experrstr != "" && !strings.HasPrefix(err.Error(), experrstr) {
+				t.Errorf("test #%d: FAIL", index)
+				t.Errorf("test #%d: expected init fail, got wrong error", index)
+				t.Errorf("test #%d: got error: %s", index, err.Error())
+				t.Errorf("test #%d: exp error: %s", index, experrstr)
+				return
+			}
+			if fail && err != nil {
+				t.Logf("test #%d: err: %+v", index, err)
+			}
+			defer closeFn()
+
+			// run watch
+			wg := &sync.WaitGroup{}
+			defer wg.Wait() // if we return early
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				t.Logf("test #%d: running Watch", index)
+				if err := res.Watch(); err != nil {
+					t.Errorf("test #%d: FAIL", index)
+					t.Errorf("test #%d: Watch failed: %s", index, err.Error())
+				}
+				close(eventChan) // done with this part
+			}()
+
+			// TODO: can we block here if the test fails early?
+			select {
+			case <-readyChan: // called by Running() in Watch
+			}
+			wg.Add(1)
+			go func() { // run timeline
+				t.Logf("test #%d: executing timeline", index)
+				defer wg.Done()
+				for ix, step := range timeline {
+
+					// magic setting of important values...
+					if s, ok := step.(*startupStep); ok {
+						s.ch = readyChan
+					}
+					if s, ok := step.(*changedStep); ok {
+						s.ch = changedChan
+					}
+					if s, ok := step.(*clearChangedStep); ok {
+						s.ch = changedChan
+					}
+
+					t.Logf("test #%d: step(%d)...", index, ix)
+					if err := step.Action(); err != nil {
+						t.Errorf("test #%d: FAIL", index)
+						t.Errorf("test #%d: step(%d) action failed: %s", index, ix, err.Error())
+						break
+					}
+					if err := step.Expect(); err != nil {
+						t.Errorf("test #%d: FAIL", index)
+						t.Errorf("test #%d: step(%d) expect failed: %s", index, ix, err.Error())
+						break
+					}
+				}
+				t.Logf("test #%d: shutting down Watch", index)
+				close(doneChan) // send Watch shutdown command
+			}()
+		Loop:
+			for {
+				select {
+				case _, ok := <-eventChan: // from Watch()
+					if !ok {
+						//t.Logf("test #%d: break!", index)
+						break Loop
+					}
+				}
+
+				t.Logf("test #%d: running CheckApply", index)
+				checkOK, err := res.CheckApply(true) // no noop!
+				if err != nil {
+					t.Errorf("test #%d: FAIL", index)
+					t.Errorf("test #%d: CheckApply failed: %s", index, err.Error())
+					return
+				}
+				//t.Logf("test #%d: CheckApply(true) (%t, %+v)", index, checkOK, err)
+				select {
+				// send a msg if we can, but never block
+				case changedChan <- checkOK:
+				default:
+				}
+			}
+
+			t.Logf("test #%d: waiting for shutdown", index)
+			wg.Wait()
+
+			if err := expect(); err != nil {
+				t.Errorf("test #%d: FAIL", index)
+				t.Errorf("test #%d: expect failed: %s", index, err.Error())
+				return
+			}
+
+			// all done!
+		})
+	}
+}

engine/resources/svc.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -28,11 +28,11 @@ import (
 	"github.com/purpleidea/mgmt/engine/traits"
 	engineUtil "github.com/purpleidea/mgmt/engine/util"
 	"github.com/purpleidea/mgmt/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
 
 	systemd "github.com/coreos/go-systemd/dbus" // change namespace
 	systemdUtil "github.com/coreos/go-systemd/util"
 	"github.com/godbus/dbus" // namespace collides with systemd wrapper
-	errwrap "github.com/pkg/errors"
 )
 
 func init() {
@@ -120,10 +120,7 @@ func (obj *SvcRes) Watch() error {
 	bus.Signal(buschan)
 	defer bus.RemoveSignal(buschan) // not needed here, but nice for symmetry
 
-	// notify engine that we're running
-	if err := obj.init.Running(); err != nil {
-		return err // exit if requested
-	}
+	obj.init.Running() // when started, notify engine that we're running
 
 	var svc = fmt.Sprintf("%s.service", obj.Name()) // systemd name
 	var send = false                                // send event?
@@ -161,7 +158,6 @@ func (obj *SvcRes) Watch() error {
 
 		if previous != invalid { // if invalid changed, send signal
 			send = true
-			obj.init.Dirty() // dirty
 		}
 
 		if invalid {
@@ -176,10 +172,8 @@ func (obj *SvcRes) Watch() error {
 				// loop so that we can see the changed invalid signal
 				obj.init.Logf("daemon reload")
 
-			case event := <-obj.init.Events:
-				if err := obj.init.Read(event); err != nil {
-					return err
-				}
+			case <-obj.init.Done: // closed by the engine to signal shutdown
+				return nil
 			}
 		} else {
 			if !activeSet {
@@ -217,35 +211,31 @@ func (obj *SvcRes) Watch() error {
 					obj.init.Logf("stopped")
 				}
 				send = true
-				obj.init.Dirty() // dirty
 
 			case err := <-subErrors:
 				return errwrap.Wrapf(err, "unknown %s error", obj)
 
-			case event := <-obj.init.Events:
-				if err := obj.init.Read(event); err != nil {
-					return err
-				}
+			case <-obj.init.Done: // closed by the engine to signal shutdown
+				return nil
 			}
 		}
 
 		if send {
 			send = false
-			if err := obj.init.Event(); err != nil {
-				return err // exit if requested
-			}
+			obj.init.Event() // notify engine of an event (this can block)
 		}
 	}
 }
 
 // CheckApply checks the resource state and applies the resource if the bool
 // input is true. It returns error info and if the state check passed or not.
-func (obj *SvcRes) CheckApply(apply bool) (checkOK bool, err error) {
+func (obj *SvcRes) CheckApply(apply bool) (bool, error) {
 	if !systemdUtil.IsRunningSystemd() {
 		return false, fmt.Errorf("systemd is not running")
 	}
 
 	var conn *systemd.Conn
+	var err error
 	if obj.Session {
 		conn, err = systemd.NewUserConnection() // user session
 	} else {
@@ -343,7 +333,12 @@ func (obj *SvcRes) CheckApply(apply bool) (checkOK bool, err error) {
 	if &status == nil {
 		return false, fmt.Errorf("systemd service action result is nil")
 	}
-	if status != "done" {
+	switch status {
+	case "done":
+		// pass
+	case "failed":
+		return false, fmt.Errorf("svc failed (selinux?)")
+	default:
 		return false, fmt.Errorf("unknown systemd return string: %v", status)
 	}
 

engine/resources/test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -125,35 +125,19 @@ func (obj *TestRes) Close() error {
 
 // Watch is the primary listener for this resource and it outputs events.
 func (obj *TestRes) Watch() error {
-	// notify engine that we're running
-	if err := obj.init.Running(); err != nil {
-		return err // exit if requested
+	obj.init.Running() // when started, notify engine that we're running
+
+	select {
+	case <-obj.init.Done: // closed by the engine to signal shutdown
 	}
 
-	var send = false // send event?
-	for {
-		select {
-		case event, ok := <-obj.init.Events:
-			if !ok {
-				return nil
-			}
-			if err := obj.init.Read(event); err != nil {
-				return err
-			}
-		}
+	//obj.init.Event() // notify engine of an event (this can block)
 
-		// do all our event sending all together to avoid duplicate msgs
-		if send {
-			send = false
-			if err := obj.init.Event(); err != nil {
-				return err // exit if requested
-			}
-		}
-	}
+	return nil
 }
 
 // CheckApply method for Test resource. Does nothing, returns happy!
-func (obj *TestRes) CheckApply(apply bool) (checkOK bool, err error) {
+func (obj *TestRes) CheckApply(apply bool) (bool, error) {
 	for key, val := range obj.init.Recv() {
 		obj.init.Logf("CheckApply: Received `%s`, changed: %t", key, val.Changed)
 	}
@@ -417,8 +401,8 @@ func (obj *TestRes) GroupCmp(r engine.GroupableRes) error {
 // TestSends is the struct of data which is sent after a successful Apply.
 type TestSends struct {
 	// Hello is some value being sent.
-	Hello  *string
-	Answer int // some other value being sent
+	Hello  *string `lang:"hello"`
+	Answer int     `lang:"answer"` // some other value being sent
 }
 
 // Sends represents the default struct of values we can send using Send/Recv.

engine/resources/test_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/resources/timer.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -75,10 +75,7 @@ func (obj *TimerRes) Watch() error {
 	obj.ticker = obj.newTicker()
 	defer obj.ticker.Stop()
 
-	// notify engine that we're running
-	if err := obj.init.Running(); err != nil {
-		return err // exit if requested
-	}
+	obj.init.Running() // when started, notify engine that we're running
 
 	var send = false // send event?
 	for {
@@ -87,20 +84,13 @@ func (obj *TimerRes) Watch() error {
 			send = true
 			obj.init.Logf("received tick")
 
-		case event, ok := <-obj.init.Events:
-			if !ok {
-				return nil
-			}
-			if err := obj.init.Read(event); err != nil {
-				return err
-			}
+		case <-obj.init.Done: // closed by the engine to signal shutdown
+			return nil
 		}
 
 		if send {
 			send = false
-			if err := obj.init.Event(); err != nil {
-				return err // exit if requested
-			}
+			obj.init.Event() // notify engine of an event (this can block)
 		}
 	}
 }

engine/resources/user.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -30,8 +30,7 @@ import (
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/engine/traits"
 	"github.com/purpleidea/mgmt/recwatch"
-
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 func init() {
@@ -119,10 +118,7 @@ func (obj *UserRes) Watch() error {
 	}
 	defer obj.recWatcher.Close()
 
-	// notify engine that we're running
-	if err := obj.init.Running(); err != nil {
-		return err // exit if requested
-	}
+	obj.init.Running() // when started, notify engine that we're running
 
 	var send = false // send event?
 	for {
@@ -142,29 +138,21 @@ func (obj *UserRes) Watch() error {
 				obj.init.Logf("Event(%s): %v", event.Body.Name, event.Body.Op)
 			}
 			send = true
-			obj.init.Dirty() // dirty
 
-		case event, ok := <-obj.init.Events:
-			if !ok {
-				return nil
-			}
-			if err := obj.init.Read(event); err != nil {
-				return err
-			}
+		case <-obj.init.Done: // closed by the engine to signal shutdown
+			return nil
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
 			send = false
-			if err := obj.init.Event(); err != nil {
-				return err // exit if requested
-			}
+			obj.init.Event() // notify engine of an event (this can block)
 		}
 	}
 }
 
 // CheckApply method for User resource.
-func (obj *UserRes) CheckApply(apply bool) (checkOK bool, err error) {
+func (obj *UserRes) CheckApply(apply bool) (bool, error) {
 	obj.init.Logf("CheckApply(%t)", apply)
 
 	var exists = true

engine/resources/virt.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -29,11 +29,12 @@ import (
 
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/engine/traits"
+	engineUtil "github.com/purpleidea/mgmt/engine/util"
 	"github.com/purpleidea/mgmt/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
 
 	"github.com/libvirt/libvirt-go"
 	libvirtxml "github.com/libvirt/libvirt-go-xml"
-	errwrap "github.com/pkg/errors"
 )
 
 func init() {
@@ -65,30 +66,53 @@ const (
 // VirtRes is a libvirt resource. A transient virt resource, which has its state
 // set to `shutoff` is one which does not exist. The parallel equivalent is a
 // file resource which removes a particular path.
+// TODO: some values inside here should be enum's!
 type VirtRes struct {
 	traits.Base // add the base methods without re-implementation
 	traits.Refreshable
 
 	init *engine.Init
 
-	URI        string             `yaml:"uri"`       // connection uri, eg: qemu:///session
-	State      string             `yaml:"state"`     // running, paused, shutoff
-	Transient  bool               `yaml:"transient"` // defined (false) or undefined (true)
-	CPUs       uint               `yaml:"cpus"`
-	MaxCPUs    uint               `yaml:"maxcpus"`
-	Memory     uint64             `yaml:"memory"` // in KBytes
-	OSInit     string             `yaml:"osinit"` // init used by lxc
-	Boot       []string           `yaml:"boot"`   // boot order. values: fd, hd, cdrom, network
-	Disk       []diskDevice       `yaml:"disk"`
-	CDRom      []cdRomDevice      `yaml:"cdrom"`
-	Network    []networkDevice    `yaml:"network"`
-	Filesystem []filesystemDevice `yaml:"filesystem"`
-	Auth       *VirtAuth          `yaml:"auth"`
+	// URI is the libvirt connection URI, eg: `qemu:///session`.
+	URI string `lang:"uri" yaml:"uri"`
+	// State is the desired vm state. Possible values include: `running`,
+	// `paused` and `shutoff`.
+	State string `lang:"state" yaml:"state"`
+	// Transient is whether the vm is defined (false) or undefined (true).
+	Transient bool `lang:"transient" yaml:"transient"`
 
-	HotCPUs bool `yaml:"hotcpus"` // allow hotplug of cpus?
-	// FIXME: values here should be enum's!
-	RestartOnDiverge string `yaml:"restartondiverge"` // restart policy: "ignore", "ifneeded", "error"
-	RestartOnRefresh bool   `yaml:"restartonrefresh"` // restart on refresh?
+	// CPUs is the desired cpu count of the machine.
+	CPUs uint `lang:"cpus" yaml:"cpus"`
+	// MaxCPUs is the maximum number of cpus allowed in the machine. You
+	// need to set this so that on boot the `hardware` knows how many cpu
+	// `slots` it might need to make room for.
+	MaxCPUs uint `lang:"maxcpus" yaml:"maxcpus"`
+	// HotCPUs specifies whether we can hot plug and unplug cpus.
+	HotCPUs bool `lang:"hotcpus" yaml:"hotcpus"`
+	// Memory is the size in KBytes of memory to include in the machine.
+	Memory uint64 `lang:"memory" yaml:"memory"`
+
+	// OSInit is the init used by lxc.
+	OSInit string `lang:"osinit" yaml:"osinit"`
+	// Boot is the boot order. Values are `fd`, `hd`, `cdrom` and `network`.
+	Boot []string `lang:"boot" yaml:"boot"`
+	// Disk is the list of disk devices to include.
+	Disk []*DiskDevice `lang:"disk" yaml:"disk"`
+	// CdRom is the list of cdrom devices to include.
+	CDRom []*CDRomDevice `lang:"cdrom" yaml:"cdrom"`
+	// Network is the list of network devices to include.
+	Network []*NetworkDevice `lang:"network" yaml:"network"`
+	// Filesystem is the list of file system devices to include.
+	Filesystem []*FilesystemDevice `lang:"filesystem" yaml:"filesystem"`
+
+	// Auth points to the libvirt credentials to use if any are necessary.
+	Auth *VirtAuth `lang:"auth" yaml:"auth"`
+
+	// RestartOnDiverge is the restart policy, and can be: `ignore`,
+	// `ifneeded` or `error`.
+	RestartOnDiverge string `lang:"restartondiverge" yaml:"restartondiverge"`
+	// RestartOnRefresh specifies if we restart on refresh signal.
+	RestartOnRefresh bool `lang:"restartonrefresh" yaml:"restartonrefresh"`
 
 	wg                  *sync.WaitGroup
 	conn                *libvirt.Connect
@@ -103,8 +127,26 @@ type VirtRes struct {
 
 // VirtAuth is used to pass credentials to libvirt.
 type VirtAuth struct {
-	Username string `yaml:"username"`
-	Password string `yaml:"password"`
+	Username string `lang:"username" yaml:"username"`
+	Password string `lang:"password" yaml:"password"`
+}
+
+// Cmp compares two VirtAuth structs. It errors if they are not identical.
+func (obj *VirtAuth) Cmp(auth *VirtAuth) error {
+	if (obj == nil) != (auth == nil) { // xor
+		return fmt.Errorf("the VirtAuth differs")
+	}
+	if obj == nil && auth == nil {
+		return nil
+	}
+
+	if obj.Username != auth.Username {
+		return fmt.Errorf("the Username differs")
+	}
+	if obj.Password != auth.Password {
+		return fmt.Errorf("the Password differs")
+	}
+	return nil
 }
 
 // Default returns some sensible defaults for this resource.
@@ -138,7 +180,7 @@ func (obj *VirtRes) Init(init *engine.Init) error {
 	var u *url.URL
 	var err error
 	if u, err = url.Parse(obj.URI); err != nil {
-		return errwrap.Wrapf(err, "%s: Parsing URI failed: %s", obj, obj.URI)
+		return errwrap.Wrapf(err, "parsing URI (`%s`) failed", obj.URI)
 	}
 	switch u.Scheme {
 	case "lxc":
@@ -149,7 +191,7 @@ func (obj *VirtRes) Init(init *engine.Init) error {
 
 	obj.conn, err = obj.connect() // gets closed in Close method of Res API
 	if err != nil {
-		return errwrap.Wrapf(err, "%s: Connection to libvirt failed in init", obj)
+		return errwrap.Wrapf(err, "connection to libvirt failed in init")
 	}
 
 	// check for hard to change properties
@@ -157,14 +199,14 @@ func (obj *VirtRes) Init(init *engine.Init) error {
 	if err == nil {
 		defer dom.Free()
 	} else if !isNotFound(err) {
-		return errwrap.Wrapf(err, "%s: Could not lookup on init", obj)
+		return errwrap.Wrapf(err, "could not lookup on init")
 	}
 
 	if err == nil {
 		// maxCPUs, err := dom.GetMaxVcpus()
 		i, err := dom.GetVcpusFlags(libvirt.DOMAIN_VCPU_MAXIMUM)
 		if err != nil {
-			return errwrap.Wrapf(err, "%s: Could not lookup MaxCPUs on init", obj)
+			return errwrap.Wrapf(err, "could not lookup MaxCPUs on init")
 		}
 		maxCPUs := uint(i)
 		if obj.MaxCPUs != maxCPUs { // max cpu slots is hard to change
@@ -177,11 +219,11 @@ func (obj *VirtRes) Init(init *engine.Init) error {
 		// event handlers so that we don't miss any events via race?
 		xmlDesc, err := dom.GetXMLDesc(0) // 0 means no flags
 		if err != nil {
-			return errwrap.Wrapf(err, "%s: Could not GetXMLDesc on init", obj)
+			return errwrap.Wrapf(err, "could not GetXMLDesc on init")
 		}
 		domXML := &libvirtxml.Domain{}
 		if err := domXML.Unmarshal(xmlDesc); err != nil {
-			return errwrap.Wrapf(err, "%s: Could not unmarshal XML on init", obj)
+			return errwrap.Wrapf(err, "could not unmarshal XML on init")
 		}
 
 		// guest agent: domain->devices->channel->target->state == connected?
@@ -326,10 +368,7 @@ func (obj *VirtRes) Watch() error {
 	}
 	defer obj.conn.DomainEventDeregister(gaCallbackID)
 
-	// notify engine that we're running
-	if err := obj.init.Running(); err != nil {
-		return err // exit if requested
-	}
+	obj.init.Running() // when started, notify engine that we're running
 
 	var send = false // send event?
 	for {
@@ -340,31 +379,26 @@ func (obj *VirtRes) Watch() error {
 			switch event {
 			case libvirt.DOMAIN_EVENT_DEFINED:
 				if obj.Transient {
-					obj.init.Dirty() // dirty
 					send = true
 				}
 			case libvirt.DOMAIN_EVENT_UNDEFINED:
 				if !obj.Transient {
-					obj.init.Dirty() // dirty
 					send = true
 				}
 			case libvirt.DOMAIN_EVENT_STARTED:
 				fallthrough
 			case libvirt.DOMAIN_EVENT_RESUMED:
 				if obj.State != "running" {
-					obj.init.Dirty() // dirty
 					send = true
 				}
 			case libvirt.DOMAIN_EVENT_SUSPENDED:
 				if obj.State != "paused" {
-					obj.init.Dirty() // dirty
 					send = true
 				}
 			case libvirt.DOMAIN_EVENT_STOPPED:
 				fallthrough
 			case libvirt.DOMAIN_EVENT_SHUTDOWN:
 				if obj.State != "shutoff" {
-					obj.init.Dirty() // dirty
 					send = true
 				}
 				processExited = true
@@ -375,7 +409,6 @@ func (obj *VirtRes) Watch() error {
 				// verify, detect and patch appropriately!
 				fallthrough
 			case libvirt.DOMAIN_EVENT_CRASHED:
-				obj.init.Dirty() // dirty
 				send = true
 				processExited = true // FIXME: is this okay for PMSUSPENDED ?
 			}
@@ -390,40 +423,32 @@ func (obj *VirtRes) Watch() error {
 
 			if state == libvirt.CONNECT_DOMAIN_EVENT_AGENT_LIFECYCLE_STATE_CONNECTED {
 				obj.guestAgentConnected = true
-				obj.init.Dirty() // dirty
 				send = true
-				obj.init.Logf("Guest agent connected")
+				obj.init.Logf("guest agent connected")
 
 			} else if state == libvirt.CONNECT_DOMAIN_EVENT_AGENT_LIFECYCLE_STATE_DISCONNECTED {
 				obj.guestAgentConnected = false
 				// ignore CONNECT_DOMAIN_EVENT_AGENT_LIFECYCLE_REASON_DOMAIN_STARTED
 				// events because they just tell you that guest agent channel was added
 				if reason == libvirt.CONNECT_DOMAIN_EVENT_AGENT_LIFECYCLE_REASON_CHANNEL {
-					obj.init.Logf("Guest agent disconnected")
+					obj.init.Logf("guest agent disconnected")
 				}
 
 			} else {
-				return fmt.Errorf("unknown %s guest agent state: %v", obj, state)
+				return fmt.Errorf("unknown guest agent state: %v", state)
 			}
 
 		case err := <-errorChan:
-			return fmt.Errorf("unknown %s libvirt error: %s", obj, err)
+			return errwrap.Wrapf(err, "unknown libvirt error")
 
-		case event, ok := <-obj.init.Events:
-			if !ok {
-				return nil
-			}
-			if err := obj.init.Read(event); err != nil {
-				return err
-			}
+		case <-obj.init.Done: // closed by the engine to signal shutdown
+			return nil
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
 			send = false
-			if err := obj.init.Event(); err != nil {
-				return err // exit if requested
-			}
+			obj.init.Event() // notify engine of an event (this can block)
 		}
 	}
 }
@@ -451,7 +476,7 @@ func (obj *VirtRes) domainCreate() (*libvirt.Domain, bool, error) {
 		if err != nil {
 			return dom, false, err // returned dom is invalid
 		}
-		obj.init.Logf("Domain transient %s", state)
+		obj.init.Logf("transient domain %s", state) // log the state
 		return dom, false, nil
 	}
 
@@ -459,20 +484,20 @@ func (obj *VirtRes) domainCreate() (*libvirt.Domain, bool, error) {
 	if err != nil {
 		return dom, false, err // returned dom is invalid
 	}
-	obj.init.Logf("Domain defined")
+	obj.init.Logf("domain defined")
 
 	if obj.State == "running" {
 		if err := dom.Create(); err != nil {
 			return dom, false, err
 		}
-		obj.init.Logf("Domain started")
+		obj.init.Logf("domain started")
 	}
 
 	if obj.State == "paused" {
 		if err := dom.CreateWithFlags(libvirt.DOMAIN_START_PAUSED); err != nil {
 			return dom, false, err
 		}
-		obj.init.Logf("Domain created paused")
+		obj.init.Logf("domain created paused")
 	}
 
 	return dom, false, nil
@@ -500,7 +525,7 @@ func (obj *VirtRes) stateCheckApply(apply bool, dom *libvirt.Domain) (bool, erro
 		}
 		if domInfo.State == libvirt.DOMAIN_BLOCKED {
 			// TODO: what should happen?
-			return false, fmt.Errorf("domain %s is blocked", obj.Name())
+			return false, fmt.Errorf("domain is blocked")
 		}
 		if !apply {
 			return false, nil
@@ -510,14 +535,14 @@ func (obj *VirtRes) stateCheckApply(apply bool, dom *libvirt.Domain) (bool, erro
 				return false, errwrap.Wrapf(err, "domain.Resume failed")
 			}
 			checkOK = false
-			obj.init.Logf("Domain resumed")
+			obj.init.Logf("domain resumed")
 			break
 		}
 		if err := dom.Create(); err != nil {
 			return false, errwrap.Wrapf(err, "domain.Create failed")
 		}
 		checkOK = false
-		obj.init.Logf("Domain created")
+		obj.init.Logf("domain created")
 
 	case "paused":
 		if domInfo.State == libvirt.DOMAIN_PAUSED {
@@ -531,14 +556,14 @@ func (obj *VirtRes) stateCheckApply(apply bool, dom *libvirt.Domain) (bool, erro
 				return false, errwrap.Wrapf(err, "domain.Suspend failed")
 			}
 			checkOK = false
-			obj.init.Logf("Domain paused")
+			obj.init.Logf("domain paused")
 			break
 		}
 		if err := dom.CreateWithFlags(libvirt.DOMAIN_START_PAUSED); err != nil {
 			return false, errwrap.Wrapf(err, "domain.CreateWithFlags failed")
 		}
 		checkOK = false
-		obj.init.Logf("Domain created paused")
+		obj.init.Logf("domain created paused")
 
 	case "shutoff":
 		if domInfo.State == libvirt.DOMAIN_SHUTOFF || domInfo.State == libvirt.DOMAIN_SHUTDOWN {
@@ -552,7 +577,7 @@ func (obj *VirtRes) stateCheckApply(apply bool, dom *libvirt.Domain) (bool, erro
 			return false, errwrap.Wrapf(err, "domain.Destroy failed")
 		}
 		checkOK = false
-		obj.init.Logf("Domain destroyed")
+		obj.init.Logf("domain destroyed")
 	}
 
 	return checkOK, nil
@@ -578,7 +603,7 @@ func (obj *VirtRes) attrCheckApply(apply bool, dom *libvirt.Domain) (bool, error
 		if err := dom.SetMemory(obj.Memory); err != nil {
 			return false, errwrap.Wrapf(err, "domain.SetMemory failed")
 		}
-		obj.init.Logf("Memory changed to %d", obj.Memory)
+		obj.init.Logf("memory changed to: %d", obj.Memory)
 	}
 
 	// check cpus
@@ -617,7 +642,7 @@ func (obj *VirtRes) attrCheckApply(apply bool, dom *libvirt.Domain) (bool, error
 				return false, errwrap.Wrapf(err, "domain.SetVcpus failed")
 			}
 			checkOK = false
-			obj.init.Logf("CPUs (hot) changed to %d", obj.CPUs)
+			obj.init.Logf("cpus (hot) changed to: %d", obj.CPUs)
 
 		case libvirt.DOMAIN_SHUTOFF, libvirt.DOMAIN_SHUTDOWN:
 			if !obj.Transient {
@@ -629,7 +654,7 @@ func (obj *VirtRes) attrCheckApply(apply bool, dom *libvirt.Domain) (bool, error
 					return false, errwrap.Wrapf(err, "domain.SetVcpus failed")
 				}
 				checkOK = false
-				obj.init.Logf("CPUs (cold) changed to %d", obj.CPUs)
+				obj.init.Logf("cpus (cold) changed to: %d", obj.CPUs)
 			}
 
 		default:
@@ -660,7 +685,7 @@ func (obj *VirtRes) attrCheckApply(apply bool, dom *libvirt.Domain) (bool, error
 				return false, errwrap.Wrapf(err, "domain.SetVcpus failed")
 			}
 			checkOK = false
-			obj.init.Logf("CPUs (guest) changed to %d", obj.CPUs)
+			obj.init.Logf("cpus (guest) changed to: %d", obj.CPUs)
 		}
 	}
 
@@ -684,7 +709,7 @@ func (obj *VirtRes) domainShutdownSync(apply bool, dom *libvirt.Domain) (bool, e
 			return false, errwrap.Wrapf(err, "domain.GetInfo failed")
 		}
 		if domInfo.State == libvirt.DOMAIN_SHUTOFF || domInfo.State == libvirt.DOMAIN_SHUTDOWN {
-			obj.init.Logf("Shutdown")
+			obj.init.Logf("shutdown")
 			break
 		}
 
@@ -696,7 +721,7 @@ func (obj *VirtRes) domainShutdownSync(apply bool, dom *libvirt.Domain) (bool, e
 			obj.processExitChan = make(chan struct{})
 			// if machine shuts down before we call this, we error;
 			// this isn't ideal, but it happened due to user error!
-			obj.init.Logf("Running shutdown")
+			obj.init.Logf("running shutdown")
 			if err := dom.Shutdown(); err != nil {
 				// FIXME: if machine is already shutdown completely, return early
 				return false, errwrap.Wrapf(err, "domain.Shutdown failed")
@@ -717,7 +742,7 @@ func (obj *VirtRes) domainShutdownSync(apply bool, dom *libvirt.Domain) (bool, e
 			// https://libvirt.org/formatdomain.html#elementsEvents
 			continue
 		case <-timeout:
-			return false, fmt.Errorf("%s: didn't shutdown after %d seconds", obj, MaxShutdownDelayTimeout)
+			return false, fmt.Errorf("didn't shutdown after %d seconds", MaxShutdownDelayTimeout)
 		}
 	}
 
@@ -727,8 +752,8 @@ func (obj *VirtRes) domainShutdownSync(apply bool, dom *libvirt.Domain) (bool, e
 // CheckApply checks the resource state and applies the resource if the bool
 // input is true. It returns error info and if the state check passed or not.
 func (obj *VirtRes) CheckApply(apply bool) (bool, error) {
-	if obj.conn == nil {
-		panic("virt: CheckApply is being called with nil connection")
+	if obj.conn == nil { // programming error?
+		return false, fmt.Errorf("got called with nil connection")
 	}
 	// if we do the restart, we must flip the flag back to false as evidence
 	var restart bool                                // do we need to do a restart?
@@ -789,7 +814,7 @@ func (obj *VirtRes) CheckApply(apply bool) (bool, error) {
 			if err := dom.Undefine(); err != nil {
 				return false, errwrap.Wrapf(err, "domain.Undefine failed")
 			}
-			obj.init.Logf("Domain undefined")
+			obj.init.Logf("domain undefined")
 		} else {
 			domXML, err := dom.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
 			if err != nil {
@@ -798,7 +823,7 @@ func (obj *VirtRes) CheckApply(apply bool) (bool, error) {
 			if _, err = obj.conn.DomainDefineXML(domXML); err != nil {
 				return false, errwrap.Wrapf(err, "conn.DomainDefineXML failed")
 			}
-			obj.init.Logf("Domain defined")
+			obj.init.Logf("domain defined")
 		}
 		checkOK = false
 	}
@@ -846,7 +871,7 @@ func (obj *VirtRes) CheckApply(apply bool) (bool, error) {
 
 	// we had to do a restart, we didn't, and we should error if it was needed
 	if obj.restartScheduled && restart == true && obj.RestartOnDiverge == "error" {
-		return false, fmt.Errorf("%s: needed restart but didn't! (RestartOnDiverge: %v)", obj, obj.RestartOnDiverge)
+		return false, fmt.Errorf("needed restart but didn't! (RestartOnDiverge: %s)", obj.RestartOnDiverge)
 	}
 
 	return checkOK, nil // w00t
@@ -970,44 +995,56 @@ type virtDevice interface {
 	GetXML(idx int) string
 }
 
-type diskDevice struct {
-	Source string `yaml:"source"`
-	Type   string `yaml:"type"`
+// DiskDevice represents a disk that is attached to the virt machine.
+type DiskDevice struct {
+	Source string `lang:"source" yaml:"source"`
+	Type   string `lang:"type" yaml:"type"`
 }
 
-type cdRomDevice struct {
-	Source string `yaml:"source"`
-	Type   string `yaml:"type"`
-}
-
-type networkDevice struct {
-	Name string `yaml:"name"`
-	MAC  string `yaml:"mac"`
-}
-
-type filesystemDevice struct {
-	Access   string `yaml:"access"`
-	Source   string `yaml:"source"`
-	Target   string `yaml:"target"`
-	ReadOnly bool   `yaml:"read_only"`
-}
-
-func (d *diskDevice) GetXML(idx int) string {
-	source, _ := util.ExpandHome(d.Source) // TODO: should we handle errors?
+// GetXML returns the XML representation of this device.
+func (obj *DiskDevice) GetXML(idx int) string {
+	source, _ := util.ExpandHome(obj.Source) // TODO: should we handle errors?
 	var b string
 	b += "<disk type='file' device='disk'>"
-	b += fmt.Sprintf("<driver name='qemu' type='%s'/>", d.Type)
+	b += fmt.Sprintf("<driver name='qemu' type='%s'/>", obj.Type)
 	b += fmt.Sprintf("<source file='%s'/>", source)
 	b += fmt.Sprintf("<target dev='vd%s' bus='virtio'/>", util.NumToAlpha(idx))
 	b += "</disk>"
 	return b
 }
 
-func (d *cdRomDevice) GetXML(idx int) string {
-	source, _ := util.ExpandHome(d.Source) // TODO: should we handle errors?
+// Cmp compares two DiskDevice's and returns an error if they are not
+// equivalent.
+func (obj *DiskDevice) Cmp(dev *DiskDevice) error {
+	if (obj == nil) != (dev == nil) { // xor
+		return fmt.Errorf("the DiskDevice differs")
+	}
+	if obj == nil && dev == nil {
+		return nil
+	}
+
+	if obj.Source != dev.Source {
+		return fmt.Errorf("the Source differs")
+	}
+	if obj.Type != dev.Type {
+		return fmt.Errorf("the Type differs")
+	}
+
+	return nil
+}
+
+// CDRomDevice represents a CDRom device that is attached to the virt machine.
+type CDRomDevice struct {
+	Source string `lang:"source" yaml:"source"`
+	Type   string `lang:"type" yaml:"type"`
+}
+
+// GetXML returns the XML representation of this device.
+func (obj *CDRomDevice) GetXML(idx int) string {
+	source, _ := util.ExpandHome(obj.Source) // TODO: should we handle errors?
 	var b string
 	b += "<disk type='file' device='cdrom'>"
-	b += fmt.Sprintf("<driver name='qemu' type='%s'/>", d.Type)
+	b += fmt.Sprintf("<driver name='qemu' type='%s'/>", obj.Type)
 	b += fmt.Sprintf("<source file='%s'/>", source)
 	b += fmt.Sprintf("<target dev='hd%s' bus='ide'/>", util.NumToAlpha(idx))
 	b += "<readonly/>"
@@ -1015,68 +1052,147 @@ func (d *cdRomDevice) GetXML(idx int) string {
 	return b
 }
 
-func (d *networkDevice) GetXML(idx int) string {
-	if d.MAC == "" {
-		d.MAC = randMAC()
+// Cmp compares two CDRomDevice's and returns an error if they are not
+// equivalent.
+func (obj *CDRomDevice) Cmp(dev *CDRomDevice) error {
+	if (obj == nil) != (dev == nil) { // xor
+		return fmt.Errorf("the CDRomDevice differs")
+	}
+	if obj == nil && dev == nil {
+		return nil
+	}
+
+	if obj.Source != dev.Source {
+		return fmt.Errorf("the Source differs")
+	}
+	if obj.Type != dev.Type {
+		return fmt.Errorf("the Type differs")
+	}
+
+	return nil
+}
+
+// NetworkDevice represents a network card that is attached to the virt machine.
+type NetworkDevice struct {
+	Name string `lang:"name" yaml:"name"`
+	MAC  string `lang:"mac" yaml:"mac"`
+}
+
+// GetXML returns the XML representation of this device.
+func (obj *NetworkDevice) GetXML(idx int) string {
+	if obj.MAC == "" {
+		obj.MAC = randMAC()
 	}
 	var b string
 	b += "<interface type='network'>"
-	b += fmt.Sprintf("<mac address='%s'/>", d.MAC)
-	b += fmt.Sprintf("<source network='%s'/>", d.Name)
+	b += fmt.Sprintf("<mac address='%s'/>", obj.MAC)
+	b += fmt.Sprintf("<source network='%s'/>", obj.Name)
 	b += "</interface>"
 	return b
 }
 
-func (d *filesystemDevice) GetXML(idx int) string {
-	source, _ := util.ExpandHome(d.Source) // TODO: should we handle errors?
+// Cmp compares two NetworkDevice's and returns an error if they are not
+// equivalent.
+func (obj *NetworkDevice) Cmp(dev *NetworkDevice) error {
+	if (obj == nil) != (dev == nil) { // xor
+		return fmt.Errorf("the NetworkDevice differs")
+	}
+	if obj == nil && dev == nil {
+		return nil
+	}
+
+	if obj.Name != dev.Name {
+		return fmt.Errorf("the Name differs")
+	}
+	if obj.MAC != dev.MAC {
+		return fmt.Errorf("the MAC differs")
+	}
+
+	return nil
+}
+
+// FilesystemDevice represents a filesystem that is attached to the virt
+// machine.
+type FilesystemDevice struct {
+	Access   string `lang:"access" yaml:"access"`
+	Source   string `lang:"source" yaml:"source"`
+	Target   string `lang:"target" yaml:"target"`
+	ReadOnly bool   `lang:"read_only" yaml:"read_only"`
+}
+
+// GetXML returns the XML representation of this device.
+func (obj *FilesystemDevice) GetXML(idx int) string {
+	source, _ := util.ExpandHome(obj.Source) // TODO: should we handle errors?
 	var b string
 	b += "<filesystem" // open
-	if d.Access != "" {
-		b += fmt.Sprintf(" accessmode='%s'", d.Access)
+	if obj.Access != "" {
+		b += fmt.Sprintf(" accessmode='%s'", obj.Access)
 	}
 	b += ">" // close
 	b += fmt.Sprintf("<source dir='%s'/>", source)
-	b += fmt.Sprintf("<target dir='%s'/>", d.Target)
-	if d.ReadOnly {
+	b += fmt.Sprintf("<target dir='%s'/>", obj.Target)
+	if obj.ReadOnly {
 		b += "<readonly/>"
 	}
 	b += "</filesystem>"
 	return b
 }
 
-// Cmp compares two resources and returns an error if they are not equivalent.
-func (obj *VirtRes) Cmp(r engine.Res) error {
-	if !obj.Compare(r) {
-		return fmt.Errorf("did not compare")
+// Cmp compares two FilesystemDevice's and returns an error if they are not
+// equivalent.
+func (obj *FilesystemDevice) Cmp(dev *FilesystemDevice) error {
+	if (obj == nil) != (dev == nil) { // xor
+		return fmt.Errorf("the FilesystemDevice differs")
 	}
+	if obj == nil && dev == nil {
+		return nil
+	}
+
+	if obj.Access != dev.Access {
+		return fmt.Errorf("the Access differs")
+	}
+	if obj.Source != dev.Source {
+		return fmt.Errorf("the Source differs")
+	}
+	if obj.Target != dev.Target {
+		return fmt.Errorf("the Target differs")
+	}
+	if obj.ReadOnly != dev.ReadOnly {
+		return fmt.Errorf("the ReadOnly differs")
+	}
+
 	return nil
 }
 
-// Compare two resources and return if they are equivalent.
-func (obj *VirtRes) Compare(r engine.Res) bool {
+// Cmp compares two resources and returns an error if they are not equivalent.
+func (obj *VirtRes) Cmp(r engine.Res) error {
 	// we can only compare VirtRes to others of the same resource kind
 	res, ok := r.(*VirtRes)
 	if !ok {
-		return false
+		return fmt.Errorf("not a %s", obj.Kind())
 	}
 
 	if obj.URI != res.URI {
-		return false
+		return fmt.Errorf("the URI differs")
 	}
 	if obj.State != res.State {
-		return false
+		return fmt.Errorf("the State differs")
 	}
 	if obj.Transient != res.Transient {
-		return false
+		return fmt.Errorf("the Transient differs")
 	}
+
 	if obj.CPUs != res.CPUs {
-		return false
+		return fmt.Errorf("the CPUs differ")
 	}
 	// we can't change this property while machine is running!
 	// we do need to return false, so that a new struct gets built,
 	// which will cause at least one Init() & CheckApply() to run.
 	if obj.MaxCPUs != res.MaxCPUs {
-		return false
+		return fmt.Errorf("the MaxCPUs differ")
+	}
+	if obj.HotCPUs != res.HotCPUs {
+		return fmt.Errorf("the HotCPUs differ")
 	}
 	// TODO: can we skip the compare of certain properties such as
 	// Memory because this object (but with different memory) can be
@@ -1084,26 +1200,61 @@ func (obj *VirtRes) Compare(r engine.Res) bool {
 	// We would need to run some sort of "old struct update", to get
 	// the new values, but that's easy to add.
 	if obj.Memory != res.Memory {
-		return false
+		return fmt.Errorf("the Memory differs")
 	}
-	// TODO:
-	//if obj.Boot != res.Boot {
-	//	return false
-	//}
-	//if obj.Disk != res.Disk {
-	//	return false
-	//}
-	//if obj.CDRom != res.CDRom {
-	//	return false
-	//}
-	//if obj.Network != res.Network {
-	//	return false
-	//}
-	//if obj.Filesystem != res.Filesystem {
-	//	return false
-	//}
 
-	return true
+	if obj.OSInit != res.OSInit {
+		return fmt.Errorf("the OSInit differs")
+	}
+	if err := engineUtil.StrListCmp(obj.Boot, res.Boot); err != nil {
+		return errwrap.Wrapf(err, "the Boot differs")
+	}
+
+	if len(obj.Disk) != len(res.Disk) {
+		return fmt.Errorf("the Disk length differs")
+	}
+	for i := range obj.Disk {
+		if err := obj.Disk[i].Cmp(res.Disk[i]); err != nil {
+			return errwrap.Wrapf(err, "the Disk differs")
+		}
+	}
+	if len(obj.CDRom) != len(res.CDRom) {
+		return fmt.Errorf("the CDRom length differs")
+	}
+	for i := range obj.CDRom {
+		if err := obj.CDRom[i].Cmp(res.CDRom[i]); err != nil {
+			return errwrap.Wrapf(err, "the CDRom differs")
+		}
+	}
+	if len(obj.Network) != len(res.Network) {
+		return fmt.Errorf("the Network length differs")
+	}
+	for i := range obj.Network {
+		if err := obj.Network[i].Cmp(res.Network[i]); err != nil {
+			return errwrap.Wrapf(err, "the Network differs")
+		}
+	}
+	if len(obj.Filesystem) != len(res.Filesystem) {
+		return fmt.Errorf("the Filesystem length differs")
+	}
+	for i := range obj.Filesystem {
+		if err := obj.Filesystem[i].Cmp(res.Filesystem[i]); err != nil {
+			return errwrap.Wrapf(err, "the Filesystem differs")
+		}
+	}
+
+	if err := obj.Auth.Cmp(res.Auth); err != nil {
+		return errwrap.Wrapf(err, "the Auth differs")
+	}
+
+	if obj.RestartOnDiverge != res.RestartOnDiverge {
+		return fmt.Errorf("the RestartOnDiverge differs")
+	}
+	if obj.RestartOnRefresh != res.RestartOnRefresh {
+		return fmt.Errorf("the RestartOnRefresh differs")
+	}
+
+	return nil
 }
 
 // VirtUID is the UID struct for FileRes.

engine/reverse.go

@@ -0,0 +1,65 @@
+// Mgmt
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package engine
+
+import (
+	"fmt"
+)
+
+// ReversibleRes is an interface that a resource can implement if it wants to
+// have some resource run when it disappears. A disappearance happens when a
+// resource is defined in one instance of the graph, and is gone in the
+// subsequent one. This is helpful for building robust programs with the engine.
+// Default implementations for most of the methods declared in this interface
+// can be obtained for your resource by anonymously adding the traits.Reversible
+// struct to your resource implementation.
+type ReversibleRes interface {
+	Res
+
+	// ReversibleMeta lets you get or set meta params for the reversible
+	// trait.
+	ReversibleMeta() *ReversibleMeta
+
+	// SetReversibleMeta lets you set all of the meta params for the
+	// reversible trait in a single call.
+	SetReversibleMeta(*ReversibleMeta)
+
+	// Reversed returns the "reverse" or "reciprocal" resource. This is used
+	// to "clean" up after a previously defined resource has been removed.
+	// Interestingly, this returns the core Res interface instead of a
+	// ReversibleRes, because there is no requirement that the reverse of a
+	// Res be the same kind of Res, and the reverse might not be reversible!
+	Reversed() (Res, error)
+}
+
+// ReversibleMeta provides some parameters specific to reversible resources.
+type ReversibleMeta struct {
+	// Disabled specifies that reversing should be disabled for this
+	// resource.
+	Disabled bool
+
+	// TODO: add options here, including whether to reverse edges, etc...
+}
+
+// Cmp compares two ReversibleMeta structs and determines if they're equivalent.
+func (obj *ReversibleMeta) Cmp(rm *ReversibleMeta) error {
+	if obj.Disabled != rm.Disabled {
+		return fmt.Errorf("values for Disabled are different")
+	}
+	return nil
+}

engine/sendrecv.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/traits/autoedge.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -25,7 +25,9 @@ import (
 // methods needed to support autoedges on resources. It may be used as a start
 // point to avoid re-implementing the straightforward methods.
 type Edgeable struct {
-	meta *engine.AutoEdgeMeta
+	// Xmeta is the stored meta. It should be called `meta` but it must be
+	// public so that the `encoding/gob` package can encode it properly.
+	Xmeta *engine.AutoEdgeMeta
 
 	// Bug5819 works around issue https://github.com/golang/go/issues/5819
 	Bug5819 interface{} // XXX: workaround
@@ -33,16 +35,16 @@ type Edgeable struct {
 
 // AutoEdgeMeta lets you get or set meta params for the automatic edges trait.
 func (obj *Edgeable) AutoEdgeMeta() *engine.AutoEdgeMeta {
-	if obj.meta == nil { // set the defaults if previously empty
-		obj.meta = &engine.AutoEdgeMeta{
+	if obj.Xmeta == nil { // set the defaults if previously empty
+		obj.Xmeta = &engine.AutoEdgeMeta{
 			Disabled: false,
 		}
 	}
-	return obj.meta
+	return obj.Xmeta
 }
 
 // SetAutoEdgeMeta lets you set all of the meta params for the automatic edges
 // trait in a single call.
 func (obj *Edgeable) SetAutoEdgeMeta(meta *engine.AutoEdgeMeta) {
-	obj.meta = meta
+	obj.Xmeta = meta
 }

engine/traits/autogroup.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -27,7 +27,9 @@ import (
 // methods needed to support autogrouping on resources. It may be used as a
 // starting point to avoid re-implementing the straightforward methods.
 type Groupable struct {
-	meta *engine.AutoGroupMeta
+	// Xmeta is the stored meta. It should be called `meta` but it must be
+	// public so that the `encoding/gob` package can encode it properly.
+	Xmeta *engine.AutoGroupMeta
 
 	isGrouped bool                  // am i contained within a group?
 	grouped   []engine.GroupableRes // list of any grouped resources
@@ -39,18 +41,18 @@ type Groupable struct {
 // AutoGroupMeta lets you get or set meta params for the automatic grouping
 // trait.
 func (obj *Groupable) AutoGroupMeta() *engine.AutoGroupMeta {
-	if obj.meta == nil { // set the defaults if previously empty
-		obj.meta = &engine.AutoGroupMeta{
+	if obj.Xmeta == nil { // set the defaults if previously empty
+		obj.Xmeta = &engine.AutoGroupMeta{
 			Disabled: false,
 		}
 	}
-	return obj.meta
+	return obj.Xmeta
 }
 
 // SetAutoGroupMeta lets you set all of the meta params for the automatic
 // grouping trait in a single call.
 func (obj *Groupable) SetAutoGroupMeta(meta *engine.AutoGroupMeta) {
-	obj.meta = meta
+	obj.Xmeta = meta
 }
 
 // GroupCmp compares two resources and decides if they're suitable for grouping.

engine/traits/base.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/traits/kind.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -17,11 +17,21 @@
 
 package traits
 
+import (
+	"encoding/gob"
+)
+
+func init() {
+	gob.Register(&Kinded{})
+}
+
 // Kinded contains a general implementation of the properties and methods needed
 // to support the resource kind. It should be used as a starting point to avoid
 // re-implementing the straightforward kind methods.
 type Kinded struct {
-	kind string
+	// Xkind is the stored kind. It should be called `kind` but it must be
+	// public so that the `encoding/gob` package can encode it properly.
+	Xkind string
 
 	// Bug5819 works around issue https://github.com/golang/go/issues/5819
 	Bug5819 interface{} // XXX: workaround
@@ -29,11 +39,11 @@ type Kinded struct {
 
 // Kind returns the string representation for the kind this resource is.
 func (obj *Kinded) Kind() string {
-	return obj.kind
+	return obj.Xkind
 }
 
 // SetKind sets the kind string for this resource. It must only be set by the
 // engine.
 func (obj *Kinded) SetKind(kind string) {
-	obj.kind = kind
+	obj.Xkind = kind
 }

engine/traits/meta.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -25,7 +25,9 @@ import (
 // to support meta parameters. It should be used as a starting point to avoid
 // re-implementing the straightforward meta methods.
 type Meta struct {
-	meta *engine.MetaParams
+	// Xmeta is the stored meta. It should be called `meta` but it must be
+	// public so that the `encoding/gob` package can encode it properly.
+	Xmeta *engine.MetaParams
 
 	// Bug5819 works around issue https://github.com/golang/go/issues/5819
 	Bug5819 interface{} // XXX: workaround
@@ -33,14 +35,14 @@ type Meta struct {
 
 // MetaParams lets you get or set meta params for this trait.
 func (obj *Meta) MetaParams() *engine.MetaParams {
-	if obj.meta == nil { // set the defaults if previously empty
-		obj.meta = engine.DefaultMetaParams.Copy()
+	if obj.Xmeta == nil { // set the defaults if previously empty
+		obj.Xmeta = engine.DefaultMetaParams.Copy()
 	}
-	return obj.meta
+	return obj.Xmeta
 }
 
 // SetMetaParams lets you set all of the meta params for the resource in a
 // single call.
 func (obj *Meta) SetMetaParams(meta *engine.MetaParams) {
-	obj.meta = meta
+	obj.Xmeta = meta
 }

engine/traits/named.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -21,7 +21,9 @@ package traits
 // to support named resources. It should be used as a starting point to avoid
 // re-implementing the straightforward name methods.
 type Named struct {
-	name string
+	// Xname is the stored name. It should be called `name` but it must be
+	// public so that the `encoding/gob` package can encode it properly.
+	Xname string
 
 	// Bug5819 works around issue https://github.com/golang/go/issues/5819
 	Bug5819 interface{} // XXX: workaround
@@ -30,11 +32,11 @@ type Named struct {
 // Name returns the unique name this resource has. It is only unique within its
 // own kind.
 func (obj *Named) Name() string {
-	return obj.name
+	return obj.Xname
 }
 
 // SetName sets the unique name for this resource. It must only be unique within
 // its own kind.
 func (obj *Named) SetName(name string) {
-	obj.name = name
+	obj.Xname = name
 }

engine/traits/refresh.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/traits/reverse.go

@@ -0,0 +1,50 @@
+// Mgmt
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package traits
+
+import (
+	"github.com/purpleidea/mgmt/engine"
+)
+
+// Reversible contains a general implementation with most of the properties and
+// methods needed to support reversing resources. It may be used as a starting
+// point to avoid re-implementing the straightforward methods.
+type Reversible struct {
+	// Xmeta is the stored meta. It should be called `meta` but it must be
+	// public so that the `encoding/gob` package can encode it properly.
+	Xmeta *engine.ReversibleMeta
+
+	// Bug5819 works around issue https://github.com/golang/go/issues/5819
+	Bug5819 interface{} // XXX: workaround
+}
+
+// ReversibleMeta lets you get or set meta params for the reversing trait.
+func (obj *Reversible) ReversibleMeta() *engine.ReversibleMeta {
+	if obj.Xmeta == nil { // set the defaults if previously empty
+		obj.Xmeta = &engine.ReversibleMeta{
+			Disabled: true, // by default we're disabled
+		}
+	}
+	return obj.Xmeta
+}
+
+// SetReversibleMeta lets you set all of the meta params for the reversing trait
+// in a single call.
+func (obj *Reversible) SetReversibleMeta(meta *engine.ReversibleMeta) {
+	obj.Xmeta = meta
+}

engine/traits/sendrecv.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -33,7 +33,7 @@ type Sendable struct {
 
 // Sends returns a struct containing the defaults of the type we send. This
 // needs to be implemented (overridden) by the struct with the Sendable trait to
-// be able to send any values. The public struct field names are the keys used.
+// be able to send any values. The field struct tag names are the keys used.
 func (obj *Sendable) Sends() interface{} {
 	return nil
 }

engine/util.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/util/cmp.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -15,16 +15,23 @@
 // You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package etcd
+package util
 
 import (
-	etcd "github.com/coreos/etcd/clientv3" // "clientv3"
+	"fmt"
 )
 
-// Client provides a simple interface specification for client requests. Both
-// EmbdEtcd and ClientEtcd implement this.
-type Client interface {
-	// TODO: add more method signatures
-	Get(path string, opts ...etcd.OpOption) (map[string]string, error)
-	Txn(ifcmps []etcd.Cmp, thenops, elseops []etcd.Op) (*etcd.TxnResponse, error)
+// StrListCmp compares two lists of strings. If they are not the same length or
+// do not contain identical strings in the same order, then this errors.
+func StrListCmp(x, y []string) error {
+	if len(x) != len(y) {
+		return fmt.Errorf("length differs")
+	}
+	for i := range x {
+		if x[i] != y[i] {
+			return fmt.Errorf("the elements at position %d differed", i)
+		}
+	}
+
+	return nil
 }

engine/util/util.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -30,9 +30,9 @@ import (
 
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/lang/types"
+	"github.com/purpleidea/mgmt/util/errwrap"
 
 	"github.com/godbus/dbus"
-	errwrap "github.com/pkg/errors"
 )
 
 const (
@@ -94,10 +94,22 @@ func B64ToRes(str string) (engine.Res, error) {
 
 // StructTagToFieldName returns a mapping from recommended alias to actual field
 // name. It returns an error if it finds a collision. It uses the `lang` tags.
-func StructTagToFieldName(res engine.Res) (map[string]string, error) {
+// It must be passed a ptr to a struct or it will error.
+func StructTagToFieldName(stptr interface{}) (map[string]string, error) {
 	// TODO: fallback to looking up yaml tags, although harder to parse
 	result := make(map[string]string) // `lang` field tag -> field name
-	st := reflect.TypeOf(res).Elem()  // elem for ptr to res
+	if stptr == nil {
+		return nil, fmt.Errorf("got nil input instead of ptr to struct")
+	}
+	typ := reflect.TypeOf(stptr)
+	if k := typ.Kind(); k != reflect.Ptr { // we only look at *Struct's
+		return nil, fmt.Errorf("input is not a ptr, got: %+v", k)
+	}
+	st := typ.Elem()                         // elem for ptr to struct (dereference the pointer)
+	if k := st.Kind(); k != reflect.Struct { // this should be a struct now
+		return nil, fmt.Errorf("input doesn't point to a struct, got: %+v", k)
+	}
+
 	for i := 0; i < st.NumField(); i++ {
 		field := st.Field(i)
 		name := field.Name
@@ -115,6 +127,62 @@ func StructTagToFieldName(res engine.Res) (map[string]string, error) {
 	return result, nil
 }
 
+// StructFieldCompat returns whether a send struct and key is compatible with a
+// recv struct and key. This inputs must both be a ptr to a string, and a valid
+// key that can be found in the struct tag.
+// TODO: add a bool to decide if *string to string or string to *string is okay.
+func StructFieldCompat(st1 interface{}, key1 string, st2 interface{}, key2 string) error {
+	m1, err := StructTagToFieldName(st1)
+	if err != nil {
+		return err
+	}
+	k1, exists := m1[key1]
+	if !exists {
+		return fmt.Errorf("key not found in send struct")
+	}
+
+	m2, err := StructTagToFieldName(st2)
+	if err != nil {
+		return err
+	}
+	k2, exists := m2[key2]
+	if !exists {
+		return fmt.Errorf("key not found in recv struct")
+	}
+
+	obj1 := reflect.Indirect(reflect.ValueOf(st1))
+	//type1 := obj1.Type()
+	value1 := obj1.FieldByName(k1)
+	kind1 := value1.Kind()
+
+	obj2 := reflect.Indirect(reflect.ValueOf(st2))
+	//type2 := obj2.Type()
+	value2 := obj2.FieldByName(k2)
+	kind2 := value2.Kind()
+
+	if kind1 != kind2 {
+		return fmt.Errorf("kind mismatch between %s and %s", kind1, kind2)
+	}
+
+	if t1, t2 := value1.Type(), value2.Type(); t1 != t2 {
+		return fmt.Errorf("type mismatch between %s and %s", t1, t2)
+	}
+
+	if !value2.CanSet() { // if we can't set, then this is pointless!
+		return fmt.Errorf("can't set")
+	}
+
+	// if we can't interface, we can't compare...
+	if !value1.CanInterface() {
+		return fmt.Errorf("can't interface the send")
+	}
+	if !value2.CanInterface() {
+		return fmt.Errorf("can't interface the recv")
+	}
+
+	return nil
+}
+
 // LowerStructFieldNameToFieldName returns a mapping from the lower case version
 // of each field name to the actual field name. It only returns public fields.
 // It returns an error if it finds a collision.

engine/util/util_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -21,6 +21,7 @@ package util
 
 import (
 	"os/user"
+	"reflect"
 	"strconv"
 	"testing"
 )
@@ -105,3 +106,61 @@ func TestCurrentUserGroupById(t *testing.T) {
 		t.Errorf("gid didn't match current user's: %s vs %s", strconv.Itoa(gid), currentGID)
 	}
 }
+
+func TestStructTagToFieldName0(t *testing.T) {
+	type foo struct {
+		A string `lang:"aaa"`
+		B bool   `lang:"bbb"`
+		C int64  `lang:"ccc"`
+	}
+	f := &foo{ // a ptr!
+		A: "hello",
+		B: true,
+		C: 13,
+	}
+	m, err := StructTagToFieldName(f) // (map[string]string, error)
+	if err != nil {
+		t.Errorf("got error: %+v", err)
+		return
+	}
+	t.Logf("got output: %+v", m)
+	expected := map[string]string{
+		"aaa": "A",
+		"bbb": "B",
+		"ccc": "C",
+	}
+	if !reflect.DeepEqual(m, expected) {
+		t.Errorf("unexpected result")
+		return
+	}
+}
+
+func TestStructTagToFieldName1(t *testing.T) {
+	type foo struct {
+		A string `lang:"aaa"`
+		B bool   `lang:"bbb"`
+		C int64  `lang:"ccc"`
+	}
+	f := foo{ // not a ptr!
+		A: "hello",
+		B: true,
+		C: 13,
+	}
+	m, err := StructTagToFieldName(f) // (map[string]string, error)
+	if err == nil {
+		t.Errorf("expected error, got nil")
+		//return
+	}
+	t.Logf("got output: %+v", m)
+	t.Logf("got error: %+v", err)
+}
+
+func TestStructTagToFieldName2(t *testing.T) {
+	m, err := StructTagToFieldName(nil) // (map[string]string, error)
+	if err == nil {
+		t.Errorf("expected error, got nil")
+		//return
+	}
+	t.Logf("got output: %+v", m)
+	t.Logf("got error: %+v", err)
+}

engine/world.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -18,6 +18,8 @@
 package engine
 
 import (
+	"context"
+
 	"github.com/purpleidea/mgmt/etcd/scheduler"
 )
 
@@ -25,22 +27,26 @@ import (
 // the GAPI to store state and exchange information throughout the cluster. It
 // is the interface each machine uses to communicate with the rest of the world.
 type World interface { // TODO: is there a better name for this interface?
-	ResWatch() chan error
-	ResExport([]Res) error
+	ResWatch(context.Context) (chan error, error)
+	ResExport(context.Context, []Res) error
 	// FIXME: should this method take a "filter" data struct instead of many args?
-	ResCollect(hostnameFilter, kindFilter []string) ([]Res, error)
+	ResCollect(ctx context.Context, hostnameFilter, kindFilter []string) ([]Res, error)
 
-	StrWatch(namespace string) chan error
+	IdealClusterSizeWatch(context.Context) (chan error, error)
+	IdealClusterSizeGet(context.Context) (uint16, error)
+	IdealClusterSizeSet(context.Context, uint16) (bool, error)
+
+	StrWatch(ctx context.Context, namespace string) (chan error, error)
 	StrIsNotExist(error) bool
-	StrGet(namespace string) (string, error)
-	StrSet(namespace, value string) error
-	StrDel(namespace string) error
+	StrGet(ctx context.Context, namespace string) (string, error)
+	StrSet(ctx context.Context, namespace, value string) error
+	StrDel(ctx context.Context, namespace string) error
 
 	// XXX: add the exchange primitives in here directly?
-	StrMapWatch(namespace string) chan error
-	StrMapGet(namespace string) (map[string]string, error)
-	StrMapSet(namespace, value string) error
-	StrMapDel(namespace string) error
+	StrMapWatch(ctx context.Context, namespace string) (chan error, error)
+	StrMapGet(ctx context.Context, namespace string) (map[string]string, error)
+	StrMapSet(ctx context.Context, namespace, value string) error
+	StrMapDel(ctx context.Context, namespace string) error
 
 	Scheduler(namespace string, opts ...scheduler.Option) (*scheduler.Result, error)
 

etcd/callback.go

@@ -0,0 +1,497 @@
+// Mgmt
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package etcd
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	"github.com/purpleidea/mgmt/etcd/interfaces"
+	"github.com/purpleidea/mgmt/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
+
+	etcd "github.com/coreos/etcd/clientv3" // "clientv3"
+	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
+)
+
+// nominateApply applies the changed watcher data onto our local caches.
+func (obj *EmbdEtcd) nominateApply(data *interfaces.WatcherData) error {
+	if data == nil { // ignore empty data
+		return nil
+	}
+
+	// If we tried to lookup the nominated members here (in etcd v3) this
+	// would sometimes block because we would lose the cluster leader once
+	// the current leader calls the MemberAdd API and it steps down trying
+	// to form a two host cluster. Instead, we can look at the event
+	// response data to read the nominated values! Since we only see what
+	// has *changed* in the response data, we have to keep track of the
+	// original state and apply the deltas. This must be idempotent in case
+	// it errors and is called again. If we're retrying and we get a data
+	// format error, it's probably not the end of the world.
+	nominated, err := applyDeltaEvents(data, obj.nominated) // map[hostname]URLs (URLsMap)
+	if err != nil && err != errInconsistentApply {          // allow missing deletes
+		return err // unexpected error, fail
+	}
+	// TODO: do we want to sort this if it becomes a list instead of a map?
+	//sort.Strings(nominated) // deterministic order
+	obj.nominated = nominated
+	return nil
+}
+
+// volunteerApply applies the changed watcher data onto our local caches.
+func (obj *EmbdEtcd) volunteerApply(data *interfaces.WatcherData) error {
+	if data == nil { // ignore empty data
+		return nil
+	}
+	volunteers, err := applyDeltaEvents(data, obj.volunteers) // map[hostname]URLs (URLsMap)
+	if err != nil && err != errInconsistentApply {            // allow missing deletes
+		return err // unexpected error, fail
+	}
+	// TODO: do we want to sort this if it becomes a list instead of a map?
+	//sort.Strings(volunteers) // deterministic order
+	obj.volunteers = volunteers
+	return nil
+}
+
+// endpointApply applies the changed watcher data onto our local caches. In this
+// particular apply function, it also sets our client with the new endpoints.
+func (obj *EmbdEtcd) endpointApply(data *interfaces.WatcherData) error {
+	if data == nil { // ignore empty data
+		return nil
+	}
+	endpoints, err := applyDeltaEvents(data, obj.endpoints) // map[hostname]URLs (URLsMap)
+	if err != nil && err != errInconsistentApply {          // allow missing deletes
+		return err // unexpected error, fail
+	}
+
+	// is the endpoint list different?
+	if err := cmpURLsMap(obj.endpoints, endpoints); err != nil {
+		obj.endpoints = endpoints // set
+		// can happen if a server drops out for example
+		obj.Logf("endpoint list changed to: %+v", endpoints)
+		obj.setEndpoints()
+	}
+	return nil
+}
+
+// nominateCb runs to respond to the nomination list change events.
+// Functionally, it controls the starting and stopping of the server process. If
+// a nominate message is received for this machine, then it means it is already
+// being added to the cluster with member add and the cluster is now waiting for
+// it to start up. When a nominate entry is removed, it's up to this function to
+// run the member remove right before it shuts its server down.
+func (obj *EmbdEtcd) nominateCb(ctx context.Context) error {
+	// Ensure that only one copy of this function is run simultaneously.
+	// This is because we don't want to cause runServer to race with
+	// destroyServer. Let us completely start up before we can cancel it. As
+	// a special case, destroyServer itself can race against itself. I don't
+	// think it's possible for contention on this mutex, but we'll leave it
+	// in for safety.
+	obj.nominatedMutex.Lock()
+	defer obj.nominatedMutex.Unlock()
+	// This ordering mutex is being added for safety, since there is no good
+	// reason for this function and volunteerCb to run simultaneously, and
+	// it might be preventing a race condition that was happening.
+	obj.orderingMutex.Lock()
+	defer obj.orderingMutex.Unlock()
+	if obj.Debug {
+		obj.Logf("nominateCb")
+		defer obj.Logf("nominateCb: done!")
+	}
+
+	// check if i have actually volunteered first of all...
+	if obj.NoServer || len(obj.ServerURLs) == 0 {
+		obj.Logf("inappropriately nominated, rogue or stale server?")
+		// TODO: should we un-nominate ourself?
+		return nil // we've done our job successfully
+	}
+
+	// This can happen when we're shutting down, build the nominated value.
+	if len(obj.nominated) == 0 {
+		obj.Logf("list of nominations is empty")
+		//return nil // don't exit, we might want to shutdown the server
+	} else {
+		obj.Logf("nominated: %v", obj.nominated)
+	}
+
+	// if there are no other peers, we create a new server
+	// TODO: do we need an || len(obj.nominated) == 0 if we're the first?
+	_, exists := obj.nominated[obj.Hostname] // am i nominated?
+	newCluster := len(obj.nominated) == 1 && exists
+	if obj.Debug {
+		obj.Logf("nominateCb: newCluster: %t; exists: %t; obj.server == nil: %t", newCluster, exists, obj.server == nil)
+	}
+
+	// TODO: server start retries should be handled inside of runServer...
+	if obj.serverAction(serverActionStart) { // start
+		// no server is running, but it should be
+		wg := &sync.WaitGroup{}
+		serverReady, ackReady := obj.ServerReady()    // must call ack!
+		serverExited, ackExited := obj.ServerExited() // must call ack!
+
+		var sendError = false
+		var serverErr error
+		obj.Logf("waiting for server...")
+		nominated, err := copyURLsMap(obj.nominated)
+		if err != nil {
+			return err
+		}
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			obj.errExitN = make(chan struct{})
+			defer close(obj.errExitN) // multi-signal for errChan close op
+			// blocks until server exits
+			serverErr = obj.runServer(newCluster, nominated)
+			// in case this exits on its own instead of with destroy
+			defer obj.destroyServer()          // run to reset some values
+			if sendError && serverErr != nil { // exited with an error
+				select {
+				case obj.errChan <- errwrap.Wrapf(serverErr, "runServer errored"):
+				}
+			}
+		}()
+
+		// block until either server is ready or an early exit occurs
+		select {
+		case <-serverReady:
+			// detach from our local return of errors from an early
+			// server exit (pre server ready) and switch to channel
+			sendError = true // gets set before the ackReady() does
+			ackReady()       // must be called
+			ackExited()      // must be called
+			// pass
+
+		case <-serverExited:
+			ackExited() // must be called
+			ackReady()  // must be called
+
+			wg.Wait() // wait for server to finish to get early err
+			return serverErr
+		}
+
+		// Once the server is online, we *must* publish this information
+		// so that (1) others know where to connect to us (2) we provide
+		// an "event" for member add since there is not any event that's
+		// currently built-in to etcd and (3) so we have a key to expire
+		// when we shutdown or crash to give us the member remove event.
+		// please see issue: https://github.com/coreos/etcd/issues/5277
+
+	} else if obj.serverAction(serverActionStop) { // stop?
+		// server is running, but it should not be
+
+		// i have been un-nominated, remove self and shutdown server!
+		// we don't need to do a member remove if i'm the last one...
+		if len(obj.nominated) != 0 { // don't call if nobody left but me!
+			// work around: https://github.com/coreos/etcd/issues/5482
+			// and it might make sense to avoid it if we're the last
+			obj.Logf("member remove: removing self: %d", obj.serverID)
+			resp, err := obj.memberRemove(ctx, obj.serverID)
+			if err != nil {
+				if obj.Debug {
+					obj.Logf("error with member remove: %v", err)
+				}
+				return errwrap.Wrapf(err, "member self remove error")
+			}
+			if resp != nil {
+				obj.Logf("member removed (self): %s (%d)", obj.Hostname, obj.serverID)
+				if err := obj.updateMemberState(resp.Members); err != nil {
+					return err
+				}
+			}
+		}
+
+		// FIXME: if we fail on destroy should we try to run some of the
+		// other cleanup tasks that usually afterwards (below) anyways ?
+		if err := obj.destroyServer(); err != nil { // sync until exited
+			return errwrap.Wrapf(err, "destroyServer errored")
+		}
+
+		// We close with this special sentinel only during destroy/exit.
+		if obj.closing {
+			return interfaces.ErrShutdown
+		}
+	}
+
+	return nil
+}
+
+// volunteerCb runs to respond to the volunteer list change events.
+// Functionally, it controls the nominating and adding of members. It typically
+// nominates a peer so that it knows it will get to be a server, which causes it
+// to start up its server. It also runs the member add operation so that the
+// cluster gets quorum safely. The member remove operation is typically run in
+// the nominateCb of that server when it is asked to shutdown. This occurs when
+// the nominate entry for that server is removed. If a server removes its
+// volunteer entry we must respond by removing the nomination so that it can
+// receive that message and shutdown.
+// FIXME: we might need to respond to member change/disconnect/shutdown events,
+// see: https://github.com/coreos/etcd/issues/5277
+// XXX: Don't allow this function to partially run if it is canceled part way
+// through... We don't want an inconsistent state where we did unnominate, but
+// didn't remove a member...
+// XXX: If the leader changes, do we need to kick the volunteerCb or anything
+// else that might have required a leader and which returned because it did not
+// have one, thus loosing an event?
+func (obj *EmbdEtcd) volunteerCb(ctx context.Context) error {
+	// Ensure that only one copy of this function is run simultaneously.
+	// It's not entirely clear if this can ever happen or if it's needed,
+	// but it's an inexpensive safety check that we can add in for now.
+	obj.volunteerMutex.Lock()
+	defer obj.volunteerMutex.Unlock()
+	// This ordering mutex is being added for safety, since there is no good
+	// reason for this function and nominateCb to run simultaneously, and it
+	// might be preventing a race condition that was happening.
+	obj.orderingMutex.Lock()
+	defer obj.orderingMutex.Unlock()
+	if obj.Debug {
+		obj.Logf("volunteerCb")
+		defer obj.Logf("volunteerCb: done!")
+	}
+
+	// FIXME: are there any situations where we don't want to short circuit
+	// here, such as if i'm the last node?
+	if obj.server == nil {
+		if obj.Debug {
+			obj.Logf("i'm not a server yet...")
+		}
+		return nil // if i'm not a server, i'm not a leader, return
+	}
+
+	// FIXME: Instead of checking this, assume yes, and use the
+	// `WithRequireLeader` wrapper, and just ignore the error from that if
+	// it's wrong... Combined with events that poke this volunteerCb when
+	// the leader changes, we shouldn't miss any events...
+	if isLeader, err := obj.isLeader(ctx); err != nil { // XXX: race!
+		return errwrap.Wrapf(err, "error determining leader")
+	} else if !isLeader {
+		if obj.Debug {
+			obj.Logf("we are not the leader...")
+		}
+		return nil
+	}
+	// i am the leader!
+
+	// Remember that the member* operations return the membership, so this
+	// means we don't need to run an extra memberList in those scenarios...
+	// However, this can get out of sync easily, so ensure that our member
+	// information is very recent.
+	if err := obj.memberStateFromList(ctx); err != nil {
+		return errwrap.Wrapf(err, "error during state sync")
+	}
+	// XXX: If we have any unstarted members here, do we want to reschedule
+	// this volunteerCb in a moment? Or will we get another event anyways?
+
+	// NOTE: There used to be an is_leader check right here...
+	// FIXME: Should we use WithRequireLeader instead? Here? Elsewhere?
+	// https://godoc.org/github.com/coreos/etcd/clientv3#WithRequireLeader
+
+	// FIXME: can this happen, and if so, is it an error or a pass-through?
+	if len(obj.volunteers) == 0 {
+		obj.Logf("list of volunteers is empty")
+		//return fmt.Errorf("volunteer list is empty")
+	} else {
+		obj.Logf("volunteers: %+v", obj.volunteers)
+	}
+
+	// TODO: do we really need to check these errors?
+	m, err := copyURLsMap(obj.membermap) // list of members...
+	if err != nil {
+		return err
+	}
+	v, err := copyURLsMap(obj.volunteers)
+	if err != nil {
+		return err
+	}
+	// Unnominate anyone that unvolunteers, so they can shutdown cleanly...
+	// FIXME: one step at a time... do we trigger subsequent steps somehow?
+	obj.Logf("chooser: (%+v)/(%+v)", m, v)
+	nominate, unnominate, err := obj.Chooser.Choose(m, v)
+	if err != nil {
+		return errwrap.Wrapf(err, "chooser error")
+	}
+
+	// Ensure that we are the *last* in the list if we're unnominating, and
+	// the *first* in the list if we're nominating. This way, we self-remove
+	// last, and we self-add first. This is least likely to hurt quorum.
+	headFn := func(x string) bool {
+		return x != obj.Hostname
+	}
+	tailFn := func(x string) bool {
+		return x == obj.Hostname
+	}
+	nominate = util.PriorityStrSliceSort(nominate, headFn)
+	unnominate = util.PriorityStrSliceSort(unnominate, tailFn)
+	obj.Logf("chooser result(+/-): %+v/%+v", nominate, unnominate)
+	var reterr error
+	leaderCtx := ctx // default ctx to use
+	if RequireLeaderCtx {
+		leaderCtx = etcd.WithRequireLeader(ctx) // FIXME: Is this correct?
+	}
+
+	for i := range nominate {
+		member := nominate[i]
+		peerURLs, exists := obj.volunteers[member] // comma separated list of urls
+		if !exists {
+			// if this happens, do we have an update race?
+			return fmt.Errorf("could not find member `%s` in volunteers map", member)
+		}
+
+		// NOTE: storing peerURLs when they're already in volunteers/ is
+		// redundant, but it seems to be necessary for a sane algorithm.
+		// nominate before we call the API so that members see it first!
+		if err := obj.nominate(leaderCtx, member, peerURLs); err != nil {
+			return errwrap.Wrapf(err, "error nominating: %s", member)
+		}
+		// XXX: can we add a ttl here, because once we nominate someone,
+		// we need to give them up to N seconds to start up after we run
+		// the MemberAdd API because if they don't, in some situations
+		// such as if we're adding the second node to the cluster, then
+		// we've lost quorum until a second member joins! If the TTL
+		// expires, we need to MemberRemove! In this special case, we
+		// need to forcefully remove the second member if we don't add
+		// them, because we'll be in a lack of quorum state and unable
+		// to do anything... As a result, we should always only add ONE
+		// member at a time!
+
+		// XXX: After we memberAdd, can we wait a timeout, and then undo
+		// the add if the member doesn't come up? We'd also need to run
+		// an unnominate too, and mark the node as temporarily failed...
+		obj.Logf("member add: %s: %v", member, peerURLs)
+		resp, err := obj.memberAdd(leaderCtx, peerURLs)
+		if err != nil {
+			// FIXME: On on error this function needs to run again,
+			// because we need to make sure to add the member here!
+			return errwrap.Wrapf(err, "member add error")
+		}
+		if resp != nil { // if we're already the right state, we get nil
+			obj.Logf("member added: %s (%d): %v", member, resp.Member.ID, peerURLs)
+			if err := obj.updateMemberState(resp.Members); err != nil {
+				return err
+			}
+			if resp.Member.Name == "" { // not started instantly ;)
+				obj.addMemberState(member, resp.Member.ID, peerURLs, nil)
+			}
+			// TODO: would this ever happen or be necessary?
+			//if member == obj.Hostname {
+			//	obj.addSelfState()
+			//}
+		}
+	}
+
+	// we must remove them from the members API or it will look like a crash
+	if l := len(unnominate); l > 0 {
+		obj.Logf("unnominated: shutting down %d members...", l)
+	}
+	for i := range unnominate {
+		member := unnominate[i]
+		memberID, exists := obj.memberIDs[member] // map[string]uint64
+		if !exists {
+			// if this happens, do we have an update race?
+			return fmt.Errorf("could not find member `%s` in memberIDs map", member)
+		}
+
+		// start a watcher to know if member was added
+		cancelCtx, cancel := context.WithCancel(leaderCtx)
+		defer cancel()
+		timeout := util.CloseAfter(cancelCtx, SelfRemoveTimeout) // chan closes
+		fn := func(members []*pb.Member) error {
+			for _, m := range members {
+				if m.Name == member || m.ID == memberID {
+					return fmt.Errorf("still present")
+				}
+			}
+
+			return nil // not found!
+		}
+		ch, err := obj.memberChange(cancelCtx, fn, MemberChangeInterval)
+		if err != nil {
+			return errwrap.Wrapf(err, "error watching for change of: %s", member)
+		}
+		if err := obj.nominate(leaderCtx, member, nil); err != nil { // unnominate
+			return errwrap.Wrapf(err, "error unnominating: %s", member)
+		}
+		// Once we issue the above unnominate, that peer will
+		// shutdown, and this might cause us to loose quorum,
+		// therefore, let that member remove itself, and then
+		// double check that it did happen in case delinquent.
+		// TODO: get built-in transactional member Add/Remove
+		// functionality to avoid a separate nominate list...
+
+		// If we're removing ourself, then let the (un)nominate callback
+		// do it. That way it removes itself cleanly on server shutdown.
+		if member == obj.Hostname { // remove in unnominate!
+			cancel()
+			obj.Logf("unnominate: removing self...")
+			continue
+		}
+
+		// cancel remove sleep and unblock early on event...
+		obj.Logf("waiting %s for %s to self remove...", SelfRemoveTimeout.String(), member)
+		select {
+		case <-timeout:
+			// pass
+		case err, ok := <-ch:
+			if ok {
+				select {
+				case <-timeout:
+					// wait until timeout finishes
+				}
+				reterr = errwrap.Append(reterr, err)
+			}
+			// removed quickly!
+		}
+		cancel()
+
+		// In case the removed member doesn't remove itself, do it!
+		resp, err := obj.memberRemove(leaderCtx, memberID)
+		if err != nil {
+			return errwrap.Wrapf(err, "member remove error")
+		}
+		if resp != nil {
+			obj.Logf("member removed (forced): %s (%d)", member, memberID)
+			if err := obj.updateMemberState(resp.Members); err != nil {
+				return err
+			}
+			// Do this I guess, but the TTL will eventually get it.
+			// Remove the other member to avoid client connections.
+			if err := obj.advertise(leaderCtx, member, nil); err != nil {
+				return err
+			}
+		}
+
+		// Remove the member from our lists to avoid blocking future
+		// possible MemberList calls which would try and connect to a
+		// missing member... The lists should get updated from the
+		// member exiting safely if it doesn't crash, but if it did
+		// and/or since it's a race to see if the update event will get
+		// seen before we need the new data, just do it now anyways.
+		// TODO: Is the above comment still true?
+		obj.rmMemberState(member) // proactively delete it
+
+		obj.Logf("member %s (%d) removed successfully!", member, memberID)
+	}
+
+	// NOTE: We could ensure that etcd reconnects here, but we can just wait
+	// for the endpoints callback which should see the state change instead.
+
+	obj.setEndpoints() // sync client with new endpoints
+	return reterr
+}

etcd/chooser/chooser.go

@@ -0,0 +1,98 @@
+// Mgmt
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package chooser
+
+import (
+	"context"
+
+	"github.com/purpleidea/mgmt/etcd/interfaces"
+
+	etcdtypes "github.com/coreos/etcd/pkg/types"
+)
+
+// Data represents the input data that is passed to the chooser.
+type Data struct {
+	// Hostname is the hostname running this chooser instance. It can be
+	// used as a unique key in the cluster.
+	Hostname string // ourself
+
+	Debug bool
+	Logf  func(format string, v ...interface{})
+}
+
+// Chooser represents the interface you must implement if you want to be able to
+// control which cluster members are added and removed. Remember that this can
+// get run from any peer (server) machine in the cluster, and that this may
+// change as different leaders are elected! Do not assume any state will remain
+// between invocations. If you want to maintain hysteresis or state, make sure
+// to synchronize it in etcd.
+type Chooser interface {
+	// Validate validates the chooser implementation to ensure the params
+	// represent a valid instantiation.
+	Validate() error
+
+	// Init initializes the chooser and passes in some useful data and
+	// handles.
+	Init(*Data) error
+
+	// Connect will be called with a client interfaces.Client that you can
+	// use if necessary to store some shared state between instances of this
+	// and watch for external changes. Sharing state between members should
+	// be avoided if possible, and there is no guarantee that your data
+	// won't be deleted in a disaster. There are no backups for this,
+	// regenerate anything you might need. Additionally, this may only be
+	// used inside the Chooser method, since Connect is only called after
+	// Init. This is however very useful for implementing special choosers.
+	// Since some operations can run on connect, it gets a context. If you
+	// cancel this context, then you might expect that Watch could die too.
+	// Both of these should get cancelled if you call Disconnect.
+	Connect(context.Context, interfaces.Client) error // we get given a namespaced client
+
+	// Disconnect tells us to cancel our use of the client interface that we
+	// got from the Connect method. We must not return until we're done.
+	Disconnect() error
+
+	// Watch is called by the engine to allow us to Watch for changes that
+	// might cause us to want to re-evaluate our nomination decision. It
+	// should error if it cannot startup. Once it is running, it should send
+	// a nil error on every event, and an error if things go wrong. When
+	// Disconnect is shutdown, then that should cause this to exit. When
+	// this sends events, Choose will usually eventually get called in
+	// response.
+	Watch() (chan error, error)
+
+	// Choose takes the current peer membership state, and the available
+	// volunteers, and produces a list of who we should add and who should
+	// quit. In general, it's best to only remove one member at a time, in
+	// particular because this will get called iteratively on future events,
+	// and it can remove subsequent members on the next iteration. One
+	// important note: when building a new cluster, we do assume that out of
+	// one available volunteer, and no members, that this first volunteer is
+	// selected. Make sure that any implementations of this function do this
+	// as well, since otherwise the hardcoded initial assumption would be
+	// proven wrong here!
+	// TODO: we could pass in two lists of hostnames instead of the full
+	// URLsMap here, but let's keep it more complicated now in case, and
+	// reduce it down later if needed...
+	// TODO: should we add a step arg here ?
+	Choose(membership, volunteers etcdtypes.URLsMap) (nominees, quitters []string, err error)
+
+	// Close runs some cleanup routines in case there is anything that you'd
+	// like to free after we're done.
+	Close() error
+}

etcd/chooser/dynamicsize.go

@@ -0,0 +1,285 @@
+// Mgmt
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package chooser
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"sync"
+
+	"github.com/purpleidea/mgmt/etcd/interfaces"
+
+	etcd "github.com/coreos/etcd/clientv3"
+	etcdtypes "github.com/coreos/etcd/pkg/types"
+)
+
+// XXX: Test causing cluster shutdowns with:
+// ETCDCTL_API=3 etcdctl --endpoints 127.0.0.1:2379 put /_mgmt/chooser/dynamicsize/idealclustersize 0
+// It is currently broken.
+
+const (
+	// DefaultIdealDynamicSize is the default target ideal dynamic cluster
+	// size used for the initial cluster.
+	DefaultIdealDynamicSize = 5
+
+	// IdealDynamicSizePath is the path key used for the chooser. It usually
+	// gets used with a namespace prefix.
+	IdealDynamicSizePath = "/dynamicsize/idealclustersize"
+)
+
+// DynamicSize is a simple implementation of the Chooser interface. This helps
+// select which machines to add and remove as we elastically grow and shrink our
+// cluster.
+// TODO: think of a better name
+type DynamicSize struct {
+	// IdealClusterSize is the ideal target size for this cluster. If it is
+	// set to zero, then it will use DefaultIdealDynamicSize as the value.
+	IdealClusterSize uint16
+
+	data   *Data // save for later
+	client interfaces.Client
+
+	ctx    context.Context
+	cancel func()
+	wg     *sync.WaitGroup
+}
+
+// Validate validates the struct.
+func (obj *DynamicSize) Validate() error {
+	// TODO: if changed to zero, treat as a cluster shutdown signal
+	if obj.IdealClusterSize < 0 {
+		return fmt.Errorf("must choose a positive IdealClusterSize value")
+	}
+	return nil
+}
+
+// Init accepts some useful data and handles.
+func (obj *DynamicSize) Init(data *Data) error {
+	if data.Hostname == "" {
+		return fmt.Errorf("can't Init with empty Hostname value")
+	}
+	if data.Logf == nil {
+		return fmt.Errorf("no Logf function was specified")
+	}
+
+	if obj.IdealClusterSize == 0 {
+		obj.IdealClusterSize = DefaultIdealDynamicSize
+	}
+
+	obj.data = data
+	obj.wg = &sync.WaitGroup{}
+	return nil
+}
+
+// Close runs some cleanup routines.
+func (obj *DynamicSize) Close() error {
+	return nil
+}
+
+// Connect is called to accept an etcd.KV namespace that we can use.
+func (obj *DynamicSize) Connect(ctx context.Context, client interfaces.Client) error {
+	obj.client = client
+	obj.ctx, obj.cancel = context.WithCancel(ctx)
+	size, err := DynamicSizeGet(obj.ctx, obj.client)
+	if err == interfaces.ErrNotExist || (err == nil && size <= 0) {
+		// unset, set in running cluster
+		changed, err := DynamicSizeSet(obj.ctx, obj.client, obj.IdealClusterSize)
+		if err == nil && changed {
+			obj.data.Logf("set dynamic cluster size to: %d", obj.IdealClusterSize)
+		}
+		return err
+	} else if err == nil && size >= 1 {
+		// unset, get from running cluster (use the valid cluster value)
+		if obj.IdealClusterSize != size {
+			obj.data.Logf("using dynamic cluster size of: %d", size)
+		}
+		obj.IdealClusterSize = size // get from exiting cluster...
+	}
+
+	return err
+}
+
+// Disconnect is called to cancel our use of the etcd.KV connection.
+func (obj *DynamicSize) Disconnect() error {
+	if obj.client != nil { // if connect was not called, don't call this...
+		obj.cancel()
+	}
+	obj.wg.Wait()
+	return nil
+}
+
+// Watch is called to send events anytime we might want to change membership. It
+// is also used to watch for changes so that when we get an event, we know to
+// honour the change in Choose.
+func (obj *DynamicSize) Watch() (chan error, error) {
+	// NOTE: The body of this function is very similar to the logic in the
+	// simple client.Watcher implementation that wraps ComplexWatcher.
+	path := IdealDynamicSizePath
+	cancelCtx, cancel := context.WithCancel(obj.ctx)
+	info, err := obj.client.ComplexWatcher(cancelCtx, path)
+	if err != nil {
+		defer cancel()
+		return nil, err
+	}
+	ch := make(chan error)
+	obj.wg.Add(1) // hook in to global wait group
+	go func() {
+		defer obj.wg.Done()
+		defer close(ch)
+		defer cancel()
+		var data *interfaces.WatcherData
+		var ok bool
+		for {
+			select {
+			case data, ok = <-info.Events: // read
+				if !ok {
+					return
+				}
+			case <-cancelCtx.Done():
+				continue // wait for ch closure, but don't block
+			}
+
+			size := obj.IdealClusterSize
+			for _, event := range data.Events { // apply each event
+				if event.Type != etcd.EventTypePut {
+					continue
+				}
+				key := string(event.Kv.Key)
+				key = key[len(data.Path):] // remove path prefix
+				val := string(event.Kv.Value)
+				if val == "" {
+					continue // ignore empty values
+				}
+				i, err := strconv.Atoi(val)
+				if err != nil {
+					continue // ignore bad values
+				}
+				size = uint16(i) // save
+			}
+			if size == obj.IdealClusterSize {
+				continue // no change
+			}
+			// set before sending the signal
+			obj.IdealClusterSize = size
+
+			if size == 0 { // zero means shutdown
+				obj.data.Logf("impending cluster shutdown...")
+			} else {
+				obj.data.Logf("got new dynamic cluster size of: %d", size)
+			}
+
+			select {
+			case ch <- data.Err: // send (might be nil!)
+			case <-cancelCtx.Done():
+				continue // wait for ch closure, but don't block
+			}
+		}
+	}()
+	return ch, nil
+}
+
+// Choose accepts a list of current membership, and a list of volunteers. From
+// that we can decide who we should add and remove. We return a list of those
+// nominated, and unnominated users respectively.
+func (obj *DynamicSize) Choose(membership, volunteers etcdtypes.URLsMap) ([]string, []string, error) {
+	// Possible nominees include anyone that has volunteered, but that
+	// isn't a member.
+	if obj.data.Debug {
+		obj.data.Logf("goal: %d members", obj.IdealClusterSize)
+	}
+	nominees := []string{}
+	for hostname := range volunteers {
+		if _, exists := membership[hostname]; !exists {
+			nominees = append(nominees, hostname)
+		}
+	}
+
+	// Possible quitters include anyone that is a member, but that is not a
+	// volunteer. (They must have unvolunteered.)
+	quitters := []string{}
+	for hostname := range membership {
+		if _, exists := volunteers[hostname]; !exists {
+			quitters = append(quitters, hostname)
+		}
+	}
+
+	// What we want to know...
+	nominated := []string{}
+	unnominated := []string{}
+
+	// We should always only add ONE member at a time!
+	// TODO: is it okay to remove multiple members at the same time?
+	if len(nominees) > 0 && len(membership)-len(quitters) < int(obj.IdealClusterSize) {
+		//unnominated = []string{} // only do one operation at a time
+		nominated = []string{nominees[0]} // FIXME: use a better picker algorithm
+
+	} else if len(quitters) == 0 && len(membership) > int(obj.IdealClusterSize) { // too many members
+		//nominated = []string{} // only do one operation at a time
+		for kicked := range membership {
+			// don't kick ourself unless we are the only one left...
+			if kicked != obj.data.Hostname || (obj.IdealClusterSize == 0 && len(membership) == 1) {
+				unnominated = []string{kicked} // FIXME: use a better picker algorithm
+				break
+			}
+		}
+	} else if len(quitters) > 0 { // must do these before new unvolunteers
+		unnominated = quitters // get rid of the quitters
+	}
+
+	return nominated, unnominated, nil // perform these changes
+}
+
+// DynamicSizeGet gets the currently set dynamic size set in the cluster.
+func DynamicSizeGet(ctx context.Context, client interfaces.Client) (uint16, error) {
+	key := IdealDynamicSizePath
+	m, err := client.Get(ctx, key) // (map[string]string, error)
+	if err != nil {
+		return 0, err
+	}
+	val, exists := m[IdealDynamicSizePath]
+	if !exists {
+		return 0, interfaces.ErrNotExist
+	}
+	i, err := strconv.Atoi(val)
+	if err != nil {
+		return 0, fmt.Errorf("bad value")
+	}
+	return uint16(i), nil
+}
+
+// DynamicSizeSet sets the dynamic size in the cluster. It returns true if it
+// changed or set the value.
+func DynamicSizeSet(ctx context.Context, client interfaces.Client, size uint16) (bool, error) {
+	key := IdealDynamicSizePath
+	val := strconv.FormatUint(uint64(size), 10) // fmt.Sprintf("%d", size)
+
+	ifCmps := []etcd.Cmp{
+		etcd.Compare(etcd.Value(key), "=", val), // desired state
+	}
+	elseOps := []etcd.Op{etcd.OpPut(key, val)}
+
+	resp, err := client.Txn(ctx, ifCmps, nil, elseOps)
+	if err != nil {
+		return false, err
+	}
+	// succeeded is set to true if the compare evaluated to true
+	changed := !resp.Succeeded
+
+	return changed, err
+}

etcd/client.go

@@ -1,94 +0,0 @@
-// Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
-// Written by James Shubin <james@shubin.ca> and the project contributors
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package etcd
-
-import (
-	"time"
-
-	etcd "github.com/coreos/etcd/clientv3" // "clientv3"
-	errwrap "github.com/pkg/errors"
-	context "golang.org/x/net/context"
-)
-
-// ClientEtcd provides a simple etcd client for deploy and status operations.
-type ClientEtcd struct {
-	Seeds []string // list of endpoints to try to connect
-
-	client *etcd.Client
-}
-
-// GetClient returns a handle to the raw etcd client object.
-func (obj *ClientEtcd) GetClient() *etcd.Client {
-	return obj.client
-}
-
-// GetConfig returns the config struct to be used for the etcd client connect.
-func (obj *ClientEtcd) GetConfig() etcd.Config {
-	cfg := etcd.Config{
-		Endpoints: obj.Seeds,
-		// RetryDialer chooses the next endpoint to use
-		// it comes with a default dialer if unspecified
-		DialTimeout: 5 * time.Second,
-	}
-	return cfg
-}
-
-// Connect connects the client to a server, and then builds the *API structs.
-// If reconnect is true, it will force a reconnect with new config endpoints.
-func (obj *ClientEtcd) Connect() error {
-	if obj.client != nil { // memoize
-		return nil
-	}
-
-	var err error
-	cfg := obj.GetConfig()
-	obj.client, err = etcd.New(cfg) // connect!
-	if err != nil {
-		return errwrap.Wrapf(err, "client connect error")
-	}
-	return nil
-}
-
-// Destroy cleans up the entire etcd client connection.
-func (obj *ClientEtcd) Destroy() error {
-	err := obj.client.Close()
-	//obj.wg.Wait()
-	return err
-}
-
-// Get runs a get on the client connection. This has the same signature as our
-// EmbdEtcd Get function.
-func (obj *ClientEtcd) Get(path string, opts ...etcd.OpOption) (map[string]string, error) {
-	resp, err := obj.client.Get(context.TODO(), path, opts...)
-	if err != nil || resp == nil {
-		return nil, err
-	}
-
-	// TODO: write a resp.ToMap() function on https://godoc.org/github.com/coreos/etcd/etcdserver/etcdserverpb#RangeResponse
-	result := make(map[string]string)
-	for _, x := range resp.Kvs {
-		result[string(x.Key)] = string(x.Value)
-	}
-	return result, nil
-}
-
-// Txn runs a transaction on the client connection. This has the same signature
-// as our EmbdEtcd Txn function.
-func (obj *ClientEtcd) Txn(ifcmps []etcd.Cmp, thenops, elseops []etcd.Op) (*etcd.TxnResponse, error) {
-	return obj.client.KV.Txn(context.TODO()).If(ifcmps...).Then(thenops...).Else(elseops...).Commit()
-}