test: Add gometalinter to our test suite

Add a bunch of new linters to our tests! We can uncomment each sub
linter as we fix up the few remaining issues.

.ackrc

@@ -1,2 +1,3 @@
 --ignore-dir=old/
 --ignore-dir=tmp/
+--ignore-dir=vendor/

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,9 @@
+## Versions:
+
+* mgmt version (eg: `mgmt --version`):
+
+* operating system/distribution (eg: `uname -a`):
+
+* golang version (eg: `go version`):
+
+## Description:

.gitignore

@@ -1,10 +1,11 @@
+.idea/
 .omv/
 .ssh/
 .vagrant/
-mgmt-documentation.pdf
 old/
 tmp/
 *_stringer.go
 mgmt
 mgmt.static
+mgmt.iml
 rpmbuild/

.gitmodules

@@ -10,3 +10,9 @@
 [submodule "vendor/gopkg.in/fsnotify.v1"]
 	path = vendor/gopkg.in/fsnotify.v1
 	url = https://gopkg.in/fsnotify.v1
+[submodule "vendor/github.com/purpleidea/go-systemd"]
+	path = vendor/github.com/purpleidea/go-systemd
+	url = https://github.com/purpleidea/go-systemd
+[submodule "vendor/honnef.co/go/augeas"]
+	path = vendor/honnef.co/go/augeas
+	url = https://github.com/dominikh/go-augeas/

.travis.yml

@@ -1,17 +1,22 @@
 language: go
 go:
-  - 1.6
-  - 1.7
+  - 1.6.x
+  - 1.7.x
+  - 1.8.x
   - tip
-sudo: false
-before_install: 'git fetch --unshallow'
+go_import_path: github.com/purpleidea/mgmt
+sudo: true
+dist: trusty
+before_install:
+  - sudo apt update
+  - git fetch --unshallow
 install: 'make deps'
 script: 'make test'
 matrix:
   fast_finish: true
   allow_failures:
     - go: tip
-    - go: 1.7
+    - go: 1.8.x
 notifications:
   irc:
     channels:
@@ -24,4 +29,7 @@ notifications:
     use_notice: false
     skip_join: false
   email:
-    - travis-ci@shubin.ca
+    recipients:
+      - travis-ci@shubin.ca
+    on_failure: change
+    on_success: change

AUTHORS

@@ -4,5 +4,7 @@ If you appreciate the work of one of the contributors, thank them a beverage!
 For a more exhaustive list please run: git log --format='%aN' | sort -u
 This list is sorted alphabetically by first name.
 
+Felix Frank
 James Shubin
+Julien Pivotto
 Paul Morgan

COPYRIGHT

@@ -1,5 +1,5 @@
 Mgmt
-Copyright (C) 2013-2016+ James Shubin and the project contributors
+Copyright (C) 2013-2017+ James Shubin and the project contributors
 Written by James Shubin <james@shubin.ca> and the project contributors
 
 This program is free software: you can redistribute it and/or modify

Makefile

@@ -1,5 +1,5 @@
 # Mgmt
-# Copyright (C) 2013-2016+ James Shubin and the project contributors
+# Copyright (C) 2013-2017+ James Shubin and the project contributors
 # Written by James Shubin <james@shubin.ca> and the project contributors
 #
 # This program is free software: you can redistribute it and/or modify
@@ -37,6 +37,11 @@ RPM = rpmbuild/RPMS/$(PROGRAM)-$(VERSION)-$(RELEASE).$(ARCH).rpm
 USERNAME := $(shell cat ~/.config/copr 2>/dev/null | grep username | awk -F '=' '{print $$2}' | tr -d ' ')
 SERVER = 'dl.fedoraproject.org'
 REMOTE_PATH = 'pub/alt/$(USERNAME)/$(PROGRAM)'
+ifneq ($(GOTAGS),)
+    BUILD_FLAGS = -tags '$(GOTAGS)'
+endif
+
+default: build
 
 #
 #	art
@@ -105,9 +110,9 @@ $(PROGRAM): main.go
 	@echo "Building: $(PROGRAM), version: $(SVERSION)..."
 ifneq ($(OLDGOLANG),)
 	@# avoid equals sign in old golang versions eg in: -X foo=bar
-	time go build -ldflags "-X main.program $(PROGRAM) -X main.version $(SVERSION)" -o $(PROGRAM);
+	time go build -ldflags "-X main.program $(PROGRAM) -X main.version $(SVERSION)" -o $(PROGRAM) $(BUILD_FLAGS);
 else
-	time go build -ldflags "-X main.program=$(PROGRAM) -X main.version=$(SVERSION)" -o $(PROGRAM);
+	time go build -i -ldflags "-X main.program=$(PROGRAM) -X main.version=$(SVERSION)" -o $(PROGRAM) $(BUILD_FLAGS);
 endif
 
 $(PROGRAM).static: main.go
@@ -115,9 +120,9 @@ $(PROGRAM).static: main.go
 	go generate
 ifneq ($(OLDGOLANG),)
 	@# avoid equals sign in old golang versions eg in: -X foo=bar
-	go build -a -installsuffix cgo -tags netgo -ldflags '-extldflags "-static" -X main.program $(PROGRAM) -X main.version $(SVERSION)' -o $(PROGRAM).static;
+	go build -a -installsuffix cgo -tags netgo -ldflags '-extldflags "-static" -X main.program $(PROGRAM) -X main.version $(SVERSION)' -o $(PROGRAM).static $(BUILD_FLAGS);
 else
-	go build -a -installsuffix cgo -tags netgo -ldflags '-extldflags "-static" -X main.program=$(PROGRAM) -X main.version=$(SVERSION)' -o $(PROGRAM).static;
+	go build -a -installsuffix cgo -tags netgo -ldflags '-extldflags "-static" -X main.program=$(PROGRAM) -X main.version=$(SVERSION)' -o $(PROGRAM).static $(BUILD_FLAGS);
 endif
 
 clean:
@@ -138,8 +143,8 @@ format: gofmt yamlfmt
 
 docs: $(PROGRAM)-documentation.pdf
 
-$(PROGRAM)-documentation.pdf: DOCUMENTATION.md
-	pandoc DOCUMENTATION.md -o '$(PROGRAM)-documentation.pdf'
+$(PROGRAM)-documentation.pdf: docs/documentation.md
+	pandoc docs/documentation.md -o docs/'$(PROGRAM)-documentation.pdf'
 
 #
 #	build aliases
@@ -183,8 +188,8 @@ $(SRPM): $(SPEC) $(SOURCE)
 #
 $(SPEC): rpmbuild/ spec.in
 	@echo Running templater...
-	#cat spec.in > $(SPEC)
-	sed -e s/__PROGRAM__/$(PROGRAM)/ -e s/__VERSION__/$(VERSION)/ -e s/__RELEASE__/$(RELEASE)/ < spec.in > $(SPEC)
+	cat spec.in > $(SPEC)
+	sed -e s/__PROGRAM__/$(PROGRAM)/g -e s/__VERSION__/$(VERSION)/g -e s/__RELEASE__/$(RELEASE)/g < spec.in > $(SPEC)
 	# append a changelog to the .spec file
 	git log --format="* %cd %aN <%aE>%n- (%h) %s%d%n" --date=local | sed -r 's/[0-9]+:[0-9]+:[0-9]+ //' >> $(SPEC)
 

README.md

@@ -2,18 +2,20 @@
 
 [![mgmt!](art/mgmt.png)](art/)
 
-[![Go Report Card](https://goreportcard.com/badge/github.com/purpleidea/mgmt)](https://goreportcard.com/report/github.com/purpleidea/mgmt)
-[![Build Status](https://secure.travis-ci.org/purpleidea/mgmt.png?branch=master)](http://travis-ci.org/purpleidea/mgmt)
-[![Documentation](https://img.shields.io/docs/markdown.png)](DOCUMENTATION.md)
-[![GoDoc](https://godoc.org/github.com/purpleidea/mgmt?status.svg)](https://godoc.org/github.com/purpleidea/mgmt)
-[![IRC](https://img.shields.io/irc/%23mgmtconfig.png)](https://webchat.freenode.net/?channels=#mgmtconfig)
-[![Jenkins](https://img.shields.io/jenkins/status.png)](https://ci.centos.org/job/purpleidea-mgmt/)
-[![COPR](https://img.shields.io/copr/builds.png)](https://copr.fedoraproject.org/coprs/purpleidea/mgmt/)
-[![arch](https://img.shields.io/arch/aur.png)](https://aur.archlinux.org/packages/mgmt/)
+[![Go Report Card](https://goreportcard.com/badge/github.com/purpleidea/mgmt?style=flat-square)](https://goreportcard.com/report/github.com/purpleidea/mgmt)
+[![Build Status](https://img.shields.io/travis/purpleidea/mgmt/master.svg?style=flat-square)](http://travis-ci.org/purpleidea/mgmt)
+[![GoDoc](https://img.shields.io/badge/godoc-reference-5272B4.svg?style=flat-square)](https://godoc.org/github.com/purpleidea/mgmt)
+[![IRC](https://img.shields.io/badge/irc-%23mgmtconfig-brightgreen.svg?style=flat-square)](https://webchat.freenode.net/?channels=#mgmtconfig)
+[![Jenkins](https://img.shields.io/badge/jenkins-status-brightgreen.svg?style=flat-square)](https://ci.centos.org/job/purpleidea-mgmt/)
 
 ## Community:
-Come join us on IRC in [#mgmtconfig](https://webchat.freenode.net/?channels=#mgmtconfig) on Freenode!
-You may like the [#mgmtconfig](https://twitter.com/hashtag/mgmtconfig) hashtag if you're on [Twitter](https://twitter.com/#!/purpleidea).
+Come join us in the `mgmt` community!
+
+| Medium | Link |
+|---|---|
+| IRC | [#mgmtconfig](https://webchat.freenode.net/?channels=#mgmtconfig) on Freenode |
+| Twitter | [@mgmtconfig](https://twitter.com/mgmtconfig) & [#mgmtconfig](https://twitter.com/hashtag/mgmtconfig) |
+| Mailing list | [mgmtconfig-list@redhat.com](https://www.redhat.com/mailman/listinfo/mgmtconfig-list) |
 
 ## Status:
 Mgmt is a fairly new project.
@@ -21,24 +23,21 @@ We're working towards being minimally useful for production environments.
 We aren't feature complete for what we'd consider a 1.x release yet.
 With your help you'll be able to influence our design and get us there sooner!
 
-## Questions:
-Please join the [#mgmtconfig](https://webchat.freenode.net/?channels=#mgmtconfig) IRC community!
-If you have a well phrased question that might benefit others, consider asking it by sending a patch to the documentation [FAQ](https://github.com/purpleidea/mgmt/blob/master/DOCUMENTATION.md#usage-and-frequently-asked-questions) section. I'll merge your question, and a patch with the answer!
-
-## Quick start:
-* Make sure you have golang version 1.6 or greater installed.
-* Clone the repository recursively, eg: `git clone --recursive https://github.com/purpleidea/mgmt/`.
-* Get the remaining golang dependencies on your own, or run `make deps` if you're comfortable with how we install them.
-* Run `make build` to get a freshly built `mgmt` binary.
-* Run `time ./mgmt run --file examples/graph0.yaml --converged-timeout=5 --tmp-prefix` to try out a very simple example!
-* To run continuously in the default mode of operation, omit the `--converged-timeout` option.
-* Have fun hacking on our future technology!
-
-## Examples:
-Please look in the [examples/](examples/) folder for more examples!
-
 ## Documentation:
-Please see: [DOCUMENTATION.md](DOCUMENTATION.md) or [PDF](https://pdfdoc-purpleidea.rhcloud.com/pdf/https://github.com/purpleidea/mgmt/blob/master/DOCUMENTATION.md).
+Please read, enjoy and help improve our documentation!
+
+| Documentation | Additional Notes |
+|---|---|
+| [general documentation](docs/documentation.md) | for everyone |
+| [quick start guide](docs/quick-start-guide.md) | for mgmt developers |
+| [resource guide](docs/resource-guide.md) | for mgmt developers |
+| [godoc API reference](https://godoc.org/github.com/purpleidea/mgmt) | for mgmt developers |
+| [prometheus guide](docs/prometheus.md) | for everyone |
+| [puppet guide](docs/puppet-guide.md) | for puppet sysadmins |
+
+## Questions:
+Please ask in the [community](#community)!
+If you have a well phrased question that might benefit others, consider asking it by sending a patch to the documentation [FAQ](https://github.com/purpleidea/mgmt/blob/master/docs/documentation.md#usage-and-frequently-asked-questions) section. I'll merge your question, and a patch with the answer!
 
 ## Roadmap:
 Please see: [TODO.md](TODO.md) for a list of upcoming work and TODO items.
@@ -50,46 +49,37 @@ Please set the `DEBUG` constant in [main.go](https://github.com/purpleidea/mgmt/
 Bonus points if you provide a [shell](https://github.com/purpleidea/mgmt/tree/master/test/shell) or [OMV](https://github.com/purpleidea/mgmt/tree/master/test/omv) reproducible test case.
 Feel free to read my article on [debugging golang programs](https://ttboj.wordpress.com/2016/02/15/debugging-golang-programs/).
 
-## Dependencies:
-* golang 1.6 or higher (required, available in most distros)
-* golang libraries (required, available with `go get`)
-
-        go get github.com/coreos/etcd/client
-        go get gopkg.in/yaml.v2
-        go get gopkg.in/fsnotify.v1
-        go get github.com/urfave/cli
-        go get github.com/coreos/go-systemd/dbus
-        go get github.com/coreos/go-systemd/util
-        go get github.com/coreos/pkg/capnslog
-
-* stringer (required for building), available as a package on some platforms, otherwise via `go get`
-
-        go get golang.org/x/tools/cmd/stringer
-
-* pandoc (optional, for building a pdf of the documentation)
-* graphviz (optional, for building a visual representation of the graph)
-
 ## Patches:
 We'd love to have your patches! Please send them by email, or as a pull request.
 
 ## On the web:
-* James Shubin; blog: [Next generation configuration mgmt](https://ttboj.wordpress.com/2016/01/18/next-generation-configuration-mgmt/)
-* James Shubin; video: [Introductory recording from DevConf.cz 2016](https://www.youtube.com/watch?v=GVhpPF0j-iE&html5=1)
-* James Shubin; video: [Introductory recording from CfgMgmtCamp.eu 2016](https://www.youtube.com/watch?v=fNeooSiIRnA&html5=1)
-* Julian Dunn; video: [On mgmt at CfgMgmtCamp.eu 2016](https://www.youtube.com/watch?v=kfF9IATUask&t=1949&html5=1)
-* Walter Heck; slides: [On mgmt at CfgMgmtCamp.eu 2016](http://www.slideshare.net/olindata/configuration-management-time-for-a-4th-generation/3)
-* Marco Marongiu; blog: [On mgmt](http://syslog.me/2016/02/15/leap-or-die/)
-* Felix Frank; blog: [From Catalog To Mgmt (on puppet to mgmt "transpiling")](https://ffrank.github.io/features/2016/02/18/from-catalog-to-mgmt/)
-* James Shubin; blog: [Automatic edges in mgmt (...and the pkg resource)](https://ttboj.wordpress.com/2016/03/14/automatic-edges-in-mgmt/)
-* James Shubin; blog: [Automatic grouping in mgmt](https://ttboj.wordpress.com/2016/03/30/automatic-grouping-in-mgmt/)
-* John Arundel; tweet: [“Puppet’s days are numbered.”](https://twitter.com/bitfield/status/732157519142002688)
-* Felix Frank; blog: [Puppet, Meet Mgmt (on puppet to mgmt internals)](https://ffrank.github.io/features/2016/06/12/puppet,-meet-mgmt/)
-* Felix Frank; blog: [Puppet Powered Mgmt (puppet to mgmt tl;dr)](https://ffrank.github.io/features/2016/06/19/puppet-powered-mgmt/)
-* James Shubin; blog: [Automatic clustering in mgmt](https://ttboj.wordpress.com/2016/06/20/automatic-clustering-in-mgmt/)
-* James Shubin; video: [Recording from CoreOSFest 2016](https://www.youtube.com/watch?v=KVmDCUA42wc&html5=1)
-* James Shubin; video: [Recording from DebConf16](http://meetings-archive.debian.net/pub/debian-meetings/2016/debconf16/Next_Generation_Config_Mgmt.webm) ([Slides](https://annex.debconf.org//debconf-share/debconf16/slides/15-next-generation-config-mgmt.pdf))
-* Felix Frank; blog: [Edging It All In (puppet and mgmt edges)](https://ffrank.github.io/features/2016/07/12/edging-it-all-in/)
-* Felix Frank; blog: [Translating All The Things (puppet to mgmt translation warnings)](https://ffrank.github.io/features/2016/08/19/translating-all-the-things/)
+| Author | Format | Subject |
+|---|---|---|
+| James Shubin | blog | [Next generation configuration mgmt](https://ttboj.wordpress.com/2016/01/18/next-generation-configuration-mgmt/) |
+| James Shubin | video | [Introductory recording from DevConf.cz 2016](https://www.youtube.com/watch?v=GVhpPF0j-iE&html5=1) |
+| James Shubin | video | [Introductory recording from CfgMgmtCamp.eu 2016](https://www.youtube.com/watch?v=fNeooSiIRnA&html5=1) |
+| Julian Dunn | video | [On mgmt at CfgMgmtCamp.eu 2016](https://www.youtube.com/watch?v=kfF9IATUask&t=1949&html5=1) |
+| Walter Heck | slides | [On mgmt at CfgMgmtCamp.eu 2016](http://www.slideshare.net/olindata/configuration-management-time-for-a-4th-generation/3) |
+| Marco Marongiu | blog | [On mgmt](http://syslog.me/2016/02/15/leap-or-die/) |
+| Felix Frank | blog | [From Catalog To Mgmt (on puppet to mgmt "transpiling")](https://ffrank.github.io/features/2016/02/18/from-catalog-to-mgmt/) |
+| James Shubin | blog | [Automatic edges in mgmt (...and the pkg resource)](https://ttboj.wordpress.com/2016/03/14/automatic-edges-in-mgmt/) |
+| James Shubin | blog | [Automatic grouping in mgmt](https://ttboj.wordpress.com/2016/03/30/automatic-grouping-in-mgmt/) |
+| John Arundel | tweet | [“Puppet’s days are numbered.”](https://twitter.com/bitfield/status/732157519142002688) |
+| Felix Frank | blog | [Puppet, Meet Mgmt (on puppet to mgmt internals)](https://ffrank.github.io/features/2016/06/12/puppet,-meet-mgmt/) |
+| Felix Frank | blog | [Puppet Powered Mgmt (puppet to mgmt tl;dr)](https://ffrank.github.io/features/2016/06/19/puppet-powered-mgmt/) |
+| James Shubin | blog | [Automatic clustering in mgmt](https://ttboj.wordpress.com/2016/06/20/automatic-clustering-in-mgmt/) |
+| James Shubin | video | [Recording from CoreOSFest 2016](https://www.youtube.com/watch?v=KVmDCUA42wc&html5=1) |
+| James Shubin | video | [Recording from DebConf16](http://meetings-archive.debian.net/pub/debian-meetings/2016/debconf16/Next_Generation_Config_Mgmt.webm) ([Slides](https://annex.debconf.org//debconf-share/debconf16/slides/15-next-generation-config-mgmt.pdf)) |
+| Felix Frank | blog | [Edging It All In (puppet and mgmt edges)](https://ffrank.github.io/features/2016/07/12/edging-it-all-in/) |
+| Felix Frank | blog | [Translating All The Things (puppet to mgmt translation warnings)](https://ffrank.github.io/features/2016/08/19/translating-all-the-things/) |
+| James Shubin | video | [Recording from systemd.conf 2016](https://www.youtube.com/watch?v=jB992Zb3nH0&html5=1) |
+| James Shubin | blog | [Remote execution in mgmt](https://ttboj.wordpress.com/2016/10/07/remote-execution-in-mgmt/) |
+| James Shubin | video | [Recording from High Load Strategy 2016](https://vimeo.com/191493409) |
+| James Shubin | video | [Recording from NLUUG 2016](https://www.youtube.com/watch?v=MmpwOQAb_SE&html5=1) |
+| James Shubin | blog | [Send/Recv in mgmt](https://ttboj.wordpress.com/2016/12/07/sendrecv-in-mgmt/) |
+| James Shubin | blog | [Metaparameters in mgmt](https://ttboj.wordpress.com/2017/03/01/metaparameters-in-mgmt/) |
+| James Shubin | video | [Recording from Incontro DevOps 2017](https://vimeo.com/212241877) |
+| Yves Brissaud | blog | [mgmt aux HumanTalks Grenoble (french)](http://log.winsos.net/2017/04/12/mgmt-aux-human-talks-grenoble.html) |
 
 ##
 

TODO.md

@@ -1,16 +1,16 @@
 # TODO
 If you're looking for something to do, look here!
 Let us know if you're working on one of the items.
+If you'd like something to work on, ping @purpleidea and I'll create an issue
+tailored especially for you! Just let me know your approximate golang skill
+level and how many hours you'd like to spend on the patch.
 
 ## Package resource
 - [ ] getfiles support on debian [bug](https://github.com/hughsie/PackageKit/issues/118)
 - [ ] directory info on fedora [bug](https://github.com/hughsie/PackageKit/issues/117)
 - [ ] dnf blocker [bug](https://github.com/hughsie/PackageKit/issues/110)
-- [ ] install signal blocker [bug](https://github.com/hughsie/PackageKit/issues/109)
 
-## File resource [bug](https://github.com/purpleidea/mgmt/issues/13) [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
-- [ ] chown/chmod support [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
-- [ ] user/group support [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
+## File resource [bug](https://github.com/purpleidea/mgmt/issues/64) [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
 - [ ] recurse limit support [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
 - [ ] fanotify support [bug](https://github.com/go-fsnotify/fsnotify/issues/114)
 
@@ -21,7 +21,6 @@ Let us know if you're working on one of the items.
 - [ ] base resource improvements
 
 ## Timer resource
-- [ ] reset on recompile
 - [ ] increment algorithm (linear, exponential, etc...) [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
 
 ## User/Group resource
@@ -29,7 +28,7 @@ Let us know if you're working on one of the items.
 - [ ] automatic edges to file resource [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
 
 ## Virt (libvirt) resource
-- [ ] base resource [bug](https://github.com/purpleidea/mgmt/issues/25)
+- [ ] base resource improvements [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
 
 ## Net (systemd-networkd) resource
 - [ ] base resource
@@ -44,7 +43,7 @@ Let us know if you're working on one of the items.
 - [ ] base resource [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
 
 ## Http resource
-- [ ] base resource
+- [ ] base resource [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
 
 ## Etcd improvements
 - [ ] fix embedded etcd master race
@@ -52,6 +51,9 @@ Let us know if you're working on one of the items.
 ## Torrent/dht file transfer
 - [ ] base plumbing
 
+## GPG/Auth improvements
+- [ ] base plumbing
+
 ## Language improvements
 - [ ] language design
 - [ ] lexer/parser

Vagrantfile

@@ -0,0 +1,47 @@
+Vagrant.configure(2) do |config|
+	config.ssh.forward_agent = true
+	config.ssh.username = 'vagrant'
+	config.vm.network "private_network", ip: "192.168.219.2"
+
+	config.vm.synced_folder ".", "/vagrant", disabled: true
+
+	config.vm.define "mgmt-dev" do |instance|
+		instance.vm.box = "fedora/24-cloud-base"
+	end
+
+	config.vm.provider "virtualbox" do |v|
+		v.memory = 1536
+		v.cpus = 2
+	end
+
+	config.vm.provision "file", source: "vagrant/motd", destination: ".motd"
+	config.vm.provision "shell", inline: "cp ~vagrant/.motd /etc/motd"
+
+	config.vm.provision "file", source: "vagrant/mgmt.bashrc", destination: ".mgmt.bashrc"
+	config.vm.provision "file", source: "~/.gitconfig", destination: ".gitconfig"
+
+	# copied from make-deps.sh (with added git)
+	config.vm.provision "shell", inline: "dnf install -y libvirt-devel golang golang-googlecode-tools-stringer hg git make"
+
+	# set up packagekit
+	config.vm.provision "shell" do |shell|
+		shell.inline = <<-SCRIPT
+			dnf install -y PackageKit
+			systemctl enable packagekit
+			systemctl start packagekit
+		SCRIPT
+	end
+
+	# set up vagrant home
+	script = <<-SCRIPT
+		grep -q 'mgmt\.bashrc' ~/.bashrc || echo '. ~/.mgmt.bashrc' >>~/.bashrc
+		. ~/.mgmt.bashrc
+		go get -u github.com/purpleidea/mgmt
+		cd ~/gopath/src/github.com/purpleidea/mgmt
+		make deps
+	SCRIPT
+	config.vm.provision "shell" do |shell|
+		shell.privileged = false
+		shell.inline = script
+	end
+end

converger/converger.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -27,26 +27,26 @@ import (
 )
 
 // TODO: we could make a new function that masks out the state of certain
-// UUID's, but at the moment the new Timer code has obsoleted the need...
+// UID's, but at the moment the new Timer code has obsoleted the need...
 
-// Converger is the general interface for implementing a convergence watcher
+// Converger is the general interface for implementing a convergence watcher.
 type Converger interface { // TODO: need a better name
-	Register() ConvergerUUID
-	IsConverged(ConvergerUUID) bool         // is the UUID converged ?
-	SetConverged(ConvergerUUID, bool) error // set the converged state of the UUID
-	Unregister(ConvergerUUID)
+	Register() UID
+	IsConverged(UID) bool         // is the UID converged ?
+	SetConverged(UID, bool) error // set the converged state of the UID
+	Unregister(UID)
 	Start()
 	Pause()
 	Loop(bool)
-	ConvergedTimer(ConvergerUUID) <-chan time.Time
+	ConvergedTimer(UID) <-chan time.Time
 	Status() map[uint64]bool
 	Timeout() int                // returns the timeout that this was created with
 	SetStateFn(func(bool) error) // sets the stateFn
 }
 
-// ConvergerUUID is the interface resources can use to notify with if converged
-// you'll need to use part of the Converger interface to Register initially too
-type ConvergerUUID interface {
+// UID is the interface resources can use to notify with if converged. You'll
+// need to use part of the Converger interface to Register initially too.
+type UID interface {
 	ID() uint64   // get Id
 	Name() string // get a friendly name
 	SetName(string)
@@ -61,7 +61,7 @@ type ConvergerUUID interface {
 	StopTimer() error
 }
 
-// converger is an implementation of the Converger interface
+// converger is an implementation of the Converger interface.
 type converger struct {
 	timeout   int              // must be zero (instant) or greater seconds to run
 	stateFn   func(bool) error // run on converged state changes with state bool
@@ -73,17 +73,18 @@ type converger struct {
 	status    map[uint64]bool
 }
 
-// convergerUUID is an implementation of the ConvergerUUID interface
-type convergerUUID struct {
+// cuid is an implementation of the UID interface.
+type cuid struct {
 	converger Converger
 	id        uint64
 	name      string // user defined, friendly name
 	mutex     sync.Mutex
 	timer     chan struct{}
 	running   bool // is the above timer running?
+	wg        sync.WaitGroup
 }
 
-// NewConverger builds a new converger struct
+// NewConverger builds a new converger struct.
 func NewConverger(timeout int, stateFn func(bool) error) *converger {
 	return &converger{
 		timeout: timeout,
@@ -95,13 +96,13 @@ func NewConverger(timeout int, stateFn func(bool) error) *converger {
 	}
 }
 
-// Register assigns a ConvergerUUID to the caller
-func (obj *converger) Register() ConvergerUUID {
+// Register assigns a UID to the caller.
+func (obj *converger) Register() UID {
 	obj.mutex.Lock()
 	defer obj.mutex.Unlock()
 	obj.lastid++
 	obj.status[obj.lastid] = false // initialize as not converged
-	return &convergerUUID{
+	return &cuid{
 		converger: obj,
 		id:        obj.lastid,
 		name:      fmt.Sprintf("%d", obj.lastid), // some default
@@ -110,32 +111,32 @@ func (obj *converger) Register() ConvergerUUID {
 	}
 }
 
-// IsConverged gets the converged status of a uuid
-func (obj *converger) IsConverged(uuid ConvergerUUID) bool {
-	if !uuid.IsValid() {
-		panic(fmt.Sprintf("Id of ConvergerUUID(%s) is nil!", uuid.Name()))
+// IsConverged gets the converged status of a uid.
+func (obj *converger) IsConverged(uid UID) bool {
+	if !uid.IsValid() {
+		panic(fmt.Sprintf("the ID of UID(%s) is nil", uid.Name()))
 	}
 	obj.mutex.RLock()
-	isConverged, found := obj.status[uuid.ID()] // lookup
+	isConverged, found := obj.status[uid.ID()] // lookup
 	obj.mutex.RUnlock()
 	if !found {
-		panic("Id of ConvergerUUID is unregistered!")
+		panic("the ID of UID is unregistered")
 	}
 	return isConverged
 }
 
-// SetConverged updates the converger with the converged state of the UUID
-func (obj *converger) SetConverged(uuid ConvergerUUID, isConverged bool) error {
-	if !uuid.IsValid() {
-		return fmt.Errorf("Id of ConvergerUUID(%s) is nil!", uuid.Name())
+// SetConverged updates the converger with the converged state of the UID.
+func (obj *converger) SetConverged(uid UID, isConverged bool) error {
+	if !uid.IsValid() {
+		return fmt.Errorf("the ID of UID(%s) is nil", uid.Name())
 	}
 	obj.mutex.Lock()
-	if _, found := obj.status[uuid.ID()]; !found {
-		panic("Id of ConvergerUUID is unregistered!")
+	if _, found := obj.status[uid.ID()]; !found {
+		panic("the ID of UID is unregistered")
 	}
-	obj.status[uuid.ID()] = isConverged // set
-	obj.mutex.Unlock()                  // unlock *before* poke or deadlock!
-	if isConverged != obj.converged {   // only poke if it would be helpful
+	obj.status[uid.ID()] = isConverged // set
+	obj.mutex.Unlock()                 // unlock *before* poke or deadlock!
+	if isConverged != obj.converged {  // only poke if it would be helpful
 		// run in a go routine so that we never block... just queue up!
 		// this allows us to send events, even if we haven't started...
 		go func() { obj.channel <- struct{}{} }()
@@ -143,7 +144,7 @@ func (obj *converger) SetConverged(uuid ConvergerUUID, isConverged bool) error {
 	return nil
 }
 
-// isConverged returns true if *every* registered uuid has converged
+// isConverged returns true if *every* registered uid has converged.
 func (obj *converger) isConverged() bool {
 	obj.mutex.RLock() // take a read lock
 	defer obj.mutex.RUnlock()
@@ -155,42 +156,42 @@ func (obj *converger) isConverged() bool {
 	return true
 }
 
-// Unregister dissociates the ConvergedUUID from the converged checking
-func (obj *converger) Unregister(uuid ConvergerUUID) {
-	if !uuid.IsValid() {
-		panic(fmt.Sprintf("Id of ConvergerUUID(%s) is nil!", uuid.Name()))
+// Unregister dissociates the ConvergedUID from the converged checking.
+func (obj *converger) Unregister(uid UID) {
+	if !uid.IsValid() {
+		panic(fmt.Sprintf("the ID of UID(%s) is nil", uid.Name()))
 	}
 	obj.mutex.Lock()
-	uuid.StopTimer() // ignore any errors
-	delete(obj.status, uuid.ID())
+	uid.StopTimer() // ignore any errors
+	delete(obj.status, uid.ID())
 	obj.mutex.Unlock()
-	uuid.InvalidateID()
+	uid.InvalidateID()
 }
 
-// Start causes a Converger object to start or resume running
+// Start causes a Converger object to start or resume running.
 func (obj *converger) Start() {
 	obj.control <- true
 }
 
-// Pause causes a Converger object to stop running temporarily
+// Pause causes a Converger object to stop running temporarily.
 func (obj *converger) Pause() { // FIXME: add a sync ACK on pause before return
 	obj.control <- false
 }
 
-// Loop is the main loop for a Converger object; it usually runs in a goroutine
-// TODO: we could eventually have each resource tell us as soon as it converges
-// and then keep track of the time delays here, to avoid callers needing select
+// Loop is the main loop for a Converger object. It usually runs in a goroutine.
+// TODO: we could eventually have each resource tell us as soon as it converges,
+// and then keep track of the time delays here, to avoid callers needing select.
 // NOTE: when we have very short timeouts, if we start before all the resources
-// have joined the map, then it might appears as if we converged before we did!
+// have joined the map, then it might appear as if we converged before we did!
 func (obj *converger) Loop(startPaused bool) {
 	if obj.control == nil {
-		panic("Converger not initialized correctly")
+		panic("converger not initialized correctly")
 	}
 	if startPaused { // start paused without racing
 		select {
 		case e := <-obj.control:
 			if !e {
-				panic("Converger expected true!")
+				panic("converger expected true")
 			}
 		}
 	}
@@ -198,13 +199,13 @@ func (obj *converger) Loop(startPaused bool) {
 		select {
 		case e := <-obj.control: // expecting "false" which means pause!
 			if e {
-				panic("Converger expected false!")
+				panic("converger expected false")
 			}
 			// now i'm paused...
 			select {
 			case e := <-obj.control:
 				if !e {
-					panic("Converger expected true!")
+					panic("converger expected true")
 				}
 				// restart
 				// kick once to refresh the check...
@@ -243,20 +244,20 @@ func (obj *converger) Loop(startPaused bool) {
 	}
 }
 
-// ConvergedTimer adds a timeout to a select call and blocks until then
+// ConvergedTimer adds a timeout to a select call and blocks until then.
 // TODO: this means we could eventually have per resource converged timeouts
-func (obj *converger) ConvergedTimer(uuid ConvergerUUID) <-chan time.Time {
+func (obj *converger) ConvergedTimer(uid UID) <-chan time.Time {
 	// be clever: if i'm already converged, this timeout should block which
 	// avoids unnecessary new signals being sent! this avoids fast loops if
 	// we have a low timeout, or in particular a timeout == 0
-	if uuid.IsConverged() {
+	if uid.IsConverged() {
 		// blocks the case statement in select forever!
 		return util.TimeAfterOrBlock(-1)
 	}
 	return util.TimeAfterOrBlock(obj.timeout)
 }
 
-// Status returns a map of the converged status of each UUID.
+// Status returns a map of the converged status of each UID.
 func (obj *converger) Status() map[uint64]bool {
 	status := make(map[uint64]bool)
 	obj.mutex.RLock() // take a read lock
@@ -279,63 +280,65 @@ func (obj *converger) SetStateFn(stateFn func(bool) error) {
 	obj.stateFn = stateFn
 }
 
-// Id returns the unique id of this UUID object
-func (obj *convergerUUID) ID() uint64 {
+// ID returns the unique id of this UID object.
+func (obj *cuid) ID() uint64 {
 	return obj.id
 }
 
-// Name returns a user defined name for the specific convergerUUID.
-func (obj *convergerUUID) Name() string {
+// Name returns a user defined name for the specific cuid.
+func (obj *cuid) Name() string {
 	return obj.name
 }
 
-// SetName sets a user defined name for the specific convergerUUID.
-func (obj *convergerUUID) SetName(name string) {
+// SetName sets a user defined name for the specific cuid.
+func (obj *cuid) SetName(name string) {
 	obj.name = name
 }
 
-// IsValid tells us if the id is valid or has already been destroyed
-func (obj *convergerUUID) IsValid() bool {
+// IsValid tells us if the id is valid or has already been destroyed.
+func (obj *cuid) IsValid() bool {
 	return obj.id != 0 // an id of 0 is invalid
 }
 
-// InvalidateID marks the id as no longer valid
-func (obj *convergerUUID) InvalidateID() {
+// InvalidateID marks the id as no longer valid.
+func (obj *cuid) InvalidateID() {
 	obj.id = 0 // an id of 0 is invalid
 }
 
-// IsConverged is a helper function to the regular IsConverged method
-func (obj *convergerUUID) IsConverged() bool {
+// IsConverged is a helper function to the regular IsConverged method.
+func (obj *cuid) IsConverged() bool {
 	return obj.converger.IsConverged(obj)
 }
 
-// SetConverged is a helper function to the regular SetConverged notification
-func (obj *convergerUUID) SetConverged(isConverged bool) error {
+// SetConverged is a helper function to the regular SetConverged notification.
+func (obj *cuid) SetConverged(isConverged bool) error {
 	return obj.converger.SetConverged(obj, isConverged)
 }
 
-// Unregister is a helper function to unregister myself
-func (obj *convergerUUID) Unregister() {
+// Unregister is a helper function to unregister myself.
+func (obj *cuid) Unregister() {
 	obj.converger.Unregister(obj)
 }
 
-// ConvergedTimer is a helper around the regular ConvergedTimer method
-func (obj *convergerUUID) ConvergedTimer() <-chan time.Time {
+// ConvergedTimer is a helper around the regular ConvergedTimer method.
+func (obj *cuid) ConvergedTimer() <-chan time.Time {
 	return obj.converger.ConvergedTimer(obj)
 }
 
 // StartTimer runs an invisible timer that automatically converges on timeout.
-func (obj *convergerUUID) StartTimer() (func() error, error) {
+func (obj *cuid) StartTimer() (func() error, error) {
 	obj.mutex.Lock()
 	if !obj.running {
 		obj.timer = make(chan struct{})
 		obj.running = true
 	} else {
 		obj.mutex.Unlock()
-		return obj.StopTimer, fmt.Errorf("Timer already started!")
+		return obj.StopTimer, fmt.Errorf("timer already started")
 	}
 	obj.mutex.Unlock()
+	obj.wg.Add(1)
 	go func() {
+		defer obj.wg.Done()
 		for {
 			select {
 			case _, ok := <-obj.timer: // reset signal channel
@@ -359,24 +362,25 @@ func (obj *convergerUUID) StartTimer() (func() error, error) {
 }
 
 // ResetTimer resets the counter to zero if using a StartTimer internally.
-func (obj *convergerUUID) ResetTimer() error {
+func (obj *cuid) ResetTimer() error {
 	obj.mutex.Lock()
 	defer obj.mutex.Unlock()
 	if obj.running {
 		obj.timer <- struct{}{} // send the reset message
 		return nil
 	}
-	return fmt.Errorf("Timer hasn't been started!")
+	return fmt.Errorf("timer hasn't been started")
 }
 
 // StopTimer stops the running timer permanently until a StartTimer is run.
-func (obj *convergerUUID) StopTimer() error {
+func (obj *cuid) StopTimer() error {
 	obj.mutex.Lock()
 	defer obj.mutex.Unlock()
 	if !obj.running {
-		return fmt.Errorf("Timer isn't running!")
+		return fmt.Errorf("timer isn't running")
 	}
 	close(obj.timer)
+	obj.wg.Wait()
 	obj.running = false
 	return nil
 }

doc.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

docs/.gitignore

@@ -0,0 +1,2 @@
+mgmt-documentation.pdf
+_build

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = mgmt
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/conf.py

@@ -0,0 +1,158 @@
+# -*- coding: utf-8 -*-
+#
+# mgmt documentation build configuration file, created by
+# sphinx-quickstart on Wed Feb 15 21:34:09 2017.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+from recommonmark.parser import CommonMarkParser
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = []
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+
+source_parsers = {
+        '.md': CommonMarkParser,
+}
+
+source_suffix = ['.rst', '.md']
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'mgmt'
+copyright = u'2013-2017+ James Shubin and the project contributors'
+author = u'James Shubin'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = u''
+# The full version, including alpha/beta/rc tags.
+release = u''
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'venv']
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = False
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+#html_theme = 'alabaster'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+
+# -- Options for HTMLHelp output ------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'mgmtdoc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'mgmt.tex', u'mgmt Documentation',
+     u'James Shubin', 'manual'),
+]
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'mgmt', u'mgmt Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'mgmt', u'mgmt Documentation',
+     author, 'mgmt', 'A next generation config management prototype!',
+     'Miscellaneous'),
+]

docs/documentation.md

@@ -1,57 +1,16 @@
-#mgmt
+# mgmt
 
-<!--
-Mgmt
-Copyright (C) 2013-2016+ James Shubin and the project contributors
-Written by James Shubin <james@shubin.ca> and the project contributors
+Available from:
+[https://github.com/purpleidea/mgmt/](https://github.com/purpleidea/mgmt/)
 
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
+This documentation is available in: [Markdown](https://github.com/purpleidea/mgmt/blob/master/docs/documentation.md) or [PDF](https://pdfdoc-purpleidea.rhcloud.com/pdf/https://github.com/purpleidea/mgmt/blob/master/docs/documentation.md) format.
 
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
--->
-
-##mgmt by [James](https://ttboj.wordpress.com/)
-####Available from:
-####[https://github.com/purpleidea/mgmt/](https://github.com/purpleidea/mgmt/)
-
-####This documentation is available in: [Markdown](https://github.com/purpleidea/mgmt/blob/master/DOCUMENTATION.md) or [PDF](https://pdfdoc-purpleidea.rhcloud.com/pdf/https://github.com/purpleidea/mgmt/blob/master/DOCUMENTATION.md) format.
-
-####Table of Contents
-
-1. [Overview](#overview)
-2. [Project description - What the project does](#project-description)
-3. [Setup - Getting started with mgmt](#setup)
-4. [Features - All things mgmt can do](#features)
-	* [Autoedges - Automatic resource relationships](#autoedges)
-	* [Autogrouping - Automatic resource grouping](#autogrouping)
-	* [Automatic clustering - Automatic cluster management](#automatic-clustering)
-	* [Remote mode - Remote "agent-less" execution](#remote-agent-less-mode)
-	* [Puppet support - write manifest code for mgmt](#puppet-support)
-5. [Resources - All built-in primitives](#resources)
-6. [Usage/FAQ - Notes on usage and frequently asked questions](#usage-and-frequently-asked-questions)
-7. [Reference - Detailed reference](#reference)
-	* [Meta parameters](#meta-parameters)
-	* [Graph definition file](#graph-definition-file)
-	* [Command line](#command-line)
-8. [Examples - Example configurations](#examples)
-9. [Development - Background on module development and reporting bugs](#development)
-10. [Authors - Authors and contact information](#authors)
-
-##Overview
+## Overview
 
 The `mgmt` tool is a next generation config management prototype. It's not yet
 ready for production, but we hope to get there soon. Get involved today!
 
-##Project Description
+## Project Description
 
 The mgmt tool is a distributed, event driven, config management tool, that
 supports parallel execution, and librarification to be used as the management
@@ -63,24 +22,27 @@ For more information, you may like to read some blog posts from the author:
 * [Automatic edges in mgmt](https://ttboj.wordpress.com/2016/03/14/automatic-edges-in-mgmt/)
 * [Automatic grouping in mgmt](https://ttboj.wordpress.com/2016/03/30/automatic-grouping-in-mgmt/)
 * [Automatic clustering in mgmt](https://ttboj.wordpress.com/2016/06/20/automatic-clustering-in-mgmt/)
+* [Remote execution in mgmt](https://ttboj.wordpress.com/2016/10/07/remote-execution-in-mgmt/)
+* [Send/Recv in mgmt](https://ttboj.wordpress.com/2016/12/07/sendrecv-in-mgmt/)
+* [Metaparameters in mgmt](https://ttboj.wordpress.com/2017/03/01/metaparameters-in-mgmt/)
 
 There is also an [introductory video](http://meetings-archive.debian.net/pub/debian-meetings/2016/debconf16/Next_Generation_Config_Mgmt.webm) available.
 Older videos and other material [is available](https://github.com/purpleidea/mgmt/#on-the-web).
 
-##Setup
+## Setup
 
 During this prototype phase, the tool can be run out of the source directory.
-You'll probably want to use ```./run.sh run --file examples/graph1.yaml``` to
+You'll probably want to use ```./run.sh run --yaml examples/graph1.yaml``` to
 get started. Beware that this _can_ cause data loss. Understand what you're
 doing first, or perform these actions in a virtual environment such as the one
 provided by [Oh-My-Vagrant](https://github.com/purpleidea/oh-my-vagrant).
 
-##Features
+## Features
 
 This section details the numerous features of mgmt and some caveats you might
 need to be aware of.
 
-###Autoedges
+### Autoedges
 
 Automatic edges, or AutoEdges, is the mechanism in mgmt by which it will
 automatically create dependencies for you between resources. For example,
@@ -89,7 +51,7 @@ automatically ensure that any file resource you declare that matches a
 file installed by your package resource will only be processed after the
 package is installed.
 
-####Controlling autoedges
+#### Controlling autoedges
 
 Though autoedges is likely to be very helpful and avoid you having to declare
 all dependencies explicitly, there are cases where this behaviour is
@@ -106,12 +68,12 @@ installation of the `mysql-server` package.
 You can disable autoedges for a resource by setting the `autoedge` key on
 the meta attributes of that resource to `false`.
 
-####Blog post
+#### Blog post
 
 You can read the introductory blog post about this topic here:
 [https://ttboj.wordpress.com/2016/03/14/automatic-edges-in-mgmt/](https://ttboj.wordpress.com/2016/03/14/automatic-edges-in-mgmt/)
 
-###Autogrouping
+### Autogrouping
 
 Automatic grouping or AutoGroup is the mechanism in mgmt by which it will
 automatically group multiple resource vertices into a single one. This is
@@ -125,12 +87,12 @@ used for other use cases too.
 You can disable autogrouping for a resource by setting the `autogroup` key on
 the meta attributes of that resource to `false`.
 
-####Blog post
+#### Blog post
 
 You can read the introductory blog post about this topic here:
 [https://ttboj.wordpress.com/2016/03/30/automatic-grouping-in-mgmt/](https://ttboj.wordpress.com/2016/03/30/automatic-grouping-in-mgmt/)
 
-###Automatic clustering
+### Automatic clustering
 
 Automatic clustering is a feature by which mgmt automatically builds, scales,
 and manages the embedded etcd cluster which is compiled into mgmt itself. It is
@@ -141,12 +103,12 @@ If you prefer to avoid this feature. you can always opt to use an existing etcd
 cluster that is managed separately from mgmt by pointing your mgmt agents at it
 with the `--seeds` variable.
 
-####Blog post
+#### Blog post
 
 You can read the introductory blog post about this topic here:
 [https://ttboj.wordpress.com/2016/06/20/automatic-clustering-in-mgmt/](https://ttboj.wordpress.com/2016/06/20/automatic-clustering-in-mgmt/)
 
-###Remote ("agent-less") mode
+### Remote ("agent-less") mode
 
 Remote mode is a special mode that lets you kick off mgmt runs on one or more
 remote machines which are only accessible via SSH. In this mode the initiating
@@ -168,11 +130,12 @@ entire set of running mgmt agents will need to all simultaneously converge for
 the group to exit. This is particularly useful for bootstrapping new clusters
 which need to exchange information that is only available at run time.
 
-####Blog post
+#### Blog post
 
-An introductory blog post about this topic will follow soon.
+You can read the introductory blog post about this topic here:
+[https://ttboj.wordpress.com/2016/10/07/remote-execution-in-mgmt/](https://ttboj.wordpress.com/2016/10/07/remote-execution-in-mgmt/)
 
-###Puppet support
+### Puppet support
 
 You can supply a Puppet manifest instead of creating the (YAML) graph manually.
 Puppet must be installed and in `mgmt`'s search path. You also need the
@@ -194,12 +157,12 @@ Invoke `mgmt` with the `--puppet` switch, which supports 3 variants:
 
 For more details and caveats see [Puppet.md](Puppet.md).
 
-####Blog post
+#### Blog post
 
 An introductory post on the Puppet support is on
 [Felix's blog](http://ffrank.github.io/features/2016/06/19/puppet-powered-mgmt/).
 
-##Resources
+## Resources
 
 This section lists all the built-in resources and their properties. The
 resource primitives in `mgmt` are typically more powerful than resources in
@@ -215,88 +178,170 @@ meta parameters aren't very useful when combined with certain resources, but
 in general, it should be fairly obvious, such as when combining the `noop` meta
 parameter with the [Noop](#Noop) resource.
 
+* [Augeas](#Augeas): Manipulate files using augeas.
 * [Exec](#Exec): Execute shell commands on the system.
 * [File](#File): Manage files and directories.
+* [Hostname](#Hostname): Manages the hostname on the system.
+* [KV](#KV): Set a key value pair in our shared world database.
 * [Msg](#Msg): Send log messages.
 * [Noop](#Noop): A simple resource that does nothing.
+* [Nspawn](#Nspawn): Manage systemd-machined nspawn containers.
+* [Password](#Password): Create random password strings.
 * [Pkg](#Pkg):  Manage system packages with PackageKit.
 * [Svc](#Svc): Manage system systemd services.
 * [Timer](#Timer): Manage system systemd services.
+* [Virt](#Virt): Manage virtual machines with libvirt.
 
-###Exec
+
+### Augeas
+
+The augeas resource uses [augeas](http://augeas.net/) commands to manipulate
+files.
+
+### Exec
 
 The exec resource can execute commands on your system.
 
-###File
+### File
 
 The file resource manages files and directories. In `mgmt`, directories are
 identified by a trailing slash in their path name. File have no such slash.
 
-####Path
+It has the following properties:
+
+- `path`: file path (directories have a trailing slash here)
+- `content`: raw file content
+- `state`: either `exists` (the default value) or `absent`
+- `mode`: octal unix file permissions
+- `owner`: username or uid for the file owner
+- `group`: group name or gid for the file group
+
+#### Path
 
 The path property specifies the file or directory that we are managing.
 
-####Content
+#### Content
 
 The content property is a string that specifies the desired file contents.
 
-####Source
+#### Source
 
 The source property points to a source file or directory path that we wish to
 copy over and use as the desired contents for our resource.
 
-####State
+#### State
 
 The state property describes the action we'd like to apply for the resource. The
 possible values are: `exists` and `absent`.
 
-####Recurse
+#### Recurse
 
 The recurse property limits whether file resource operations should recurse into
 and monitor directory contents with a depth greater than one.
 
-####Force
+#### Force
 
 The force property is required if we want the file resource to be able to change
 a file into a directory or vice-versa. If such a change is needed, but the force
 property is not set to `true`, then this file resource will error.
 
-###Msg
+### Hostname
+
+The hostname resource manages static, transient/dynamic and pretty hostnames
+on the system and watches them for changes.
+
+#### static_hostname
+The static hostname is the one configured in /etc/hostname or a similar
+file.
+It is chosen by the local user. It is not always in sync with the current
+host name as returned by the gethostname() system call.
+
+#### transient_hostname
+The transient / dynamic hostname is the one configured via the kernel's
+sethostbyname().
+It can be different from the static hostname in case DHCP or mDNS have been
+configured to change the name based on network information.
+
+#### pretty_hostname
+The pretty hostname is a free-form UTF8 host name for presentation to the user.
+
+#### hostname
+Hostname is the fallback value for all 3 fields above, if only `hostname` is
+specified, it will set all 3 fields to this value.
+
+### KV
+
+The KV resource sets a key and value pair in the global world database. This is
+quite useful for setting a flag after a number of resources have run. It will
+ignore database updates to the value that are greater in compare order than the
+requested key if the `SkipLessThan` parameter is set to true. If we receive a
+refresh, then the stored value will be reset to the requested value even if the
+stored value is greater.
+
+#### Key
+The string key used to store the key.
+
+#### Value
+The string value to set. This can also be set via Send/Recv.
+
+#### SkipLessThan
+If this parameter is set to `true`, then it will ignore updating the value as
+long as the database versions are greater than the requested value. The compare
+operation used is based on the `SkipCmpStyle` parameter.
+
+#### SkipCmpStyle
+By default this converts the string values to integers and compares them as you
+would expect.
+
+### Msg
 
 The msg resource sends messages to the main log, or an external service such
 as systemd's journal.
 
-###Noop
+### Noop
 
 The noop resource does absolutely nothing. It does have some utility in testing
 `mgmt` and also as a placeholder in the resource graph.
 
-###Pkg
+### Nspawn
+
+The nspawn resource is used to manage systemd-machined style containers.
+
+### Password
+
+The password resource can generate a random string to be used as a password. It
+will re-generate the password if it receives a refresh notification.
+
+### Pkg
 
 The pkg resource is used to manage system packages. This resource works on many
 different distributions because it uses the underlying packagekit facility which
 supports different backends for different environments. This ensures that we
 have great Debian (deb/dpkg) and Fedora (rpm/dnf) support simultaneously.
 
-###Svc
+### Svc
 
 The service resource is still very WIP. Please help us my improving it!
 
-###Timer
+### Timer
 
 This resource needs better documentation. Please help us my improving it!
 
-##Usage and frequently asked questions
+### Virt
+
+The virt resource can manage virtual machines via libvirt.
+
+## Usage and frequently asked questions
 (Send your questions as a patch to this FAQ! I'll review it, merge it, and
 respond by commit with the answer.)
 
-###Why did you start this project?
+### Why did you start this project?
 
 I wanted a next generation config management solution that didn't have all of
 the design flaws or limitations that the current generation of tools do, and no
 tool existed!
 
-###Why did you use etcd? What about consul?
+### Why did you use etcd? What about consul?
 
 Etcd and consul are both written in golang, which made them the top two
 contenders for my prototype. Ultimately a choice had to be made, and etcd was
@@ -304,7 +349,7 @@ chosen, but it was also somewhat arbitrary. If there is available interest,
 good reasoning, *and* patches, then we would consider either switching or
 supporting both, but this is not a high priority at this time.
 
-###Can I use an existing etcd cluster instead of the automatic embedded servers?
+### Can I use an existing etcd cluster instead of the automatic embedded servers?
 
 Yes, it's possible to use an existing etcd cluster instead of the automatic,
 elastic embedded etcd servers. To do so, simply point to the cluster with the
@@ -315,7 +360,7 @@ The downside to this approach is that you won't benefit from the automatic
 elastic nature of the embedded etcd servers, and that you're responsible if you
 accidentally break your etcd cluster, or if you use an unsupported version.
 
-###What does the error message about an inconsistent dataDir mean?
+### What does the error message about an inconsistent dataDir mean?
 
 If you get an error message similar to:
 
@@ -333,7 +378,7 @@ starting up, and as a result, a default endpoint never gets added. The solution
 is to either reconcile the mistake, and if there is no important data saved, you
 can remove the etcd dataDir. This is typically `/var/lib/mgmt/etcd/member/`.
 
-###Why do resources have both a `Compare` method and an `IFF` (on the UUID) method?
+### Why do resources have both a `Compare` method and an `IFF` (on the UID) method?
 
 The `Compare()` methods are for determining if two resources are effectively the
 same, which is used to make graph change delta's efficient. This is when we want
@@ -342,21 +387,21 @@ vertices. Since we want to make this process efficient, we only update the parts
 that are different, and leave everything else alone. This `Compare()` method can
 tell us if two resources are the same.
 
-The `IFF()` method is part of the whole UUID system, which is for discerning if
-a resource meets the requirements another expects for an automatic edge. This is
-because the automatic edge system assumes a unified UUID pattern to test for
+The `IFF()` method is part of the whole UID system, which is for discerning if a
+resource meets the requirements another expects for an automatic edge. This is
+because the automatic edge system assumes a unified UID pattern to test for
 equality. In the future it might be helpful or sane to merge the two similar
 comparison functions although for now they are separate because they are
 actually answer different questions.
 
-###Did you know that there is a band named `MGMT`?
+### Did you know that there is a band named `MGMT`?
 
 I didn't realize this when naming the project, and it is accidental. After much
 anguishing, I chose the name because it was short and I thought it was
 appropriately descriptive. If you need a less ambiguous search term or phrase,
 you can try using `mgmtconfig` or `mgmt config`.
 
-###You didn't answer my question, or I have a question!
+### You didn't answer my question, or I have a question!
 
 It's best to ask on [IRC](https://webchat.freenode.net/?channels=#mgmtconfig)
 to see if someone can help you. Once we get a big enough community going, we'll
@@ -366,40 +411,41 @@ and I'll do my best to help. If you have a good question, please add it as a
 patch to this documentation. I'll merge your question, and add a patch with the
 answer!
 
-##Reference
+## Reference
 Please note that there are a number of undocumented options. For more
 information on these options, please view the source at:
 [https://github.com/purpleidea/mgmt/](https://github.com/purpleidea/mgmt/).
 If you feel that a well used option needs documenting here, please patch it!
 
-###Overview of reference
+### Overview of reference
 * [Meta parameters](#meta-parameters): List of available resource meta parameters.
 * [Graph definition file](#graph-definition-file): Main graph definition file.
 * [Command line](#command-line): Command line parameters.
+* [Compilation options](#compilation-options): Compilation options.
 
-###Meta parameters
+### Meta parameters
 These meta parameters are special parameters (or properties) which can apply to
 any resource. The usefulness of doing so will depend on the particular meta
 parameter and resource combination.
 
-####AutoEdge
+#### AutoEdge
 Boolean. Should we generate auto edges for this resource?
 
-####AutoGroup
+#### AutoGroup
 Boolean. Should we attempt to automatically group this resource with others?
 
-####Noop
+#### Noop
 Boolean. Should the Apply portion of the CheckApply method of the resource
 make any changes? Noop is a concatenation of no-operation.
 
-####Retry
+#### Retry
 Integer. The number of times to retry running the resource on error. Use -1 for
 infinite. This currently applies for both the Watch operation (which can fail)
 and for the CheckApply operation. While they could have separate values, I've
 decided to use the same ones for both until there's a proper reason to want to
 do something differently for the Watch errors.
 
-####Delay
+#### Delay
 Integer. Number of milliseconds to wait between retries. The same value is
 shared between the Watch and CheckApply retries. This currently applies for both
 the Watch operation (which can fail) and for the CheckApply operation. While
@@ -407,63 +453,113 @@ they could have separate values, I've decided to use the same ones for both
 until there's a proper reason to want to do something differently for the Watch
 errors.
 
-###Graph definition file
+#### Poll
+Integer. Number of seconds to wait between `CheckApply` checks. If this is
+greater than zero, then the standard event based `Watch` mechanism for this
+resource is replaced with a simple polling mechanism. In general, this is not
+recommended, unless you have a very good reason for doing so.
+
+Please keep in mind that if you have a resource which changes every `I` seconds,
+and you poll it every `J` seconds, and you've asked for a converged timeout of
+`K` seconds, and `I <= J <= K`, then your graph will likely never converge.
+
+When polling, the system detects that a resource is not converged if its
+`CheckApply` method returns false. This allows a resource which changes every
+`I` seconds, and which is polled every `J` seconds, and with a converged timeout
+of `K` seconds to still converge when `J <= K`, as long as `I > J || I > K`,
+which is another way of saying that if the resource finally settles down to give
+the graph enough time, it can probably converge.
+
+#### Limit
+Float. Maximum rate of `CheckApply` runs started per second. Useful to limit
+an especially _eventful_ process from causing excessive checks to run. This
+defaults to `+Infinity` which adds no limiting. If you change this value, you
+will also need to change the `Burst` value to a non-zero value. Please see the
+[rate](https://godoc.org/golang.org/x/time/rate) package for more information.
+
+#### Burst
+Integer. Burst is the maximum number of runs which can happen without invoking
+the rate limiter as designated by the `Limit` value. If the `Limit` is not set
+to `+Infinity`, this must be a non-zero value. Please see the
+[rate](https://godoc.org/golang.org/x/time/rate) package for more information.
+
+#### Sema
+List of string ids. Sema is a P/V style counting semaphore which can be used to
+limit parallelism during the CheckApply phase of resource execution. Each
+resource can have `N` different semaphores which share a graph global namespace.
+Each semaphore has a maximum count associated with it. The default value of the
+size is 1 (one) if size is unspecified. Each string id is the unique id of the
+semaphore. If the id contains a trailing colon (:) followed by a positive
+integer, then that value is the max size for that semaphore. Valid semaphore
+id's include: `some_id`, `hello:42`, `not:smart:4` and `:13`. It is expected
+that the last bare example be only used by the engine to add a global semaphore.
+
+### Graph definition file
 graph.yaml is the compiled graph definition file. The format is currently
 undocumented, but by looking through the [examples/](https://github.com/purpleidea/mgmt/tree/master/examples)
 you can probably figure out most of it, as it's fairly intuitive.
 
-###Command line
+### Command line
 The main interface to the `mgmt` tool is the command line. For the most recent
 documentation, please run `mgmt --help`.
 
-####`--file <graph.yaml>`
+#### `--yaml <graph.yaml>`
 Point to a graph file to run.
 
-####`--converged-timeout <seconds>`
+#### `--converged-timeout <seconds>`
 Exit if the machine has converged for approximately this many seconds.
 
-####`--max-runtime <seconds>`
+#### `--max-runtime <seconds>`
 Exit when the agent has run for approximately this many seconds. This is not
 generally recommended, but may be useful for users who know what they're doing.
 
-####`--noop`
+#### `--noop`
 Globally force all resources into no-op mode. This also disables the export to
 etcd functionality, but does not disable resource collection, however all
 resources that are collected will have their individual noop settings set.
 
-####`--remote <graph.yaml>`
+#### `--sema <size>`
+Globally add a counting semaphore of this size to each resource in the graph.
+The semaphore will get given an id of `:size`. In other words if you specify a
+size of 42, you can expect a semaphore if named: `:42`. It is expected that
+consumers of the semaphore metaparameter always include a prefix to avoid a
+collision with this globally defined semaphore. The size value must be greater
+than zero at this time. The traditional non-parallel execution found in config
+management tools such as `Puppet` can be obtained with `--sema 1`.
+
+#### `--remote <graph.yaml>`
 Point to a graph file to run on the remote host specified within. This parameter
 can be used multiple times if you'd like to remotely run on multiple hosts in
 parallel.
 
-####`--allow-interactive`
+#### `--allow-interactive`
 Allow interactive prompting for SSH passwords if there is no authentication
 method that works.
 
-####`--ssh-priv-id-rsa`
+#### `--ssh-priv-id-rsa`
 Specify the path for finding SSH keys. This defaults to `~/.ssh/id_rsa`. To
 never use this method of authentication, set this to the empty string.
 
-####`--cconns`
+#### `--cconns`
 The maximum number of concurrent remote ssh connections to run. This defaults
 to `0`, which means unlimited.
 
-####`--no-caching`
+#### `--no-caching`
 Don't allow remote caching of the remote execution binary. This will require
 the binary to be copied over for every remote execution, but it limits the
 likelihood that there is leftover information from the configuration process.
 
-####`--prefix <path>`
+#### `--prefix <path>`
 Specify a path to a custom working directory prefix. This directory will get
 created if it does not exist. This usually defaults to `/var/lib/mgmt/`. This
 can't be combined with the `--tmp-prefix` option. It can be combined with the
 `--allow-tmp-prefix` option.
 
-####`--tmp-prefix`
+#### `--tmp-prefix`
 If this option is specified, a temporary prefix will be used instead of the
 default prefix. This can't be combined with the `--prefix` option.
 
-####`--allow-tmp-prefix`
+#### `--allow-tmp-prefix`
 If this option is specified, we will attempt to fall back to a temporary prefix
 if the primary prefix couldn't be created. This is useful for avoiding failures
 in environments where the primary prefix may or may not be available, but you'd
@@ -471,7 +567,35 @@ like to try. The canonical example is when running `mgmt` with `--remote` there
 might be a cached copy of the binary in the primary prefix, but in case there's
 no binary available continue working in a temporary directory to avoid failure.
 
-##Examples
+### Compilation options
+
+You can control some compilation variables by using environment variables.
+
+#### Disable libvirt support
+
+If you wish to compile mgmt without libvirt, you can use the following command:
+
+```
+GOTAGS=novirt make build
+```
+
+#### Disable augeas support
+
+If you wish to compile mgmt without augeas support, you can use the following command:
+
+```
+GOTAGS=noaugeas make build
+```
+
+#### Combining compile-time flags
+
+You can combine multiple tags by using a space-separated list:
+
+```
+GOTAGS="noaugeas novirt" make build
+```
+
+## Examples
 For example configurations, please consult the [examples/](https://github.com/purpleidea/mgmt/tree/master/examples) directory in the git
 source repository. It is available from:
 
@@ -499,7 +623,7 @@ EOF
 sudo systemctl daemon-reload
 ```
 
-##Development
+## Development
 
 This is a project that I started in my free time in 2013. Development is driven
 by all of our collective patches! Dive right in, and start hacking!
@@ -509,9 +633,9 @@ You can follow along [on my technical blog](https://ttboj.wordpress.com/).
 
 To report any bugs, please file a ticket at: [https://github.com/purpleidea/mgmt/issues](https://github.com/purpleidea/mgmt/issues).
 
-##Authors
+## Authors
 
-Copyright (C) 2013-2016+ James Shubin and the project contributors
+Copyright (C) 2013-2017+ James Shubin and the project contributors
 
 Please see the
 [AUTHORS](https://github.com/purpleidea/mgmt/tree/master/AUTHORS) file

docs/index.rst

@@ -0,0 +1,17 @@
+.. mgmt documentation master file, created by
+   sphinx-quickstart on Wed Feb 15 21:34:09 2017.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to mgmt's documentation!
+================================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   documentation
+   quick-start-guide
+   resource-guide
+   prometheus
+   puppet-guide

docs/prometheus.md

@@ -0,0 +1,66 @@
+# Prometheus support
+
+Mgmt comes with a built-in prometheus support. It is disabled by default, and
+can be enabled with the `--prometheus` command line switch.
+
+By default, the prometheus instance will listen on [`127.0.0.1:9233`][pd]. You
+can change this setting by using the `--prometheus-listen` cli option:
+
+To have mgmt prometheus bind interface on 0.0.0.0:45001, use:
+`./mgmt r --prometheus --prometheus-listen :45001`
+
+## Metrics
+
+Mgmt exposes three kinds of resources: _go_ metrics, _etcd_ metrics and _mgmt_
+metrics.
+
+### go metrics
+
+We use the [prometheus go_collector][pgc] to expose go metrics. Those metrics
+are mainly useful for debugging and perf testing.
+
+### etcd metrics
+
+mgmt exposes etcd metrics. Read more in the [upstream documentation][etcdm]
+
+### mgmt metrics
+
+Here is a list of the metrics we provide:
+
+- `mgmt_resources_total`: The number of resources that mgmt is managing
+- `mgmt_checkapply_total`: The number of CheckApply's that mgmt has run
+- `mgmt_failures_total`: The number of resources that have failed
+- `mgmt_failures_current`: The number of resources that have failed
+- `mgmt_graph_start_time_seconds`: Start time of the current graph since unix epoch in seconds
+
+For each metric, you will get some extra labels:
+
+- `kind`: The kind of mgmt resource
+
+For `mgmt_checkapply_total`, those extra labels are set:
+
+- `eventful`: "true" or "false", if the CheckApply triggered some changes
+- `errorful`: "true" or "false", if the CheckApply reported an error
+- `apply`: "true" or "false", if the CheckApply ran in apply or noop mode
+
+## Alerting
+
+You can use prometheus to alert you upon changes or failures. We do not provide
+such templates yet, but we plan to provide some examples in this repository.
+Patches welcome!
+
+## Grafana
+
+We do not have grafana dashboards yet. Patches welcome!
+
+## External resources
+
+- [prometheus website](https://prometheus.io/)
+- [prometheus documentation](https://prometheus.io/docs/introduction/overview/)
+- [prometheus best practices regarding metrics
+  naming](https://prometheus.io/docs/practices/naming/)
+- [grafana website](http://grafana.org/)
+
+[pgc]: https://github.com/prometheus/client_golang/blob/master/prometheus/go_collector.go
+[etcdm]: https://coreos.com/etcd/docs/latest/metrics.html
+[pd]: https://github.com/prometheus/prometheus/wiki/Default-port-allocation

docs/puppet-guide.md

@@ -1,22 +1,13 @@
-#mgmt Puppet support
-
-1. [Prerequisites](#prerequisites)
-    * [Testing the Puppet side](#testing-the-puppet-side)
-2. [Writing a suitable manifest](#writing-a-suitable-manifest)
-    * [Unsupported attributes](#unsupported-attributes)
-    * [Unsupported resources](#unsupported-resources)
-    * [Avoiding common warnings](#avoiding-common-warnings)
-3. [Configuring Puppet](#configuring-puppet)
-4. [Caveats](#caveats)
+# Puppet guide
 
 `mgmt` can use Puppet as its source for the configuration graph.
 This document goes into detail on how this works, and lists
 some pitfalls and limitations.
 
 For basic instructions on how to use the Puppet support, see
-the [main documentation](DOCUMENTATION.md#puppet-support).
+the [main documentation](documentation.md#puppet-support).
 
-##Prerequisites
+## Prerequisites
 
 You need Puppet installed in your system. It is not important how you
 get it. On the most common Linux distributions, you can use packages
@@ -29,14 +20,16 @@ Any release of Puppet's 3.x and 4.x series should be suitable for use with
 `mgmt`. Most importantly, make sure to install the `ffrank-mgmtgraph` Puppet
 module (referred to below as "the translator module").
 
-    puppet module install ffrank-mgmtgraph
+```
+puppet module install ffrank-mgmtgraph
+```
 
 Please note that the module is not required on your Puppet master (if you
 use a master/agent setup). It's needed on the machine that runs `mgmt`.
 You can install the module on the master anyway, so that it gets distributed
 to your agents through Puppet's `pluginsync` mechanism.
 
-###Testing the Puppet side
+### Testing the Puppet side
 
 The following command should run successfully and print a YAML hash on your
 terminal:
@@ -48,9 +41,9 @@ puppet mgmtgraph print --code 'file { "/tmp/mgmt-test": ensure => present }'
 You can use this CLI to test any manifests before handing them straight
 to `mgmt`.
 
-##Writing a suitable manifest
+## Writing a suitable manifest
 
-###Unsupported attributes
+### Unsupported attributes
 
 `mgmt` inherited its resource module from Puppet, so by and large, it's quite
 possible to express `mgmt` graphs in terms of Puppet manifests. However,
@@ -62,8 +55,10 @@ For example, at the time of writing this, the `file` type in `mgmt` had no
 notion of permissions (the file `mode`) yet. This lead to the following
 warning (among others that will be discussed below):
 
-    $ puppet mgmtgraph print --code 'file { "/tmp/foo": mode => "0600" }'
-    Warning: cannot translate: File[/tmp/foo] { mode => "600" } (attribute is ignored)
+```
+$ puppet mgmtgraph print --code 'file { "/tmp/foo": mode => "0600" }'
+Warning: cannot translate: File[/tmp/foo] { mode => "600" } (attribute is ignored)
+```
 
 This is a heads-up for the user, because the resulting `mgmt` graph will
 in fact not pass this information to the `/tmp/foo` file resource, and
@@ -71,7 +66,7 @@ in fact not pass this information to the `/tmp/foo` file resource, and
 manifests that are written expressly for `mgmt` is not sensible and should
 be avoided.
 
-###Unsupported resources
+### Unsupported resources
 
 Puppet has a fairly large number of
 [built-in types](https://docs.puppet.com/puppet/latest/reference/type.html),
@@ -91,28 +86,32 @@ this overhead can amount to several orders of magnitude.
 
 Avoid Puppet types that `mgmt` does not implement (yet).
 
-###Avoiding common warnings
+### Avoiding common warnings
 
 Many resource parameters in Puppet take default values. For the most part,
 the translator module just ignores them. However, there are cases in which
 Puppet will default to convenient behavior that `mgmt` cannot quite replicate.
 For example, translating a plain `file` resource will lead to a warning message:
 
-    $ puppet mgmtgraph print --code 'file { "/tmp/mgmt-test": }'
-    Warning: File[/tmp/mgmt-test] uses the 'puppet' file bucket, which mgmt cannot do. There will be no backup copies!
+```
+$ puppet mgmtgraph print --code 'file { "/tmp/mgmt-test": }'
+Warning: File[/tmp/mgmt-test] uses the 'puppet' file bucket, which mgmt cannot do. There will be no backup copies!
+```
 
 The reason is that per default, Puppet assumes the following parameter value
 (among others)
 
 ```puppet
 file { "/tmp/mgmt-test":
-  backup => 'puppet',
+	backup => 'puppet',
 }
 ```
 
 To avoid this, specify the parameter explicitly:
 
-    $ puppet mgmtgraph print --code 'file { "/tmp/mgmt-test": backup => false }'
+```
+$ puppet mgmtgraph print --code 'file { "/tmp/mgmt-test": backup => false }'
+```
 
 This is tedious in a more complex manifest. A good simplification is the
 following [resource default](https://docs.puppet.com/puppet/latest/reference/lang_defaults.html)
@@ -125,7 +124,7 @@ File { backup => false }
 If you encounter similar warnings from other types and/or parameters,
 use the same approach to silence them if possible.
 
-##Configuring Puppet
+## Configuring Puppet
 
 Since `mgmt` uses an actual Puppet CLI behind the scenes, you might
 need to tweak some of Puppet's runtime options in order to make it
@@ -143,16 +142,20 @@ control all of them, through its `--puppet-conf` option. It allows
 you to specify which `puppet.conf` file should be used during
 translation.
 
-    mgmt run --puppet /opt/my-manifest.pp --puppet-conf /etc/mgmt/puppet.conf
+```
+mgmt run --puppet /opt/my-manifest.pp --puppet-conf /etc/mgmt/puppet.conf
+```
 
 Within this file, you can just specify any needed options in the
 `[main]` section:
 
-    [main]
-    server=mgmt-master.example.net
-    vardir=/var/lib/mgmt/puppet
+```
+[main]
+server=mgmt-master.example.net
+vardir=/var/lib/mgmt/puppet
+```
 
-##Caveats
+## Caveats
 
 Please see the [README](https://github.com/ffrank/puppet-mgmtgraph/blob/master/README.md)
 of the translator module for the current state of supported and unsupported

docs/quick-start-guide.md

@@ -0,0 +1,93 @@
+# Quick start guide
+
+## Introduction
+This guide is intended for developers. Once `mgmt` is minimally viable, we'll
+publish a quick start guide for users too. In the meantime, please contribute!
+If you're brand new to `mgmt`, it's probably a good idea to start by reading the
+[introductory article](https://ttboj.wordpress.com/2016/01/18/next-generation-configuration-mgmt/)
+or to watch an [introductory video](https://github.com/purpleidea/mgmt/#on-the-web).
+Once you're familiar with the general idea, please start hacking...
+
+## Vagrant
+If you would like to avoid doing the following steps manually, we have prepared
+a [Vagrant](https://www.vagrantup.com/) environment for your convenience. From
+the project directory, run a `vagrant up`, and then a `vagrant status`. From
+there, you can `vagrant ssh` into the `mgmt` machine. The MOTD will explain the
+rest.
+
+## Dependencies
+Software projects have a few different kinds of dependencies. There are _build_
+dependencies, _runtime_ dependencies, and additionally, a few extra dependencies
+required for running the _test_ suite.
+
+### Build
+* `golang` 1.6 or higher (required, available in most distros)
+* golang libraries (required, available with `go get ./...`) a partial list includes:
+```
+github.com/coreos/etcd/client
+gopkg.in/yaml.v2
+gopkg.in/fsnotify.v1
+github.com/urfave/cli
+github.com/coreos/go-systemd/dbus
+github.com/coreos/go-systemd/util
+github.com/libvirt/libvirt-go
+```
+* `stringer` (optional), available as a package on some platforms, otherwise via `go get`
+```
+golang.org/x/tools/cmd/stringer
+```
+* `pandoc` (optional), for building a pdf of the documentation
+
+### Runtime
+A relatively modern GNU/Linux system should be able to run `mgmt` without any
+problems. Since `mgmt` runs as a single statically compiled binary, all of the
+library dependencies are included. It is expected, that certain advanced
+resources require host specific facilities to work. These requirements are
+listed below:
+
+| Resource | Dependency        | Version |
+|----------|-------------------|---------|
+| file     | inotify           | ?       |
+| hostname | systemd-hostnamed | ?       |
+| nspawn   | systemd-nspawn    | ?       |
+| pkg      | packagekitd       | ?       |
+| svc      | systemd           | ?       |
+| virt     | libvirtd          | ?       |
+
+For building a visual representation of the graph, `graphviz` is required.
+
+### Testing
+* golint `github.com/golang/lint/golint`
+
+## Quick start
+* Make sure you have golang version 1.6 or greater installed.
+* If you do not have a GOPATH yet, create one and export it:
+```
+mkdir $HOME/gopath
+export GOPATH=$HOME/gopath
+```
+* You might also want to add the GOPATH to your `~/.bashrc` or `~/.profile`.
+* For more information you can read the [GOPATH documentation](https://golang.org/cmd/go/#hdr-GOPATH_environment_variable).
+* Next download the mgmt code base, and switch to that directory:
+```
+mkdir -p $GOPATH/src/github.com/purpleidea/
+cd $GOPATH/src/github.com/purpleidea/
+git clone --recursive https://github.com/purpleidea/mgmt/
+cd $GOPATH/src/github.com/purpleidea/mgmt
+```
+* Run `make deps` to install system and golang dependencies. Take a look at `misc/make-deps.sh` for details.
+* Run `make build` to get a freshly built `mgmt` binary.
+* Run `time ./mgmt run --yaml examples/graph0.yaml --converged-timeout=5 --tmp-prefix` to try out a very simple example!
+* To run continuously in the default mode of operation, omit the `--converged-timeout` option.
+* Have fun hacking on our future technology!
+
+## Examples
+Please look in the [examples/](../examples/) folder for some examples!
+
+## Installation
+Installation of `mgmt` from distribution packages currently needs improvement.
+At the moment we have:
+* [COPR](https://copr.fedoraproject.org/coprs/purpleidea/mgmt/)
+* [Arch](https://aur.archlinux.org/packages/mgmt/)
+
+Please contribute more! We'd especially like to see a Debian package!

docs/resource-guide.md

@@ -0,0 +1,576 @@
+# Resource guide
+
+## Overview
+
+The `mgmt` tool has built-in resource primitives which make up the building
+blocks of any configuration. Each instance of a resource is mapped to a single
+vertex in the resource [graph](https://en.wikipedia.org/wiki/Directed_acyclic_graph).
+This guide is meant to instruct developers on how to write a brand new resource.
+Since `mgmt` and the core resources are written in golang, some prior golang
+knowledge is assumed.
+
+## Theory
+
+Resources in `mgmt` are similar to resources in other systems in that they are
+[idempotent](https://en.wikipedia.org/wiki/Idempotence). Our resources are
+uniquely different in that they can detect when their state has changed, and as
+a result can run to revert or repair this change instantly. For some background
+on this design, please read the
+[original article](https://ttboj.wordpress.com/2016/01/18/next-generation-configuration-mgmt/)
+on the subject.
+
+## Resource API
+
+To implement a resource in `mgmt` it must satisfy the
+[`Res`](https://github.com/purpleidea/mgmt/blob/master/resources/resources.go)
+interface. What follows are each of the method signatures and a description of
+each.
+
+### Default
+```golang
+Default() Res
+```
+
+This returns a populated resource struct as a `Res`. It shouldn't populate any
+values which already have the correct default as the golang zero value. In
+general it is preferable if the zero values make for the correct defaults.
+
+#### Example
+```golang
+// Default returns some sensible defaults for this resource.
+func (obj *FooRes) Default() Res {
+	return &FooRes{
+		Answer: 42, // sometimes, defaults shouldn't be the zero value
+	}
+}
+```
+
+### Validate
+```golang
+Validate() error
+```
+
+This method is used to validate if the populated resource struct is a valid
+representation of the resource kind. If it does not conform to the resource
+specifications, it should generate an error. If you notice that this method is
+quite large, it might be an indication that you should reconsider the parameter
+list and interface to this resource. This method is called _before_ `Init`.
+
+#### Example
+```golang
+// Validate reports any problems with the struct definition.
+func (obj *FooRes) Validate() error {
+	if obj.Answer != 42 { // validate whatever you want
+		return fmt.Errorf("expected an answer of 42")
+	}
+	return obj.BaseRes.Validate() // remember to call the base method!
+}
+```
+
+### Init
+```golang
+Init() error
+```
+
+This is called to initialize the resource. If something goes wrong, it should
+return an error. It should set the resource `kind`, do any resource specific
+work, and finish by calling the `Init` method of the base resource.
+
+#### Example
+```golang
+// Init initializes the Foo resource.
+func (obj *FooRes) Init() error {
+	obj.BaseRes.Kind = "foo" // must lower case resource kind
+	// run the resource specific initialization, and error if anything fails
+	if some_error {
+		return err // something went wrong!
+	}
+	return obj.BaseRes.Init() // call the base resource init
+}
+```
+
+This method is always called after `Validate` has run successfully, with the
+exception that we can't prevent a malicious or buggy `libmgmt` user to not run
+this. In other words, you should expect `Validate` to have run first, but you
+shouldn't allow `Init` to dangerously `rm -rf /$the_world` if your code only
+checks `$the_world` in `Validate`. Remember to always program safely!
+
+### Close
+```golang
+Close() error
+```
+
+This is called to cleanup after the resource. It is usually not necessary, but
+can be useful if you'd like to properly close a persistent connection that you
+opened in the `Init` method and were using throughout the resource.
+
+#### Example
+```golang
+// Close runs some cleanup code for this resource.
+func (obj *FooRes) Close() error {
+	err := obj.conn.Close() // close some internal connection
+
+	// call base close, b/c we're overriding
+	if e := obj.BaseRes.Close(); err == nil {
+		err = e
+	} else if e != nil {
+		err = multierr.Append(err, e) // list of errors
+	}
+	return err
+}
+```
+
+You should probably check the return errors of your internal methods, and pass
+on an error if something went wrong. Remember to always call the base `Close`
+method! If you plan to return early if you hit an internal error, then at least
+call it with a defer!
+
+### CheckApply
+```golang
+CheckApply(apply bool) (checkOK bool, err error)
+```
+
+`CheckApply` is where the real _work_ is done. Under normal circumstances, this
+function should check if the state of this resource is correct, and if so, it
+should return: `(true, nil)`. If the `apply` variable is set to `true`, then
+this means that we should then proceed to run the changes required to bring the
+resource into the correct state. If the `apply` variable is set to `false`, then
+the resource is operating in _noop_ mode and _no operations_ should be executed!
+
+After having executed the necessary operations to bring the resource back into
+the desired state, or after having detected that the state was incorrect, but
+that changes can't be made because `apply` is `false`, you should then return
+`(false, nil)`.
+
+You must cause the resource to converge during a single execution of this
+function. If you cannot, then you must return an error! The exception to this
+rule is that if an external force changes the state of the resource while it is
+being remedied, it is possible to return from this function even though the
+resource isn't now converged. This is not a bug, as the resources `Watch`
+facility will detect the change, ultimately resulting in a subsequent call to
+`CheckApply`.
+
+#### Example
+```golang
+// CheckApply does the idempotent work of checking and applying resource state.
+func (obj *FooRes) CheckApply(apply bool) (bool, error) {
+	// check the state
+	if state_is_okay { return true, nil } // done early! :)
+	// state was bad
+	if !apply { return false, nil } // don't apply; !stateok, nil
+	// do the apply!
+	return false, nil // after success applying
+	if any_error { return false, err } // anytime there's an err!
+}
+```
+
+The `CheckApply` function is called by the `mgmt` engine when it believes a call
+is necessary. Under certain conditions when a `Watch` call does not invalidate
+the state of the resource, and no refresh call was sent, its execution might be
+skipped. This is an engine optimization, and not a bug. It is mentioned here in
+the documentation in case you are confused as to why a debug message you've
+added to the code isn't always printed.
+
+#### Refresh notifications
+Some resources may choose to support receiving refresh notifications. In general
+these should be avoided if possible, but nevertheless, they do make sense in
+certain situations. Resources that support these need to verify if one was sent
+during the CheckApply phase of execution. This is accomplished by calling the
+`Refresh() bool` method of the resource, and inspecting the return value. This
+is only necessary if you plan to perform a refresh action. Refresh actions
+should still respect the `apply` variable, and no system changes should be made
+if it is `false`. Refresh notifications are generated by any resource when an
+action is applied by that resource and are transmitted through graph edges which
+have enabled their propagation. Resources that currently perform some refresh
+action include `svc`, `timer`, and `password`.
+
+#### Paired execution
+For many resources it is not uncommon to see `CheckApply` run twice in rapid
+succession. This is usually not a pathological occurrence, but rather a healthy
+pattern which is a consequence of the event system. When the state of the
+resource is incorrect, `CheckApply` will run to remedy the state. In response to
+having just changed the state, it is usually the case that this repair will
+trigger the `Watch` code! In response, a second `CheckApply` is triggered, which
+will likely find the state to now be correct.
+
+#### Summary
+* Anytime an error occurs during `CheckApply`, you should return `(false, err)`.
+* If the state is correct and no changes are needed, return `(true, nil)`.
+* You should only make changes to the system if `apply` is set to `true`.
+* After checking the state and possibly applying the fix, return `(false, nil)`.
+* Returning `(true, err)` is a programming error and will cause a `Fatal`.
+
+### Watch
+```golang
+Watch() error
+```
+
+`Watch` is a main loop that runs and sends messages when it detects that the
+state of the resource might have changed. To send a message you should write to
+the input event channel using the `Event` helper method. The Watch function
+should run continuously until a shutdown message is received. If at any time
+something goes wrong, you should return an error, and the `mgmt` engine will
+handle possibly restarting the main loop based on the `retry` meta parameters.
+
+It is better to send an event notification which turns out to be spurious, than
+to miss a possible event. Resources which can miss events are incorrect and need
+to be re-engineered so that this isn't the case. If you have an idea for a
+resource which would fit this criteria, but you can't find a solution, please
+contact the `mgmt` maintainers so that this problem can be investigated and a
+possible system level engineering fix can be found.
+
+You may have trouble deciding how much resource state checking should happen in
+the `Watch` loop versus deferring it all to the `CheckApply` method. You may
+want to put some simple fast path checking in `Watch` to avoid generating
+obviously spurious events, but in general it's best to keep the `Watch` method
+as simple as possible. Contact the `mgmt` maintainers if you're not sure.
+
+If the resource is activated in `polling` mode, the `Watch` method will not get
+executed. As a result, the resource must still work even if the main loop is not
+running.
+
+#### Select
+The lifetime of most resources `Watch` method should be spent in an infinite
+loop that is bounded by a `select` call. The `select` call is the point where
+our method hands back control to the engine (and the kernel) so that we can
+sleep until something of interest wakes us up. In this loop we must process
+events from the engine via the `<-obj.Events()` call, and receive events for our
+resource itself!
+
+#### Events
+If we receive an internal event from the `<-obj.Events()` method, we can read it
+with the ReadEvent helper function. This function tells us if we should shutdown
+our resource, and if we should generate an event. When we want to send an event,
+we use the `Event` helper function. It is also important to mark the resource
+state as `dirty` if we believe it might have changed. We do this with the
+`StateOK(false)` function.
+
+#### Startup
+Once the `Watch` function has finished starting up successfully, it is important
+to generate one event to notify the `mgmt` engine that we're now listening
+successfully, so that it can run an initial `CheckApply` to ensure we're safely
+tracking a healthy state and that we didn't miss anything when `Watch` was down
+or from before `mgmt` was running. It does this by calling the `Running` method.
+
+#### Converged
+The engine might be asked to shutdown when the entire state of the system has
+not seen any changes for some duration of time. The engine can determine this
+automatically, but each resource can block this if it is absolutely necessary.
+To do this, the `Watch` method should get the `ConvergedUID` handle that has
+been prepared for it by the engine. This is done by calling the `ConvergerUID`
+method on the resource object. The result can be used to set the converged
+status with `SetConverged`, and to notify when the particular timeout has been
+reached by waiting on `ConvergedTimer`.
+
+Instead of interacting with the `ConvergedUID` with these two methods, we can
+instead use the `StartTimer` and `ResetTimer` methods which accomplish the same
+thing, but provide a `select`-free interface for different coding situations.
+
+This particular facility is most likely not required for most resources. It may
+prove to be useful if a resource wants to start off a long operation, but avoid
+sending out erroneous `Event` messages to keep things alive until it finishes.
+
+#### Example
+```golang
+// Watch is the listener and main loop for this resource.
+func (obj *FooRes) Watch() error {
+	// setup the Foo resource
+	var err error
+	if err, obj.foo = OpenFoo(); err != nil {
+		return err // we couldn't startup
+	}
+	defer obj.whatever.CloseFoo() // shutdown our
+
+	// notify engine that we're running
+	if err := obj.Running(); err != nil {
+		return err // bubble up a NACK...
+	}
+
+	var send = false // send event?
+	var exit *error
+	for {
+		select {
+		case event := <-obj.Events():
+			// we avoid sending events on unpause
+			if exit, send = obj.ReadEvent(event); exit != nil {
+				return *exit // exit
+			}
+
+		// the actual events!
+		case event := <-obj.foo.Events:
+			if is_an_event {
+				send = true // used below
+				obj.StateOK(false) // dirty
+			}
+
+		// event errors
+		case err := <-obj.foo.Errors:
+			return err // will cause a retry or permanent failure
+		}
+
+		// do all our event sending all together to avoid duplicate msgs
+		if send {
+			send = false
+			obj.Event() // send the event!
+		}
+	}
+}
+```
+
+#### Summary
+* Remember to call the appropriate `converger` methods throughout the resource.
+* Remember to call `Startup` when the `Watch` is running successfully.
+* Remember to process internal events and shutdown promptly if asked to.
+* Ensure the design of your resource is well thought out.
+* Have a look at the existing resources for a rough idea of how this all works.
+
+### Compare
+```golang
+Compare(Res) bool
+```
+
+Each resource must have a `Compare` method. This takes as input another resource
+and must return whether they are identical or not. This is used for identifying
+if an existing resource can be used in place of a new one with a similar set of
+parameters. In particular, when switching from one graph to a new (possibly
+identical) graph, this avoids recomputing the state for resources which don't
+change or that are sufficiently similar that they don't need to be swapped out.
+
+In general if all the resource properties are identical, then they usually don't
+need to be changed. On occasion, not all of them need to be compared, in
+particular if they store some generated state, or if they aren't significant in
+some way.
+
+#### Example
+```golang
+// Compare two resources and return if they are equivalent.
+func (obj *FooRes) Compare(r Res) bool {
+	// we can only compare FooRes to others of the same resource kind
+	res, ok := r.(*FooRes)
+	if !ok {
+		return false
+	}
+	if !obj.BaseRes.Compare(res) { // call base Compare
+		return false
+	}
+	if obj.Name != res.Name {
+		return false
+	}
+
+	if obj.whatever != res.whatever {
+		return false
+	}
+	if obj.Flag != res.Flag {
+		return false
+	}
+
+	return true // they must match!
+}
+```
+
+### UIDs
+```golang
+UIDs() []ResUID
+```
+
+The `UIDs` method returns a list of `ResUID` interfaces that represent the
+particular resource uniquely. This is used with the AutoEdges API to determine
+if another resource can match a dependency to this one.
+
+### AutoEdges
+```golang
+AutoEdges() (AutoEdge, error)
+```
+
+This returns a struct that implements the `AutoEdge` interface. This struct
+is used to match other resources that might be relevant dependencies for this
+resource.
+
+### CollectPattern
+```golang
+CollectPattern() string
+```
+
+This is currently a stub and will be updated once the DSL is further along.
+
+### UnmarshalYAML
+```golang
+UnmarshalYAML(unmarshal func(interface{}) error) error // optional
+```
+
+This is optional, but recommended for any resource that will have a YAML
+accessible struct, and an entry in the `GraphConfig` struct. It is not required
+because to do so would mean that third-party or custom resources (such as those
+someone writes to use with `libmgmt`) would have to implement this needlessly.
+
+The signature intentionally matches what is required to satisfy the `go-yaml`
+[Unmarshaler](https://godoc.org/gopkg.in/yaml.v2#Unmarshaler) interface.
+
+#### Example
+```golang
+// UnmarshalYAML is the custom unmarshal handler for this struct.
+// It is primarily useful for setting the defaults.
+func (obj *FooRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type rawRes FooRes // indirection to avoid infinite recursion
+
+	def := obj.Default()     // get the default
+	res, ok := def.(*FooRes) // put in the right format
+	if !ok {
+		return fmt.Errorf("could not convert to FooRes")
+	}
+	raw := rawRes(*res) // convert; the defaults go here
+
+	if err := unmarshal(&raw); err != nil {
+		return err
+	}
+
+	*obj = FooRes(raw) // restore from indirection with type conversion!
+	return nil
+}
+```
+
+## Further considerations
+There is some additional information that any resource writer will need to know.
+Each issue is listed separately below!
+
+### Resource struct
+Each resource will implement methods as pointer receivers on a resource struct.
+The resource struct must include an anonymous reference to the `BaseRes` struct.
+The naming convention for resources is that they end with a `Res` suffix. If
+you'd like your resource to be accessible by the `YAML` graph API (GAPI), then
+you'll need to include the appropriate YAML fields as shown below.
+
+#### Example
+```golang
+type FooRes struct {
+	BaseRes `yaml:",inline"` // base properties
+
+	Whatever string `yaml:"whatever"` // you pick!
+	Bar int // no yaml, used as public output value for send/recv
+	Baz bool `yaml:"baz"` // something else
+
+	something string // some private field
+}
+```
+
+### YAML
+In addition to labelling your resource struct with YAML fields, you must also
+add an entry to the internal `GraphConfig` struct. It is a fairly straight
+forward one line patch.
+
+```golang
+type GraphConfig struct {
+// [snip...]
+	Resources struct {
+		Noop []*resources.NoopRes `yaml:"noop"`
+		File []*resources.FileRes `yaml:"file"`
+		// [snip...]
+		Foo []*resources.FooRes `yaml:"foo"` // tada :)
+	}
+}
+```
+
+It's also recommended that you add the [UnmarshalYAML](#unmarshalyaml) method to
+your resources so that unspecified values are given sane defaults.
+
+### Gob registration
+All resources must be registered with the `golang` _gob_ module so that they can
+be encoded and decoded. Make sure to include the following code snippet for this
+to work.
+
+```golang
+import "encoding/gob"
+func init() { // special golang method that runs once
+	gob.Register(&FooRes{}) // substitude your resource here
+}
+```
+
+## Automatic edges
+Automatic edges in `mgmt` are well described in [this article](https://ttboj.wordpress.com/2016/03/14/automatic-edges-in-mgmt/).
+The best example of this technique can be seen in the `svc` resource.
+Unfortunately no further documentation about this subject has been written. To
+expand this section, please send a patch! Please contact us if you'd like to
+work on a resource that uses this feature, or to add it to an existing one!
+
+## Automatic grouping
+Automatic grouping in `mgmt` is well described in [this article](https://ttboj.wordpress.com/2016/03/30/automatic-grouping-in-mgmt/).
+The best example of this technique can be seen in the `pkg` resource.
+Unfortunately no further documentation about this subject has been written. To
+expand this section, please send a patch! Please contact us if you'd like to
+work on a resource that uses this feature, or to add it to an existing one!
+
+
+## Send/Recv
+In `mgmt` there is a novel concept called _Send/Recv_. For some background,
+please [read the introductory article](https://ttboj.wordpress.com/2016/12/07/sendrecv-in-mgmt/).
+When using this feature, the engine will automatically send the user specified
+value to the intended destination without requiring any resource specific code.
+Any time that one of the destination values is changed, the engine automatically
+marks the resource state as `dirty`. To detect if a particular value was
+received, and if it changed (during this invocation of CheckApply) from the
+previous value, you can query the Recv parameter. It will contain a `map` of all
+the keys which can be received on, and the value has a `Changed` property which
+will indicate whether the value was updated on this particular `CheckApply`
+invocation. The type of the sending key must match that of the receiving one.
+This can _only_ be done inside of the `CheckApply` function!
+
+```golang
+// inside CheckApply, probably near the top
+if val, exists := obj.Recv["SomeKey"]; exists {
+	log.Printf("SomeKey was sent to us from: %s.%s", val.Res, val.Key)
+	if val.Changed {
+		log.Printf("SomeKey was just updated!")
+		// you may want to invalidate some local cache
+	}
+}
+```
+
+Astute readers will note that there isn't anything that prevents a user from
+sending an identically typed value to some arbitrary (public) key that the
+resource author hadn't considered! While this is true, resources should probably
+work within this problem space anyways. The rule of thumb is that any public
+parameter which is normally used in a resource can be used safely.
+
+One subtle scenario is that if a resource creates a local cache or stores a
+computation that depends on the value of a public parameter and will require
+invalidation should that public parameter change, then you must detect that
+scenario and invalidate the cache when it occurs. This *must* be processed
+before there is a possibility of failure in CheckApply, because if we fail (and
+possibly run again) the subsequent send->recv transfer might not have a new
+value to copy, and therefore we won't see this notification of change.
+Therefore, it is important to process these promptly, if they must not be lost,
+such as for cache invalidation.
+
+Remember, `Send/Recv` only changes your resource code if you cache state.
+
+## Composite resources
+Composite resources are resources which embed one or more existing resources.
+This is useful to prevent code duplication in higher level resource scenarios.
+The best example of this technique can be seen in the `nspawn` resource which
+can be seen to partially embed a `svc` resource, but without its `Watch`.
+Unfortunately no further documentation about this subject has been written. To
+expand this section, please send a patch! Please contact us if you'd like to
+work on a resource that uses this feature, or to add it to an existing one!
+
+## Frequently asked questions
+(Send your questions as a patch to this FAQ! I'll review it, merge it, and
+respond by commit with the answer.)
+
+### Can I write resources in a different language?
+Currently `golang` is the only supported language for built-in resources. We
+might consider allowing external resources to be imported in the future. This
+will likely require a language that can expose a C-like API, such as `python` or
+`ruby`. Custom `golang` resources are already possible when using mgmt as a lib.
+Higher level resource collections will be possible once the `mgmt` DSL is ready.
+
+### What new resource primitives need writing?
+There are still many ideas for new resources that haven't been written yet. If
+you'd like to contribute one, please contact us and tell us about your idea!
+
+### Where can I find more information about mgmt?
+Additional blog posts, videos and other material [is available!](https://github.com/purpleidea/mgmt/#on-the-web).
+
+## Suggestions
+If you have any ideas for API changes or other improvements to resource writing,
+please let us know! We're still pre 1.0 and pre 0.1 and happy to break API in
+order to get it right!

etcd/methods.go

@@ -0,0 +1,412 @@
+// Mgmt
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package etcd
+
+import (
+	"fmt"
+	"log"
+	"strconv"
+	"strings"
+
+	etcd "github.com/coreos/etcd/clientv3"
+	rpctypes "github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
+	etcdtypes "github.com/coreos/etcd/pkg/types"
+	context "golang.org/x/net/context"
+)
+
+// TODO: Could all these Etcd*(obj *EmbdEtcd, ...) functions which deal with the
+// interface between etcd paths and behaviour be grouped into a single struct ?
+
+// Nominate nominates a particular client to be a server (peer).
+func Nominate(obj *EmbdEtcd, hostname string, urls etcdtypes.URLs) error {
+	if obj.flags.Trace {
+		log.Printf("Trace: Etcd: Nominate(%v): %v", hostname, urls.String())
+		defer log.Printf("Trace: Etcd: Nominate(%v): Finished!", hostname)
+	}
+	// nominate someone to be a server
+	nominate := fmt.Sprintf("/%s/nominated/%s", NS, hostname)
+	ops := []etcd.Op{} // list of ops in this txn
+	if urls != nil {
+		ops = append(ops, etcd.OpPut(nominate, urls.String())) // TODO: add a TTL? (etcd.WithLease)
+
+	} else { // delete message if set to erase
+		ops = append(ops, etcd.OpDelete(nominate))
+	}
+
+	if _, err := obj.Txn(nil, ops, nil); err != nil {
+		return fmt.Errorf("nominate failed") // exit in progress?
+	}
+	return nil
+}
+
+// Nominated returns a urls map of nominated etcd server volunteers.
+// NOTE: I know 'nominees' might be more correct, but is less consistent here
+func Nominated(obj *EmbdEtcd) (etcdtypes.URLsMap, error) {
+	path := fmt.Sprintf("/%s/nominated/", NS)
+	keyMap, err := obj.Get(path, etcd.WithPrefix()) // map[string]string, bool
+	if err != nil {
+		return nil, fmt.Errorf("nominated isn't available: %v", err)
+	}
+	nominated := make(etcdtypes.URLsMap)
+	for key, val := range keyMap { // loop through directory of nominated
+		if !strings.HasPrefix(key, path) {
+			continue
+		}
+		name := key[len(path):] // get name of nominee
+		if val == "" {          // skip "erased" values
+			continue
+		}
+		urls, err := etcdtypes.NewURLs(strings.Split(val, ","))
+		if err != nil {
+			return nil, fmt.Errorf("nominated data format error: %v", err)
+		}
+		nominated[name] = urls // add to map
+		if obj.flags.Debug {
+			log.Printf("Etcd: Nominated(%v): %v", name, val)
+		}
+	}
+	return nominated, nil
+}
+
+// Volunteer offers yourself up to be a server if needed.
+func Volunteer(obj *EmbdEtcd, urls etcdtypes.URLs) error {
+	if obj.flags.Trace {
+		log.Printf("Trace: Etcd: Volunteer(%v): %v", obj.hostname, urls.String())
+		defer log.Printf("Trace: Etcd: Volunteer(%v): Finished!", obj.hostname)
+	}
+	// volunteer to be a server
+	volunteer := fmt.Sprintf("/%s/volunteers/%s", NS, obj.hostname)
+	ops := []etcd.Op{} // list of ops in this txn
+	if urls != nil {
+		// XXX: adding a TTL is crucial! (i think)
+		ops = append(ops, etcd.OpPut(volunteer, urls.String())) // value is usually a peer "serverURL"
+
+	} else { // delete message if set to erase
+		ops = append(ops, etcd.OpDelete(volunteer))
+	}
+
+	if _, err := obj.Txn(nil, ops, nil); err != nil {
+		return fmt.Errorf("volunteering failed") // exit in progress?
+	}
+	return nil
+}
+
+// Volunteers returns a urls map of available etcd server volunteers.
+func Volunteers(obj *EmbdEtcd) (etcdtypes.URLsMap, error) {
+	if obj.flags.Trace {
+		log.Printf("Trace: Etcd: Volunteers()")
+		defer log.Printf("Trace: Etcd: Volunteers(): Finished!")
+	}
+	path := fmt.Sprintf("/%s/volunteers/", NS)
+	keyMap, err := obj.Get(path, etcd.WithPrefix())
+	if err != nil {
+		return nil, fmt.Errorf("volunteers aren't available: %v", err)
+	}
+	volunteers := make(etcdtypes.URLsMap)
+	for key, val := range keyMap { // loop through directory of volunteers
+		if !strings.HasPrefix(key, path) {
+			continue
+		}
+		name := key[len(path):] // get name of volunteer
+		if val == "" {          // skip "erased" values
+			continue
+		}
+		urls, err := etcdtypes.NewURLs(strings.Split(val, ","))
+		if err != nil {
+			return nil, fmt.Errorf("volunteers data format error: %v", err)
+		}
+		volunteers[name] = urls // add to map
+		if obj.flags.Debug {
+			log.Printf("Etcd: Volunteer(%v): %v", name, val)
+		}
+	}
+	return volunteers, nil
+}
+
+// AdvertiseEndpoints advertises the list of available client endpoints.
+func AdvertiseEndpoints(obj *EmbdEtcd, urls etcdtypes.URLs) error {
+	if obj.flags.Trace {
+		log.Printf("Trace: Etcd: AdvertiseEndpoints(%v): %v", obj.hostname, urls.String())
+		defer log.Printf("Trace: Etcd: AdvertiseEndpoints(%v): Finished!", obj.hostname)
+	}
+	// advertise endpoints
+	endpoints := fmt.Sprintf("/%s/endpoints/%s", NS, obj.hostname)
+	ops := []etcd.Op{} // list of ops in this txn
+	if urls != nil {
+		// TODO: add a TTL? (etcd.WithLease)
+		ops = append(ops, etcd.OpPut(endpoints, urls.String())) // value is usually a "clientURL"
+
+	} else { // delete message if set to erase
+		ops = append(ops, etcd.OpDelete(endpoints))
+	}
+
+	if _, err := obj.Txn(nil, ops, nil); err != nil {
+		return fmt.Errorf("endpoint advertising failed") // exit in progress?
+	}
+	return nil
+}
+
+// Endpoints returns a urls map of available etcd server endpoints.
+func Endpoints(obj *EmbdEtcd) (etcdtypes.URLsMap, error) {
+	if obj.flags.Trace {
+		log.Printf("Trace: Etcd: Endpoints()")
+		defer log.Printf("Trace: Etcd: Endpoints(): Finished!")
+	}
+	path := fmt.Sprintf("/%s/endpoints/", NS)
+	keyMap, err := obj.Get(path, etcd.WithPrefix())
+	if err != nil {
+		return nil, fmt.Errorf("endpoints aren't available: %v", err)
+	}
+	endpoints := make(etcdtypes.URLsMap)
+	for key, val := range keyMap { // loop through directory of endpoints
+		if !strings.HasPrefix(key, path) {
+			continue
+		}
+		name := key[len(path):] // get name of volunteer
+		if val == "" {          // skip "erased" values
+			continue
+		}
+		urls, err := etcdtypes.NewURLs(strings.Split(val, ","))
+		if err != nil {
+			return nil, fmt.Errorf("endpoints data format error: %v", err)
+		}
+		endpoints[name] = urls // add to map
+		if obj.flags.Debug {
+			log.Printf("Etcd: Endpoint(%v): %v", name, val)
+		}
+	}
+	return endpoints, nil
+}
+
+// SetHostnameConverged sets whether a specific hostname is converged.
+func SetHostnameConverged(obj *EmbdEtcd, hostname string, isConverged bool) error {
+	if obj.flags.Trace {
+		log.Printf("Trace: Etcd: SetHostnameConverged(%s): %v", hostname, isConverged)
+		defer log.Printf("Trace: Etcd: SetHostnameConverged(%v): Finished!", hostname)
+	}
+	converged := fmt.Sprintf("/%s/converged/%s", NS, hostname)
+	op := []etcd.Op{etcd.OpPut(converged, fmt.Sprintf("%t", isConverged))}
+	if _, err := obj.Txn(nil, op, nil); err != nil { // TODO: do we need a skipConv flag here too?
+		return fmt.Errorf("set converged failed") // exit in progress?
+	}
+	return nil
+}
+
+// HostnameConverged returns a map of every hostname's converged state.
+func HostnameConverged(obj *EmbdEtcd) (map[string]bool, error) {
+	if obj.flags.Trace {
+		log.Printf("Trace: Etcd: HostnameConverged()")
+		defer log.Printf("Trace: Etcd: HostnameConverged(): Finished!")
+	}
+	path := fmt.Sprintf("/%s/converged/", NS)
+	keyMap, err := obj.ComplexGet(path, true, etcd.WithPrefix()) // don't un-converge
+	if err != nil {
+		return nil, fmt.Errorf("converged values aren't available: %v", err)
+	}
+	converged := make(map[string]bool)
+	for key, val := range keyMap { // loop through directory...
+		if !strings.HasPrefix(key, path) {
+			continue
+		}
+		name := key[len(path):] // get name of key
+		if val == "" {          // skip "erased" values
+			continue
+		}
+		b, err := strconv.ParseBool(val)
+		if err != nil {
+			return nil, fmt.Errorf("converged data format error: %v", err)
+		}
+		converged[name] = b // add to map
+	}
+	return converged, nil
+}
+
+// AddHostnameConvergedWatcher adds a watcher with a callback that runs on
+// hostname state changes.
+func AddHostnameConvergedWatcher(obj *EmbdEtcd, callbackFn func(map[string]bool) error) (func(), error) {
+	path := fmt.Sprintf("/%s/converged/", NS)
+	internalCbFn := func(re *RE) error {
+		// TODO: get the value from the response, and apply delta...
+		// for now, just run a get operation which is easier to code!
+		m, err := HostnameConverged(obj)
+		if err != nil {
+			return err
+		}
+		return callbackFn(m) // call my function
+	}
+	return obj.AddWatcher(path, internalCbFn, true, true, etcd.WithPrefix()) // no block and no converger reset
+}
+
+// SetClusterSize sets the ideal target cluster size of etcd peers.
+func SetClusterSize(obj *EmbdEtcd, value uint16) error {
+	if obj.flags.Trace {
+		log.Printf("Trace: Etcd: SetClusterSize(): %v", value)
+		defer log.Printf("Trace: Etcd: SetClusterSize(): Finished!")
+	}
+	key := fmt.Sprintf("/%s/idealClusterSize", NS)
+
+	if err := obj.Set(key, strconv.FormatUint(uint64(value), 10)); err != nil {
+		return fmt.Errorf("function SetClusterSize failed: %v", err) // exit in progress?
+	}
+	return nil
+}
+
+// GetClusterSize gets the ideal target cluster size of etcd peers.
+func GetClusterSize(obj *EmbdEtcd) (uint16, error) {
+	key := fmt.Sprintf("/%s/idealClusterSize", NS)
+	keyMap, err := obj.Get(key)
+	if err != nil {
+		return 0, fmt.Errorf("function GetClusterSize failed: %v", err)
+	}
+
+	val, exists := keyMap[key]
+	if !exists || val == "" {
+		return 0, fmt.Errorf("function GetClusterSize failed: %v", err)
+	}
+
+	v, err := strconv.ParseUint(val, 10, 16)
+	if err != nil {
+		return 0, fmt.Errorf("function GetClusterSize failed: %v", err)
+	}
+	return uint16(v), nil
+}
+
+// MemberAdd adds a member to the cluster.
+func MemberAdd(obj *EmbdEtcd, peerURLs etcdtypes.URLs) (*etcd.MemberAddResponse, error) {
+	//obj.Connect(false) // TODO: ?
+	ctx := context.Background()
+	var response *etcd.MemberAddResponse
+	var err error
+	for {
+		if obj.exiting { // the exit signal has been sent!
+			return nil, fmt.Errorf("exiting etcd")
+		}
+		obj.rLock.RLock()
+		response, err = obj.client.MemberAdd(ctx, peerURLs.StringSlice())
+		obj.rLock.RUnlock()
+		if err == nil {
+			break
+		}
+		if ctx, err = obj.CtxError(ctx, err); err != nil {
+			return nil, err
+		}
+	}
+	return response, nil
+}
+
+// MemberRemove removes a member by mID and returns if it worked, and also
+// if there was an error. This is because it might have run without error, but
+// the member wasn't found, for example.
+func MemberRemove(obj *EmbdEtcd, mID uint64) (bool, error) {
+	//obj.Connect(false) // TODO: ?
+	ctx := context.Background()
+	for {
+		if obj.exiting { // the exit signal has been sent!
+			return false, fmt.Errorf("exiting etcd")
+		}
+		obj.rLock.RLock()
+		_, err := obj.client.MemberRemove(ctx, mID)
+		obj.rLock.RUnlock()
+		if err == nil {
+			break
+		} else if err == rpctypes.ErrMemberNotFound {
+			// if we get this, member already shut itself down :)
+			return false, nil
+		}
+		if ctx, err = obj.CtxError(ctx, err); err != nil {
+			return false, err
+		}
+	}
+	return true, nil
+}
+
+// Members returns information on cluster membership.
+// The member ID's are the keys, because an empty names means unstarted!
+// TODO: consider queueing this through the main loop with CtxError(ctx, err)
+func Members(obj *EmbdEtcd) (map[uint64]string, error) {
+	//obj.Connect(false) // TODO: ?
+	ctx := context.Background()
+	var response *etcd.MemberListResponse
+	var err error
+	for {
+		if obj.exiting { // the exit signal has been sent!
+			return nil, fmt.Errorf("exiting etcd")
+		}
+		obj.rLock.RLock()
+		if obj.flags.Trace {
+			log.Printf("Trace: Etcd: Members(): Endpoints are: %v", obj.client.Endpoints())
+		}
+		response, err = obj.client.MemberList(ctx)
+		obj.rLock.RUnlock()
+		if err == nil {
+			break
+		}
+		if ctx, err = obj.CtxError(ctx, err); err != nil {
+			return nil, err
+		}
+	}
+
+	members := make(map[uint64]string)
+	for _, x := range response.Members {
+		members[x.ID] = x.Name // x.Name will be "" if unstarted!
+	}
+	return members, nil
+}
+
+// Leader returns the current leader of the etcd server cluster.
+func Leader(obj *EmbdEtcd) (string, error) {
+	//obj.Connect(false) // TODO: ?
+	var err error
+	membersMap := make(map[uint64]string)
+	if membersMap, err = Members(obj); err != nil {
+		return "", err
+	}
+	addresses := obj.LocalhostClientURLs() // heuristic, but probably correct
+	if len(addresses) == 0 {
+		// probably a programming error...
+		return "", fmt.Errorf("programming error")
+	}
+	endpoint := addresses[0].Host // FIXME: arbitrarily picked the first one
+
+	// part two
+	ctx := context.Background()
+	var response *etcd.StatusResponse
+	for {
+		if obj.exiting { // the exit signal has been sent!
+			return "", fmt.Errorf("exiting etcd")
+		}
+
+		obj.rLock.RLock()
+		response, err = obj.client.Maintenance.Status(ctx, endpoint)
+		obj.rLock.RUnlock()
+		if err == nil {
+			break
+		}
+		if ctx, err = obj.CtxError(ctx, err); err != nil {
+			return "", err
+		}
+	}
+
+	// isLeader: response.Header.MemberId == response.Leader
+	for id, name := range membersMap {
+		if id == response.Leader {
+			return name, nil
+		}
+	}
+	return "", fmt.Errorf("members map is not current") // not found
+}

etcd/resources.go

@@ -0,0 +1,182 @@
+// Mgmt
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package etcd
+
+import (
+	"fmt"
+	"log"
+	"strings"
+
+	"github.com/purpleidea/mgmt/resources"
+	"github.com/purpleidea/mgmt/util"
+
+	etcd "github.com/coreos/etcd/clientv3"
+)
+
+// WatchResources returns a channel that outputs events when exported resources
+// change.
+// TODO: Filter our watch (on the server side if possible) based on the
+// collection prefixes and filters that we care about...
+func WatchResources(obj *EmbdEtcd) chan error {
+	ch := make(chan error, 1) // buffer it so we can measure it
+	path := fmt.Sprintf("/%s/exported/", NS)
+	callback := func(re *RE) error {
+		// TODO: is this even needed? it used to happen on conn errors
+		log.Printf("Etcd: Watch: Path: %v", path) // event
+		if re == nil || re.response.Canceled {
+			return fmt.Errorf("watch is empty") // will cause a CtxError+retry
+		}
+		// we normally need to check if anything changed since the last
+		// event, since a set (export) with no changes still causes the
+		// watcher to trigger and this would cause an infinite loop. we
+		// don't need to do this check anymore because we do the export
+		// transactionally, and only if a change is needed. since it is
+		// atomic, all the changes arrive together which avoids dupes!!
+		if len(ch) == 0 { // send event only if one isn't pending
+			// this check avoids multiple events all queueing up and then
+			// being released continuously long after the changes stopped
+			// do not block!
+			ch <- nil // event
+		}
+		return nil
+	}
+	_, _ = obj.AddWatcher(path, callback, true, false, etcd.WithPrefix()) // no need to check errors
+	return ch
+}
+
+// SetResources exports all of the resources which we pass in to etcd.
+func SetResources(obj *EmbdEtcd, hostname string, resourceList []resources.Res) error {
+	// key structure is /$NS/exported/$hostname/resources/$uid = $data
+
+	var kindFilter []string // empty to get from everyone
+	hostnameFilter := []string{hostname}
+	// this is not a race because we should only be reading keys which we
+	// set, and there should not be any contention with other hosts here!
+	originals, err := GetResources(obj, hostnameFilter, kindFilter)
+	if err != nil {
+		return err
+	}
+
+	if len(originals) == 0 && len(resourceList) == 0 { // special case of no add or del
+		return nil
+	}
+
+	ifs := []etcd.Cmp{} // list matching the desired state
+	ops := []etcd.Op{}  // list of ops in this transaction
+	for _, res := range resourceList {
+		if res.GetKind() == "" {
+			log.Fatalf("Etcd: SetResources: Error: Empty kind: %v", res.GetName())
+		}
+		uid := fmt.Sprintf("%s/%s", res.GetKind(), res.GetName())
+		path := fmt.Sprintf("/%s/exported/%s/resources/%s", NS, hostname, uid)
+		if data, err := resources.ResToB64(res); err == nil {
+			ifs = append(ifs, etcd.Compare(etcd.Value(path), "=", data)) // desired state
+			ops = append(ops, etcd.OpPut(path, data))
+		} else {
+			return fmt.Errorf("can't convert to B64: %v", err)
+		}
+	}
+
+	match := func(res resources.Res, resourceList []resources.Res) bool { // helper lambda
+		for _, x := range resourceList {
+			if res.GetKind() == x.GetKind() && res.GetName() == x.GetName() {
+				return true
+			}
+		}
+		return false
+	}
+
+	hasDeletes := false
+	// delete old, now unused resources here...
+	for _, res := range originals {
+		if res.GetKind() == "" {
+			log.Fatalf("Etcd: SetResources: Error: Empty kind: %v", res.GetName())
+		}
+		uid := fmt.Sprintf("%s/%s", res.GetKind(), res.GetName())
+		path := fmt.Sprintf("/%s/exported/%s/resources/%s", NS, hostname, uid)
+
+		if match(res, resourceList) { // if we match, no need to delete!
+			continue
+		}
+
+		ops = append(ops, etcd.OpDelete(path))
+
+		hasDeletes = true
+	}
+
+	// if everything is already correct, do nothing, otherwise, run the ops!
+	// it's important to do this in one transaction, and atomically, because
+	// this way, we only generate one watch event, and only when it's needed
+	if hasDeletes { // always run, ifs don't matter
+		_, err = obj.Txn(nil, ops, nil) // TODO: does this run? it should!
+	} else {
+		_, err = obj.Txn(ifs, nil, ops) // TODO: do we need to look at response?
+	}
+	return err
+}
+
+// GetResources collects all of the resources which match a filter from etcd.
+// If the kindfilter or hostnameFilter is empty, then it assumes no filtering...
+// TODO: Expand this with a more powerful filter based on what we eventually
+// support in our collect DSL. Ideally a server side filter like WithFilter()
+// We could do this if the pattern was /$NS/exported/$kind/$hostname/$uid = $data.
+func GetResources(obj *EmbdEtcd, hostnameFilter, kindFilter []string) ([]resources.Res, error) {
+	// key structure is /$NS/exported/$hostname/resources/$uid = $data
+	path := fmt.Sprintf("/%s/exported/", NS)
+	resourceList := []resources.Res{}
+	keyMap, err := obj.Get(path, etcd.WithPrefix(), etcd.WithSort(etcd.SortByKey, etcd.SortAscend))
+	if err != nil {
+		return nil, fmt.Errorf("could not get resources: %v", err)
+	}
+	for key, val := range keyMap {
+		if !strings.HasPrefix(key, path) { // sanity check
+			continue
+		}
+
+		str := strings.Split(key[len(path):], "/")
+		if len(str) != 4 {
+			return nil, fmt.Errorf("unexpected chunk count")
+		}
+		hostname, r, kind, name := str[0], str[1], str[2], str[3]
+		if r != "resources" {
+			return nil, fmt.Errorf("unexpected chunk pattern")
+		}
+		if kind == "" {
+			return nil, fmt.Errorf("unexpected kind chunk")
+		}
+
+		// FIXME: ideally this would be a server side filter instead!
+		if len(hostnameFilter) > 0 && !util.StrInList(hostname, hostnameFilter) {
+			continue
+		}
+
+		// FIXME: ideally this would be a server side filter instead!
+		if len(kindFilter) > 0 && !util.StrInList(kind, kindFilter) {
+			continue
+		}
+
+		if obj, err := resources.B64ToRes(val); err == nil {
+			obj.SetKind(kind) // cheap init
+			log.Printf("Etcd: Get: (Hostname, Kind, Name): (%s, %s, %s)", hostname, kind, name)
+			resourceList = append(resourceList, obj)
+		} else {
+			return nil, fmt.Errorf("can't convert from B64: %v", err)
+		}
+	}
+	return resourceList, nil
+}

etcd/str.go

@@ -0,0 +1,105 @@
+// Mgmt
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package etcd
+
+import (
+	"errors"
+	"fmt"
+
+	etcd "github.com/coreos/etcd/clientv3"
+	errwrap "github.com/pkg/errors"
+)
+
+// ErrNotExist is returned when GetStr can not find the requested key.
+// TODO: https://dave.cheney.net/2016/04/07/constant-errors
+var ErrNotExist = errors.New("errNotExist")
+
+// WatchStr returns a channel which spits out events on key activity.
+// FIXME: It should close the channel when it's done, and spit out errors when
+// something goes wrong.
+func WatchStr(obj *EmbdEtcd, key string) chan error {
+	// new key structure is /$NS/strings/$key = $data
+	path := fmt.Sprintf("/%s/strings/%s", NS, key)
+	ch := make(chan error, 1)
+	// FIXME: fix our API so that we get a close event on shutdown.
+	callback := func(re *RE) error {
+		// TODO: is this even needed? it used to happen on conn errors
+		//log.Printf("Etcd: Watch: Path: %v", path) // event
+		if re == nil || re.response.Canceled {
+			return fmt.Errorf("watch is empty") // will cause a CtxError+retry
+		}
+		if len(ch) == 0 { // send event only if one isn't pending
+			ch <- nil // event
+		}
+		return nil
+	}
+	_, _ = obj.AddWatcher(path, callback, true, false, etcd.WithPrefix()) // no need to check errors
+	return ch
+}
+
+// GetStr collects the string which matches a global namespace in etcd.
+func GetStr(obj *EmbdEtcd, key string) (string, error) {
+	// new key structure is /$NS/strings/$key = $data
+	path := fmt.Sprintf("/%s/strings/%s", NS, key)
+	keyMap, err := obj.Get(path, etcd.WithPrefix())
+	if err != nil {
+		return "", errwrap.Wrapf(err, "could not get strings in: %s", key)
+	}
+
+	if len(keyMap) == 0 {
+		return "", ErrNotExist
+	}
+
+	if count := len(keyMap); count != 1 {
+		return "", fmt.Errorf("returned %d entries", count)
+	}
+
+	val, exists := keyMap[path]
+	if !exists {
+		return "", fmt.Errorf("path `%s` is missing", path)
+	}
+
+	//log.Printf("Etcd: GetStr(%s): %s", key, val)
+	return val, nil
+}
+
+// SetStr sets a key and hostname pair to a certain value. If the value is
+// nil, then it deletes the key. Otherwise the value should point to a string.
+// TODO: TTL or delete disconnect?
+func SetStr(obj *EmbdEtcd, key string, data *string) error {
+	// key structure is /$NS/strings/$key = $data
+	path := fmt.Sprintf("/%s/strings/%s", NS, key)
+	ifs := []etcd.Cmp{} // list matching the desired state
+	ops := []etcd.Op{}  // list of ops in this transaction (then)
+	els := []etcd.Op{}  // list of ops in this transaction (else)
+	if data == nil {    // perform a delete
+		// TODO: use https://github.com/coreos/etcd/pull/7417 if merged
+		//ifs = append(ifs, etcd.KeyExists(path))
+		ifs = append(ifs, etcd.Compare(etcd.Version(path), ">", 0))
+		ops = append(ops, etcd.OpDelete(path))
+	} else {
+		data := *data                                                // get the real value
+		ifs = append(ifs, etcd.Compare(etcd.Value(path), "=", data)) // desired state
+		els = append(els, etcd.OpPut(path, data))
+	}
+
+	// it's important to do this in one transaction, and atomically, because
+	// this way, we only generate one watch event, and only when it's needed
+	_, err := obj.Txn(ifs, ops, els) // TODO: do we need to look at response?
+	return errwrap.Wrapf(err, "could not set strings in: %s", key)
+}

etcd/strmap.go

@@ -0,0 +1,115 @@
+// Mgmt
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package etcd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/purpleidea/mgmt/util"
+
+	etcd "github.com/coreos/etcd/clientv3"
+	errwrap "github.com/pkg/errors"
+)
+
+// WatchStrMap returns a channel which spits out events on key activity.
+// FIXME: It should close the channel when it's done, and spit out errors when
+// something goes wrong.
+func WatchStrMap(obj *EmbdEtcd, key string) chan error {
+	// new key structure is /$NS/strings/$key/$hostname = $data
+	path := fmt.Sprintf("/%s/strings/%s", NS, key)
+	ch := make(chan error, 1)
+	// FIXME: fix our API so that we get a close event on shutdown.
+	callback := func(re *RE) error {
+		// TODO: is this even needed? it used to happen on conn errors
+		//log.Printf("Etcd: Watch: Path: %v", path) // event
+		if re == nil || re.response.Canceled {
+			return fmt.Errorf("watch is empty") // will cause a CtxError+retry
+		}
+		if len(ch) == 0 { // send event only if one isn't pending
+			ch <- nil // event
+		}
+		return nil
+	}
+	_, _ = obj.AddWatcher(path, callback, true, false, etcd.WithPrefix()) // no need to check errors
+	return ch
+}
+
+// GetStrMap collects all of the strings which match a namespace in etcd.
+func GetStrMap(obj *EmbdEtcd, hostnameFilter []string, key string) (map[string]string, error) {
+	// old key structure is /$NS/strings/$hostname/$key = $data
+	// new key structure is /$NS/strings/$key/$hostname = $data
+	// FIXME: if we have the $key as the last token (old key structure), we
+	// can allow the key to contain the slash char, otherwise we need to
+	// verify that one isn't present in the input string.
+	path := fmt.Sprintf("/%s/strings/%s", NS, key)
+	keyMap, err := obj.Get(path, etcd.WithPrefix(), etcd.WithSort(etcd.SortByKey, etcd.SortAscend))
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "could not get strings in: %s", key)
+	}
+	result := make(map[string]string)
+	for key, val := range keyMap {
+		if !strings.HasPrefix(key, path) { // sanity check
+			continue
+		}
+
+		str := strings.Split(key[len(path):], "/")
+		if len(str) != 2 {
+			return nil, fmt.Errorf("unexpected chunk count of %d", len(str))
+		}
+		_, hostname := str[0], str[1]
+
+		if hostname == "" {
+			return nil, fmt.Errorf("unexpected chunk length of %d", len(hostname))
+		}
+
+		// FIXME: ideally this would be a server side filter instead!
+		if len(hostnameFilter) > 0 && !util.StrInList(hostname, hostnameFilter) {
+			continue
+		}
+		//log.Printf("Etcd: GetStr(%s): (Hostname, Data): (%s, %s)", key, hostname, val)
+		result[hostname] = val
+	}
+	return result, nil
+}
+
+// SetStrMap sets a key and hostname pair to a certain value. If the value is
+// nil, then it deletes the key. Otherwise the value should point to a string.
+// TODO: TTL or delete disconnect?
+func SetStrMap(obj *EmbdEtcd, hostname, key string, data *string) error {
+	// key structure is /$NS/strings/$key/$hostname = $data
+	path := fmt.Sprintf("/%s/strings/%s/%s", NS, key, hostname)
+	ifs := []etcd.Cmp{} // list matching the desired state
+	ops := []etcd.Op{}  // list of ops in this transaction (then)
+	els := []etcd.Op{}  // list of ops in this transaction (else)
+	if data == nil {    // perform a delete
+		// TODO: use https://github.com/coreos/etcd/pull/7417 if merged
+		//ifs = append(ifs, etcd.KeyExists(path))
+		ifs = append(ifs, etcd.Compare(etcd.Version(path), ">", 0))
+		ops = append(ops, etcd.OpDelete(path))
+	} else {
+		data := *data                                                // get the real value
+		ifs = append(ifs, etcd.Compare(etcd.Value(path), "=", data)) // desired state
+		els = append(els, etcd.OpPut(path, data))
+	}
+
+	// it's important to do this in one transaction, and atomically, because
+	// this way, we only generate one watch event, and only when it's needed
+	_, err := obj.Txn(ifs, ops, els) // TODO: do we need to look at response?
+	return errwrap.Wrapf(err, "could not set strings in: %s", key)
+}

etcd/world.go

@@ -0,0 +1,95 @@
+// Mgmt
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package etcd
+
+import (
+	"github.com/purpleidea/mgmt/resources"
+)
+
+// World is an etcd backed implementation of the World interface.
+type World struct {
+	Hostname string // uuid for the consumer of these
+	EmbdEtcd *EmbdEtcd
+}
+
+// ResWatch returns a channel which spits out events on possible exported
+// resource changes.
+func (obj *World) ResWatch() chan error {
+	return WatchResources(obj.EmbdEtcd)
+}
+
+// ResExport exports a list of resources under our hostname namespace.
+// Subsequent calls replace the previously set collection atomically.
+func (obj *World) ResExport(resourceList []resources.Res) error {
+	return SetResources(obj.EmbdEtcd, obj.Hostname, resourceList)
+}
+
+// ResCollect gets the collection of exported resources which match the filter.
+// It does this atomically so that a call always returns a complete collection.
+func (obj *World) ResCollect(hostnameFilter, kindFilter []string) ([]resources.Res, error) {
+	// XXX: should we be restricted to retrieving resources that were
+	// exported with a tag that allows or restricts our hostname? We could
+	// enforce that here if the underlying API supported it... Add this?
+	return GetResources(obj.EmbdEtcd, hostnameFilter, kindFilter)
+}
+
+// StrWatch returns a channel which spits out events on possible string changes.
+func (obj *World) StrWatch(namespace string) chan error {
+	return WatchStr(obj.EmbdEtcd, namespace)
+}
+
+// StrIsNotExist returns whether the error from StrGet is a key missing error.
+func (obj *World) StrIsNotExist(err error) bool {
+	return err == ErrNotExist
+}
+
+// StrGet returns the value for the the given namespace.
+func (obj *World) StrGet(namespace string) (string, error) {
+	return GetStr(obj.EmbdEtcd, namespace)
+}
+
+// StrSet sets the namespace value to a particular string.
+func (obj *World) StrSet(namespace, value string) error {
+	return SetStr(obj.EmbdEtcd, namespace, &value)
+}
+
+// StrDel deletes the value in a particular namespace.
+func (obj *World) StrDel(namespace string) error {
+	return SetStr(obj.EmbdEtcd, namespace, nil)
+}
+
+// StrMapWatch returns a channel which spits out events on possible string changes.
+func (obj *World) StrMapWatch(namespace string) chan error {
+	return WatchStrMap(obj.EmbdEtcd, namespace)
+}
+
+// StrMapGet returns a map of hostnames to values in the given namespace.
+func (obj *World) StrMapGet(namespace string) (map[string]string, error) {
+	return GetStrMap(obj.EmbdEtcd, []string{}, namespace)
+}
+
+// StrMapSet sets the namespace value to a particular string under the identity
+// of its own hostname.
+func (obj *World) StrMapSet(namespace, value string) error {
+	return SetStrMap(obj.EmbdEtcd, obj.Hostname, namespace, &value)
+}
+
+// StrMapDel deletes the value in a particular namespace.
+func (obj *World) StrMapDel(namespace string) error {
+	return SetStrMap(obj.EmbdEtcd, obj.Hostname, namespace, nil)
+}

event/event.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -22,14 +22,14 @@ import (
 	"fmt"
 )
 
-//go:generate stringer -type=EventName -output=eventname_stringer.go
+//go:generate stringer -type=Kind -output=kind_stringer.go
 
-// EventName represents the type of event being passed.
-type EventName int
+// Kind represents the type of event being passed.
+type Kind int
 
-// The different event names are used in different contexts.
+// The different event kinds are used in different contexts.
 const (
-	EventNil EventName = iota
+	EventNil Kind = iota
 	EventExit
 	EventStart
 	EventPause
@@ -43,11 +43,10 @@ type Resp chan error
 
 // Event is the main struct that stores event information and responses.
 type Event struct {
-	Name EventName
+	Kind Kind
 	Resp Resp // channel to send an ack response on, nil to skip
 	//Wg   *sync.WaitGroup // receiver barrier to Wait() for everyone else on
-	Msg      string // some words for fun
-	Activity bool   // did something interesting happen?
+	Err error // store an error in our event
 }
 
 // ACK sends a single acknowledgement on the channel if one was requested.
@@ -80,7 +79,7 @@ func NewResp() Resp {
 // ACK sends a true value to resp.
 func (resp Resp) ACK() {
 	if resp != nil {
-		resp <- nil
+		resp <- nil // TODO: close instead?
 	}
 }
 
@@ -114,7 +113,7 @@ func (resp Resp) ACKWait() {
 	}
 }
 
-// GetActivity returns the activity value.
-func (event *Event) GetActivity() bool {
-	return event.Activity
+// Error returns the stored error value.
+func (event *Event) Error() error {
+	return event.Err
 }

examples/augeas1.yaml

@@ -0,0 +1,11 @@
+---
+graph: mygraph
+resources:
+  augeas:
+  - name: sshd_config
+    lens: Sshd.lns
+    file: "/etc/ssh/sshd_config"
+    sets:
+    - path: X11Forwarding
+      value: false
+edges:

examples/autoedges3.yaml

@@ -5,11 +5,13 @@ resources:
   - name: drbd-utils
     meta:
       autoedge: true
+      noop: true
     state: installed
   file:
   - name: file1
     meta:
       autoedge: true
+      noop: true
     path: "/etc/drbd.conf"
     content: |
       # this is an mgmt test
@@ -17,13 +19,14 @@ resources:
   - name: file2
     meta:
       autoedge: true
+      noop: true
     path: "/etc/drbd.d/"
-    content: |
-      i am a directory
+    source: /dev/null
     state: exists
   svc:
   - name: drbd
     meta:
       autoedge: true
+      noop: true
     state: stopped
 edges: []

examples/deep-dirs.yaml

@@ -0,0 +1,9 @@
+---
+graph: mygraph
+resources:
+  file:
+  - name: file1
+    path: "/tmp/mgmt/a/b/c/f1"
+    content: |
+      i am f1
+    state: exists

examples/etcd1a.yaml

@@ -2,15 +2,10 @@
 graph: mygraph
 resources:
   file:
-  - name: file1a
-    path: "/tmp/mgmtA/f1a"
+  - name: "@@filea"
+    path: "/tmp/mgmtA/fA"
     content: |
-      i am f1
-    state: exists
-  - name: "@@file2a"
-    path: "/tmp/mgmtA/f2a"
-    content: |
-      i am f2, exported from host A
+      i am fA, exported from host A
     state: exists
 collect:
 - kind: file

examples/etcd1b.yaml

@@ -2,15 +2,10 @@
 graph: mygraph
 resources:
   file:
-  - name: file1b
-    path: "/tmp/mgmtB/f1b"
+  - name: "@@fileb"
+    path: "/tmp/mgmtB/fB"
     content: |
-      i am f1
-    state: exists
-  - name: "@@file2b"
-    path: "/tmp/mgmtB/f2b"
-    content: |
-      i am f2, exported from host B
+      i am fB, exported from host B
     state: exists
 collect:
 - kind: file

examples/etcd1c.yaml

@@ -2,15 +2,10 @@
 graph: mygraph
 resources:
   file:
-  - name: file1c
-    path: "/tmp/mgmtC/f1c"
+  - name: "@@filec"
+    path: "/tmp/mgmtC/fC"
     content: |
-      i am f1
-    state: exists
-  - name: "@@file2c"
-    path: "/tmp/mgmtC/f2c"
-    content: |
-      i am f2, exported from host C
+      i am fC, exported from host C
     state: exists
 collect:
 - kind: file

examples/etcd1d.yaml

@@ -2,15 +2,10 @@
 graph: mygraph
 resources:
   file:
-  - name: file1d
-    path: "/tmp/mgmtD/f1d"
+  - name: "@@filed"
+    path: "/tmp/mgmtD/fD"
     content: |
-      i am f1
-    state: exists
-  - name: "@@file2d"
-    path: "/tmp/mgmtD/f2d"
-    content: |
-      i am f2, exported from host D
+      i am fD, exported from host D
     state: exists
 collect:
 - kind: file

examples/etcd1e.yaml

@@ -0,0 +1,13 @@
+---
+graph: mygraph
+resources:
+  file:
+  - name: "@@filee"
+    path: "/tmp/mgmtE/fE"
+    content: |
+      i am fE, exported from host E
+    state: exists
+collect:
+- kind: file
+  pattern: "/tmp/mgmtE/"
+edges: []

examples/exec3-sema.yaml

@@ -0,0 +1,67 @@
+---
+graph: parallel
+resources:
+  exec:
+  - name: pkg10
+    meta:
+      sema: ['mylock:1', 'otherlock:42']
+    cmd: sleep 10s
+    shell: ''
+    timeout: 0
+    watchcmd: ''
+    watchshell: ''
+    ifcmd: ''
+    ifshell: ''
+    pollint: 0
+    state: present
+  - name: svc10
+    meta:
+      sema: ['mylock:1']
+    cmd: sleep 10s
+    shell: ''
+    timeout: 0
+    watchcmd: ''
+    watchshell: ''
+    ifcmd: ''
+    ifshell: ''
+    pollint: 0
+    state: present
+  - name: exec10
+    meta:
+      sema: ['mylock:1']
+    cmd: sleep 10s
+    shell: ''
+    timeout: 0
+    watchcmd: ''
+    watchshell: ''
+    ifcmd: ''
+    ifshell: ''
+    pollint: 0
+    state: present
+  - name: pkg15
+    meta:
+      sema: ['mylock:1', 'otherlock:42']
+    cmd: sleep 15s
+    shell: ''
+    timeout: 0
+    watchcmd: ''
+    watchshell: ''
+    ifcmd: ''
+    ifshell: ''
+    pollint: 0
+    state: present
+edges:
+- name: e1
+  from:
+    kind: exec
+    name: pkg10
+  to:
+    kind: exec
+    name: svc10
+- name: e2
+  from:
+    kind: exec
+    name: svc10
+  to:
+    kind: exec
+    name: exec10

examples/file0.yaml

@@ -0,0 +1,10 @@
+---
+graph: mygraph
+resources:
+  file:
+  - name: file0
+    path: "/tmp/mgmt/f1"
+    content: |
+      i am f0
+    state: exists
+edges: []

examples/file3.yaml

@@ -1,14 +1,13 @@
 ---
 graph: mygraph
-comment: You can test Watch and CheckApply failures with chmod ugo-r and chmod ugo-w.
 resources:
   file:
   - name: file1
-    path: "/tmp/mgmt/f1"
     meta:
-      retry: 3
-      delay: 5000
+      limit: .inf
+      burst: 0
+    path: "/tmp/mgmt/hello"
     content: |
-      i am f1
+      i am a file
     state: exists
 edges: []

examples/file4.yaml

@@ -0,0 +1,10 @@
+---
+graph: mygraph
+resources:
+  file:
+  - name: file1
+    path: "/tmp/mgmt/hello"
+    content: |
+      i am a file
+    state: exists
+edges: []

examples/hostname.yaml

@@ -0,0 +1,7 @@
+---
+graph: mygraph
+resources:
+  hostname:
+    - name: Hostname Watcher @ TestHost
+      hostname: test.hostname.example.com
+edges: []

examples/kv1.yaml

@@ -0,0 +1,8 @@
+---
+graph: mygraph
+resources:
+  kv:
+  - name: kv1
+    key: "hello"
+    value: "world"
+edges: []

examples/kv2.yaml

@@ -0,0 +1,7 @@
+---
+graph: mygraph
+resources:
+  kv:
+  - name: kv1
+    key: "iamdeleted"
+edges: []

examples/kv3.yaml

@@ -0,0 +1,9 @@
+---
+graph: mygraph
+resources:
+  kv:
+  - name: kv1
+    key: "stage"
+    value: "3"
+    skiplessthan: true
+edges: []

examples/kv4.yaml

@@ -0,0 +1,31 @@
+---
+graph: mygraph
+resources:
+  kv:
+  - name: kv1
+    key: "stage"
+    value: "1"
+    skiplessthan: true
+  - name: kv2
+    key: "stage"
+    value: "2"
+    skiplessthan: true
+  - name: kv3
+    key: "stage"
+    value: "3"
+    skiplessthan: true
+edges:
+- name: e1
+  from:
+    kind: kv
+    name: kv1
+  to:
+    kind: kv
+    name: kv2
+- name: e2
+  from:
+    kind: kv
+    name: kv2
+  to:
+    kind: kv
+    name: kv3

examples/lib/exec-send-recv.go

@@ -0,0 +1,246 @@
+// libmgmt example of send->recv
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/purpleidea/mgmt/gapi"
+	mgmt "github.com/purpleidea/mgmt/lib"
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/resources"
+)
+
+// MyGAPI implements the main GAPI interface.
+type MyGAPI struct {
+	Name     string // graph name
+	Interval uint   // refresh interval, 0 to never refresh
+
+	data        gapi.Data
+	initialized bool
+	closeChan   chan struct{}
+	wg          sync.WaitGroup // sync group for tunnel go routines
+}
+
+// NewMyGAPI creates a new MyGAPI struct and calls Init().
+func NewMyGAPI(data gapi.Data, name string, interval uint) (*MyGAPI, error) {
+	obj := &MyGAPI{
+		Name:     name,
+		Interval: interval,
+	}
+	return obj, obj.Init(data)
+}
+
+// Init initializes the MyGAPI struct.
+func (obj *MyGAPI) Init(data gapi.Data) error {
+	if obj.initialized {
+		return fmt.Errorf("already initialized")
+	}
+	if obj.Name == "" {
+		return fmt.Errorf("the graph name must be specified")
+	}
+	obj.data = data // store for later
+	obj.closeChan = make(chan struct{})
+	obj.initialized = true
+	return nil
+}
+
+// Graph returns a current Graph.
+func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
+	if !obj.initialized {
+		return nil, fmt.Errorf("libmgmt: MyGAPI is not initialized")
+	}
+
+	g, err := pgraph.NewGraph(obj.Name)
+	if err != nil {
+		return nil, err
+	}
+
+	// FIXME: these are being specified temporarily until it's the default!
+	metaparams := resources.DefaultMetaParams
+
+	exec1 := &resources.ExecRes{
+		BaseRes: resources.BaseRes{
+			Name:       "exec1",
+			MetaParams: metaparams,
+		},
+		Cmd:   "echo hello world && echo goodbye world 1>&2", // to stdout && stderr
+		Shell: "/bin/bash",
+	}
+	g.AddVertex(exec1)
+
+	output := &resources.FileRes{
+		BaseRes: resources.BaseRes{
+			Name:       "output",
+			MetaParams: metaparams,
+			// send->recv!
+			Recv: map[string]*resources.Send{
+				"Content": {Res: exec1, Key: "Output"},
+			},
+		},
+		Path:  "/tmp/mgmt/output",
+		State: "present",
+	}
+	g.AddVertex(output)
+	g.AddEdge(exec1, output, &resources.Edge{Name: "e0"})
+
+	stdout := &resources.FileRes{
+		BaseRes: resources.BaseRes{
+			Name:       "stdout",
+			MetaParams: metaparams,
+			// send->recv!
+			Recv: map[string]*resources.Send{
+				"Content": {Res: exec1, Key: "Stdout"},
+			},
+		},
+		Path:  "/tmp/mgmt/stdout",
+		State: "present",
+	}
+	g.AddVertex(stdout)
+	g.AddEdge(exec1, stdout, &resources.Edge{Name: "e1"})
+
+	stderr := &resources.FileRes{
+		BaseRes: resources.BaseRes{
+			Name:       "stderr",
+			MetaParams: metaparams,
+			// send->recv!
+			Recv: map[string]*resources.Send{
+				"Content": {Res: exec1, Key: "Stderr"},
+			},
+		},
+		Path:  "/tmp/mgmt/stderr",
+		State: "present",
+	}
+	g.AddVertex(stderr)
+	g.AddEdge(exec1, stderr, &resources.Edge{Name: "e2"})
+
+	//g, err := config.NewGraphFromConfig(obj.data.Hostname, obj.data.World, obj.data.Noop)
+	return g, nil
+}
+
+// Next returns nil errors every time there could be a new graph.
+func (obj *MyGAPI) Next() chan gapi.Next {
+	ch := make(chan gapi.Next)
+	obj.wg.Add(1)
+	go func() {
+		defer obj.wg.Done()
+		defer close(ch) // this will run before the obj.wg.Done()
+		if !obj.initialized {
+			next := gapi.Next{
+				Err:  fmt.Errorf("libmgmt: MyGAPI is not initialized"),
+				Exit: true, // exit, b/c programming error?
+			}
+			ch <- next
+			return
+		}
+		startChan := make(chan struct{}) // start signal
+		close(startChan)                 // kick it off!
+
+		ticker := make(<-chan time.Time)
+		if obj.data.NoStreamWatch || obj.Interval <= 0 {
+			ticker = nil
+		} else {
+			// arbitrarily change graph every interval seconds
+			t := time.NewTicker(time.Duration(obj.Interval) * time.Second)
+			defer t.Stop()
+			ticker = t.C
+		}
+		for {
+			select {
+			case <-startChan: // kick the loop once at start
+				startChan = nil // disable
+				// pass
+			case <-ticker:
+				// pass
+			case <-obj.closeChan:
+				return
+			}
+
+			log.Printf("libmgmt: Generating new graph...")
+			select {
+			case ch <- gapi.Next{}: // trigger a run
+			case <-obj.closeChan:
+				return
+			}
+		}
+	}()
+	return ch
+}
+
+// Close shuts down the MyGAPI.
+func (obj *MyGAPI) Close() error {
+	if !obj.initialized {
+		return fmt.Errorf("libmgmt: MyGAPI is not initialized")
+	}
+	close(obj.closeChan)
+	obj.wg.Wait()
+	obj.initialized = false // closed = true
+	return nil
+}
+
+// Run runs an embedded mgmt server.
+func Run() error {
+
+	obj := &mgmt.Main{}
+	obj.Program = "libmgmt" // TODO: set on compilation
+	obj.Version = "0.0.1"   // TODO: set on compilation
+	obj.TmpPrefix = true    // disable for easy debugging
+	//prefix := "/tmp/testprefix/"
+	//obj.Prefix = &p // enable for easy debugging
+	obj.IdealClusterSize = -1
+	obj.ConvergedTimeout = -1
+	obj.Noop = false // FIXME: careful!
+
+	obj.GAPI = &MyGAPI{ // graph API
+		Name:     "libmgmt", // TODO: set on compilation
+		Interval: 60 * 10,   // arbitrarily change graph every 15 seconds
+	}
+
+	if err := obj.Init(); err != nil {
+		return err
+	}
+
+	// install the exit signal handler
+	exit := make(chan struct{})
+	defer close(exit)
+	go func() {
+		signals := make(chan os.Signal, 1)
+		signal.Notify(signals, os.Interrupt) // catch ^C
+		//signal.Notify(signals, os.Kill) // catch signals
+		signal.Notify(signals, syscall.SIGTERM)
+
+		select {
+		case sig := <-signals: // any signal will do
+			if sig == os.Interrupt {
+				log.Println("Interrupted by ^C")
+				obj.Exit(nil)
+				return
+			}
+			log.Println("Interrupted by signal")
+			obj.Exit(fmt.Errorf("killed by %v", sig))
+			return
+		case <-exit:
+			return
+		}
+	}()
+
+	if err := obj.Run(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func main() {
+	log.Printf("Hello!")
+	if err := Run(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+		return
+	}
+	log.Printf("Goodbye!")
+}

examples/lib/libmgmt-subgraph0.go

@@ -0,0 +1,255 @@
+// libmgmt example of flattened subgraph
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/purpleidea/mgmt/gapi"
+	mgmt "github.com/purpleidea/mgmt/lib"
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/resources"
+
+	errwrap "github.com/pkg/errors"
+)
+
+// MyGAPI implements the main GAPI interface.
+type MyGAPI struct {
+	Name     string // graph name
+	Interval uint   // refresh interval, 0 to never refresh
+
+	data        gapi.Data
+	initialized bool
+	closeChan   chan struct{}
+	wg          sync.WaitGroup // sync group for tunnel go routines
+}
+
+// NewMyGAPI creates a new MyGAPI struct and calls Init().
+func NewMyGAPI(data gapi.Data, name string, interval uint) (*MyGAPI, error) {
+	obj := &MyGAPI{
+		Name:     name,
+		Interval: interval,
+	}
+	return obj, obj.Init(data)
+}
+
+// Init initializes the MyGAPI struct.
+func (obj *MyGAPI) Init(data gapi.Data) error {
+	if obj.initialized {
+		return fmt.Errorf("already initialized")
+	}
+	if obj.Name == "" {
+		return fmt.Errorf("the graph name must be specified")
+	}
+	obj.data = data // store for later
+	obj.closeChan = make(chan struct{})
+	obj.initialized = true
+	return nil
+}
+
+func (obj *MyGAPI) subGraph() (*pgraph.Graph, error) {
+	g, err := pgraph.NewGraph(obj.Name)
+	if err != nil {
+		return nil, err
+	}
+
+	metaparams := resources.DefaultMetaParams
+
+	f1 := &resources.FileRes{
+		BaseRes: resources.BaseRes{
+			Name:       "file1",
+			MetaParams: metaparams,
+		},
+		Path: "/tmp/mgmt/sub1",
+
+		State: "present",
+	}
+	g.AddVertex(f1)
+
+	n1 := &resources.NoopRes{
+		BaseRes: resources.BaseRes{
+			Name:       "noop1",
+			MetaParams: metaparams,
+		},
+	}
+	g.AddVertex(n1)
+
+	return g, nil
+}
+
+// Graph returns a current Graph.
+func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
+	if !obj.initialized {
+		return nil, fmt.Errorf("libmgmt: MyGAPI is not initialized")
+	}
+
+	g, err := pgraph.NewGraph(obj.Name)
+	if err != nil {
+		return nil, err
+	}
+
+	// FIXME: these are being specified temporarily until it's the default!
+	metaparams := resources.DefaultMetaParams
+
+	content := "I created a subgraph!\n"
+	f0 := &resources.FileRes{
+		BaseRes: resources.BaseRes{
+			Name:       "README",
+			MetaParams: metaparams,
+		},
+		Path:    "/tmp/mgmt/README",
+		Content: &content,
+		State:   "present",
+	}
+	g.AddVertex(f0)
+
+	subGraph, err := obj.subGraph()
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "running subGraph() failed")
+	}
+
+	edgeGenFn := func(v1, v2 pgraph.Vertex) pgraph.Edge {
+		edge := &resources.Edge{
+			Name: fmt.Sprintf("edge: %s->%s", v1, v2),
+		}
+
+		// if we want to do something specific based on input
+		_, v2IsFile := v2.(*resources.FileRes)
+		if v1 == f0 && v2IsFile {
+			edge.Notify = true
+		}
+
+		return edge
+	}
+	g.AddEdgeVertexGraph(f0, subGraph, edgeGenFn)
+
+	//g, err := config.NewGraphFromConfig(obj.data.Hostname, obj.data.World, obj.data.Noop)
+	return g, nil
+}
+
+// Next returns nil errors every time there could be a new graph.
+func (obj *MyGAPI) Next() chan gapi.Next {
+	ch := make(chan gapi.Next)
+	obj.wg.Add(1)
+	go func() {
+		defer obj.wg.Done()
+		defer close(ch) // this will run before the obj.wg.Done()
+		if !obj.initialized {
+			next := gapi.Next{
+				Err:  fmt.Errorf("libmgmt: MyGAPI is not initialized"),
+				Exit: true, // exit, b/c programming error?
+			}
+			ch <- next
+			return
+		}
+		startChan := make(chan struct{}) // start signal
+		close(startChan)                 // kick it off!
+
+		ticker := make(<-chan time.Time)
+		if obj.data.NoStreamWatch || obj.Interval <= 0 {
+			ticker = nil
+		} else {
+			// arbitrarily change graph every interval seconds
+			t := time.NewTicker(time.Duration(obj.Interval) * time.Second)
+			defer t.Stop()
+			ticker = t.C
+		}
+		for {
+			select {
+			case <-startChan: // kick the loop once at start
+				startChan = nil // disable
+				// pass
+			case <-ticker:
+				// pass
+			case <-obj.closeChan:
+				return
+			}
+
+			log.Printf("libmgmt: Generating new graph...")
+			select {
+			case ch <- gapi.Next{}: // trigger a run
+			case <-obj.closeChan:
+				return
+			}
+		}
+	}()
+	return ch
+}
+
+// Close shuts down the MyGAPI.
+func (obj *MyGAPI) Close() error {
+	if !obj.initialized {
+		return fmt.Errorf("libmgmt: MyGAPI is not initialized")
+	}
+	close(obj.closeChan)
+	obj.wg.Wait()
+	obj.initialized = false // closed = true
+	return nil
+}
+
+// Run runs an embedded mgmt server.
+func Run() error {
+
+	obj := &mgmt.Main{}
+	obj.Program = "libmgmt" // TODO: set on compilation
+	obj.Version = "0.0.1"   // TODO: set on compilation
+	obj.TmpPrefix = true    // disable for easy debugging
+	//prefix := "/tmp/testprefix/"
+	//obj.Prefix = &p // enable for easy debugging
+	obj.IdealClusterSize = -1
+	obj.ConvergedTimeout = -1
+	obj.Noop = false // FIXME: careful!
+
+	obj.GAPI = &MyGAPI{ // graph API
+		Name:     "libmgmt", // TODO: set on compilation
+		Interval: 60 * 10,   // arbitrarily change graph every 15 seconds
+	}
+
+	if err := obj.Init(); err != nil {
+		return err
+	}
+
+	// install the exit signal handler
+	exit := make(chan struct{})
+	defer close(exit)
+	go func() {
+		signals := make(chan os.Signal, 1)
+		signal.Notify(signals, os.Interrupt) // catch ^C
+		//signal.Notify(signals, os.Kill) // catch signals
+		signal.Notify(signals, syscall.SIGTERM)
+
+		select {
+		case sig := <-signals: // any signal will do
+			if sig == os.Interrupt {
+				log.Println("Interrupted by ^C")
+				obj.Exit(nil)
+				return
+			}
+			log.Println("Interrupted by signal")
+			obj.Exit(fmt.Errorf("killed by %v", sig))
+			return
+		case <-exit:
+			return
+		}
+	}()
+
+	if err := obj.Run(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func main() {
+	log.Printf("Hello!")
+	if err := Run(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+		return
+	}
+	log.Printf("Goodbye!")
+}

examples/lib/libmgmt-subgraph1.go

@@ -0,0 +1,243 @@
+// libmgmt example of graph resource
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/purpleidea/mgmt/gapi"
+	mgmt "github.com/purpleidea/mgmt/lib"
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/resources"
+)
+
+// MyGAPI implements the main GAPI interface.
+type MyGAPI struct {
+	Name     string // graph name
+	Interval uint   // refresh interval, 0 to never refresh
+
+	data        gapi.Data
+	initialized bool
+	closeChan   chan struct{}
+	wg          sync.WaitGroup // sync group for tunnel go routines
+}
+
+// NewMyGAPI creates a new MyGAPI struct and calls Init().
+func NewMyGAPI(data gapi.Data, name string, interval uint) (*MyGAPI, error) {
+	obj := &MyGAPI{
+		Name:     name,
+		Interval: interval,
+	}
+	return obj, obj.Init(data)
+}
+
+// Init initializes the MyGAPI struct.
+func (obj *MyGAPI) Init(data gapi.Data) error {
+	if obj.initialized {
+		return fmt.Errorf("already initialized")
+	}
+	if obj.Name == "" {
+		return fmt.Errorf("the graph name must be specified")
+	}
+	obj.data = data // store for later
+	obj.closeChan = make(chan struct{})
+	obj.initialized = true
+	return nil
+}
+
+// Graph returns a current Graph.
+func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
+	if !obj.initialized {
+		return nil, fmt.Errorf("libmgmt: MyGAPI is not initialized")
+	}
+
+	g, err := pgraph.NewGraph(obj.Name)
+	if err != nil {
+		return nil, err
+	}
+
+	// FIXME: these are being specified temporarily until it's the default!
+	metaparams := resources.DefaultMetaParams
+
+	content := "I created a subgraph!\n"
+	f0 := &resources.FileRes{
+		BaseRes: resources.BaseRes{
+			Name:       "README",
+			MetaParams: metaparams,
+		},
+		Path:    "/tmp/mgmt/README",
+		Content: &content,
+		State:   "present",
+	}
+	g.AddVertex(f0)
+
+	// create a subgraph to add *into* a graph resource
+	subGraph, err := pgraph.NewGraph(fmt.Sprintf("%s->subgraph", obj.Name))
+	if err != nil {
+		return nil, err
+	}
+
+	// add elements into the sub graph
+	f1 := &resources.FileRes{
+		BaseRes: resources.BaseRes{
+			Name:       "file1",
+			MetaParams: metaparams,
+		},
+		Path: "/tmp/mgmt/sub1",
+
+		State: "present",
+	}
+	subGraph.AddVertex(f1)
+
+	n1 := &resources.NoopRes{
+		BaseRes: resources.BaseRes{
+			Name:       "noop1",
+			MetaParams: metaparams,
+		},
+	}
+	subGraph.AddVertex(n1)
+
+	e0 := &resources.Edge{Name: "e0"}
+	e0.Notify = true // send a notification from v0 to v1
+	subGraph.AddEdge(f1, n1, e0)
+
+	// create the actual resource to hold the sub graph
+	subGraphRes0 := &resources.GraphRes{ // TODO: should we name this SubGraphRes ?
+		BaseRes: resources.BaseRes{
+			Name:       "subgraph1",
+			MetaParams: metaparams,
+		},
+		Graph: subGraph,
+	}
+	g.AddVertex(subGraphRes0) // add it to the main graph
+
+	//g, err := config.NewGraphFromConfig(obj.data.Hostname, obj.data.World, obj.data.Noop)
+	return g, nil
+}
+
+// Next returns nil errors every time there could be a new graph.
+func (obj *MyGAPI) Next() chan gapi.Next {
+	ch := make(chan gapi.Next)
+	obj.wg.Add(1)
+	go func() {
+		defer obj.wg.Done()
+		defer close(ch) // this will run before the obj.wg.Done()
+		if !obj.initialized {
+			next := gapi.Next{
+				Err:  fmt.Errorf("libmgmt: MyGAPI is not initialized"),
+				Exit: true, // exit, b/c programming error?
+			}
+			ch <- next
+			return
+		}
+		startChan := make(chan struct{}) // start signal
+		close(startChan)                 // kick it off!
+
+		ticker := make(<-chan time.Time)
+		if obj.data.NoStreamWatch || obj.Interval <= 0 {
+			ticker = nil
+		} else {
+			// arbitrarily change graph every interval seconds
+			t := time.NewTicker(time.Duration(obj.Interval) * time.Second)
+			defer t.Stop()
+			ticker = t.C
+		}
+		for {
+			select {
+			case <-startChan: // kick the loop once at start
+				startChan = nil // disable
+				// pass
+			case <-ticker:
+				// pass
+			case <-obj.closeChan:
+				return
+			}
+
+			log.Printf("libmgmt: Generating new graph...")
+			select {
+			case ch <- gapi.Next{}: // trigger a run
+			case <-obj.closeChan:
+				return
+			}
+		}
+	}()
+	return ch
+}
+
+// Close shuts down the MyGAPI.
+func (obj *MyGAPI) Close() error {
+	if !obj.initialized {
+		return fmt.Errorf("libmgmt: MyGAPI is not initialized")
+	}
+	close(obj.closeChan)
+	obj.wg.Wait()
+	obj.initialized = false // closed = true
+	return nil
+}
+
+// Run runs an embedded mgmt server.
+func Run() error {
+
+	obj := &mgmt.Main{}
+	obj.Program = "libmgmt" // TODO: set on compilation
+	obj.Version = "0.0.1"   // TODO: set on compilation
+	obj.TmpPrefix = true    // disable for easy debugging
+	//prefix := "/tmp/testprefix/"
+	//obj.Prefix = &p // enable for easy debugging
+	obj.IdealClusterSize = -1
+	obj.ConvergedTimeout = -1
+	obj.Noop = false // FIXME: careful!
+
+	obj.GAPI = &MyGAPI{ // graph API
+		Name:     "libmgmt", // TODO: set on compilation
+		Interval: 60 * 10,   // arbitrarily change graph every 15 seconds
+	}
+
+	if err := obj.Init(); err != nil {
+		return err
+	}
+
+	// install the exit signal handler
+	exit := make(chan struct{})
+	defer close(exit)
+	go func() {
+		signals := make(chan os.Signal, 1)
+		signal.Notify(signals, os.Interrupt) // catch ^C
+		//signal.Notify(signals, os.Kill) // catch signals
+		signal.Notify(signals, syscall.SIGTERM)
+
+		select {
+		case sig := <-signals: // any signal will do
+			if sig == os.Interrupt {
+				log.Println("Interrupted by ^C")
+				obj.Exit(nil)
+				return
+			}
+			log.Println("Interrupted by signal")
+			obj.Exit(fmt.Errorf("killed by %v", sig))
+			return
+		case <-exit:
+			return
+		}
+	}()
+
+	if err := obj.Run(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func main() {
+	log.Printf("Hello!")
+	if err := Run(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+		return
+	}
+	log.Printf("Goodbye!")
+}

examples/lib/libmgmt1.go

@@ -0,0 +1,207 @@
+// libmgmt example
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/purpleidea/mgmt/gapi"
+	mgmt "github.com/purpleidea/mgmt/lib"
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/resources"
+	"github.com/purpleidea/mgmt/yamlgraph"
+)
+
+// MyGAPI implements the main GAPI interface.
+type MyGAPI struct {
+	Name     string // graph name
+	Interval uint   // refresh interval, 0 to never refresh
+
+	data        gapi.Data
+	initialized bool
+	closeChan   chan struct{}
+	wg          sync.WaitGroup // sync group for tunnel go routines
+}
+
+// NewMyGAPI creates a new MyGAPI struct and calls Init().
+func NewMyGAPI(data gapi.Data, name string, interval uint) (*MyGAPI, error) {
+	obj := &MyGAPI{
+		Name:     name,
+		Interval: interval,
+	}
+	return obj, obj.Init(data)
+}
+
+// Init initializes the MyGAPI struct.
+func (obj *MyGAPI) Init(data gapi.Data) error {
+	if obj.initialized {
+		return fmt.Errorf("already initialized")
+	}
+	if obj.Name == "" {
+		return fmt.Errorf("the graph name must be specified")
+	}
+	obj.data = data // store for later
+	obj.closeChan = make(chan struct{})
+	obj.initialized = true
+	return nil
+}
+
+// Graph returns a current Graph.
+func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
+	if !obj.initialized {
+		return nil, fmt.Errorf("libmgmt: MyGAPI is not initialized")
+	}
+
+	n1 := &resources.NoopRes{
+		BaseRes: resources.BaseRes{
+			Name:       "noop1",
+			MetaParams: resources.DefaultMetaParams,
+		},
+	}
+
+	// we can still build a graph via the yaml method
+	gc := &yamlgraph.GraphConfig{
+		Graph: obj.Name,
+		Resources: yamlgraph.Resources{ // must redefine anonymous struct :(
+			// in alphabetical order
+			Exec:  []*resources.ExecRes{},
+			File:  []*resources.FileRes{},
+			Msg:   []*resources.MsgRes{},
+			Noop:  []*resources.NoopRes{n1},
+			Pkg:   []*resources.PkgRes{},
+			Svc:   []*resources.SvcRes{},
+			Timer: []*resources.TimerRes{},
+			Virt:  []*resources.VirtRes{},
+		},
+		//Collector: []collectorResConfig{},
+		//Edges:     []Edge{},
+		Comment: "comment!",
+	}
+
+	g, err := gc.NewGraphFromConfig(obj.data.Hostname, obj.data.World, obj.data.Noop)
+	return g, err
+}
+
+// Next returns nil errors every time there could be a new graph.
+func (obj *MyGAPI) Next() chan gapi.Next {
+	ch := make(chan gapi.Next)
+	obj.wg.Add(1)
+	go func() {
+		defer obj.wg.Done()
+		defer close(ch) // this will run before the obj.wg.Done()
+		if !obj.initialized {
+			next := gapi.Next{
+				Err:  fmt.Errorf("libmgmt: MyGAPI is not initialized"),
+				Exit: true, // exit, b/c programming error?
+			}
+			ch <- next
+		}
+		startChan := make(chan struct{}) // start signal
+		close(startChan)                 // kick it off!
+
+		ticker := make(<-chan time.Time)
+		if obj.data.NoStreamWatch || obj.Interval <= 0 {
+			ticker = nil
+		} else {
+			// arbitrarily change graph every interval seconds
+			t := time.NewTicker(time.Duration(obj.Interval) * time.Second)
+			defer t.Stop()
+			ticker = t.C
+		}
+		for {
+			select {
+			case <-startChan: // kick the loop once at start
+				startChan = nil // disable
+				// pass
+			case <-ticker:
+				// pass
+			case <-obj.closeChan:
+				return
+			}
+
+			log.Printf("libmgmt: Generating new graph...")
+			select {
+			case ch <- gapi.Next{}: // trigger a run
+			case <-obj.closeChan:
+				return
+			}
+		}
+	}()
+	return ch
+}
+
+// Close shuts down the MyGAPI.
+func (obj *MyGAPI) Close() error {
+	if !obj.initialized {
+		return fmt.Errorf("libmgmt: MyGAPI is not initialized")
+	}
+	close(obj.closeChan)
+	obj.wg.Wait()
+	obj.initialized = false // closed = true
+	return nil
+}
+
+// Run runs an embedded mgmt server.
+func Run() error {
+
+	obj := &mgmt.Main{}
+	obj.Program = "libmgmt" // TODO: set on compilation
+	obj.Version = "0.0.1"   // TODO: set on compilation
+	obj.TmpPrefix = true
+	obj.IdealClusterSize = -1
+	obj.ConvergedTimeout = -1
+	obj.Noop = true
+
+	obj.GAPI = &MyGAPI{ // graph API
+		Name:     "libmgmt", // TODO: set on compilation
+		Interval: 15,        // arbitrarily change graph every 15 seconds
+	}
+
+	if err := obj.Init(); err != nil {
+		return err
+	}
+
+	// install the exit signal handler
+	exit := make(chan struct{})
+	defer close(exit)
+	go func() {
+		signals := make(chan os.Signal, 1)
+		signal.Notify(signals, os.Interrupt) // catch ^C
+		//signal.Notify(signals, os.Kill) // catch signals
+		signal.Notify(signals, syscall.SIGTERM)
+
+		select {
+		case sig := <-signals: // any signal will do
+			if sig == os.Interrupt {
+				log.Println("Interrupted by ^C")
+				obj.Exit(nil)
+				return
+			}
+			log.Println("Interrupted by signal")
+			obj.Exit(fmt.Errorf("killed by %v", sig))
+			return
+		case <-exit:
+			return
+		}
+	}()
+
+	if err := obj.Run(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func main() {
+	log.Printf("Hello!")
+	if err := Run(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+		return
+	}
+	log.Printf("Goodbye!")
+}

examples/lib/libmgmt2.go

@@ -0,0 +1,209 @@
+// libmgmt example
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"strconv"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/purpleidea/mgmt/gapi"
+	mgmt "github.com/purpleidea/mgmt/lib"
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/resources"
+)
+
+// MyGAPI implements the main GAPI interface.
+type MyGAPI struct {
+	Name     string // graph name
+	Count    uint   // number of resources to create
+	Interval uint   // refresh interval, 0 to never refresh
+
+	data        gapi.Data
+	initialized bool
+	closeChan   chan struct{}
+	wg          sync.WaitGroup // sync group for tunnel go routines
+}
+
+// NewMyGAPI creates a new MyGAPI struct and calls Init().
+func NewMyGAPI(data gapi.Data, name string, interval uint, count uint) (*MyGAPI, error) {
+	obj := &MyGAPI{
+		Name:     name,
+		Count:    count,
+		Interval: interval,
+	}
+	return obj, obj.Init(data)
+}
+
+// Init initializes the MyGAPI struct.
+func (obj *MyGAPI) Init(data gapi.Data) error {
+	if obj.initialized {
+		return fmt.Errorf("already initialized")
+	}
+	if obj.Name == "" {
+		return fmt.Errorf("the graph name must be specified")
+	}
+	obj.data = data // store for later
+	obj.closeChan = make(chan struct{})
+	obj.initialized = true
+	return nil
+}
+
+// Graph returns a current Graph.
+func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
+	if !obj.initialized {
+		return nil, fmt.Errorf("libmgmt: MyGAPI is not initialized")
+	}
+
+	g, err := pgraph.NewGraph(obj.Name)
+	if err != nil {
+		return nil, err
+	}
+	var vertex pgraph.Vertex
+	for i := uint(0); i < obj.Count; i++ {
+		n := &resources.NoopRes{
+			BaseRes: resources.BaseRes{
+				Name:       fmt.Sprintf("noop%d", i),
+				MetaParams: resources.DefaultMetaParams,
+			},
+		}
+		g.AddVertex(n)
+		if i > 0 {
+			g.AddEdge(vertex, n, &resources.Edge{Name: fmt.Sprintf("e%d", i)})
+		}
+		vertex = n // save
+	}
+
+	//g, err := config.NewGraphFromConfig(obj.data.Hostname, obj.data.World, obj.data.Noop)
+	return g, nil
+}
+
+// Next returns nil errors every time there could be a new graph.
+func (obj *MyGAPI) Next() chan gapi.Next {
+	ch := make(chan gapi.Next)
+	obj.wg.Add(1)
+	go func() {
+		defer obj.wg.Done()
+		defer close(ch) // this will run before the obj.wg.Done()
+		if !obj.initialized {
+			next := gapi.Next{
+				Err:  fmt.Errorf("libmgmt: MyGAPI is not initialized"),
+				Exit: true, // exit, b/c programming error?
+			}
+			ch <- next
+		}
+		startChan := make(chan struct{}) // start signal
+		close(startChan)                 // kick it off!
+
+		ticker := make(<-chan time.Time)
+		if obj.data.NoStreamWatch || obj.Interval <= 0 {
+			ticker = nil
+		} else {
+			// arbitrarily change graph every interval seconds
+			t := time.NewTicker(time.Duration(obj.Interval) * time.Second)
+			defer t.Stop()
+			ticker = t.C
+		}
+		for {
+			select {
+			case <-startChan: // kick the loop once at start
+				startChan = nil // disable
+				// pass
+			case <-ticker:
+				// pass
+			case <-obj.closeChan:
+				return
+			}
+
+			log.Printf("libmgmt: Generating new graph...")
+			select {
+			case ch <- gapi.Next{}: // trigger a run
+			case <-obj.closeChan:
+				return
+			}
+		}
+	}()
+	return ch
+}
+
+// Close shuts down the MyGAPI.
+func (obj *MyGAPI) Close() error {
+	if !obj.initialized {
+		return fmt.Errorf("libmgmt: MyGAPI is not initialized")
+	}
+	close(obj.closeChan)
+	obj.wg.Wait()
+	obj.initialized = false // closed = true
+	return nil
+}
+
+// Run runs an embedded mgmt server.
+func Run(count uint) error {
+
+	obj := &mgmt.Main{}
+	obj.Program = "libmgmt" // TODO: set on compilation
+	obj.Version = "0.0.1"   // TODO: set on compilation
+	obj.TmpPrefix = true
+	obj.IdealClusterSize = -1
+	obj.ConvergedTimeout = -1
+	obj.Noop = true
+
+	obj.GAPI = &MyGAPI{ // graph API
+		Name:     "libmgmt", // TODO: set on compilation
+		Count:    count,     // number of vertices to add
+		Interval: 15,        // arbitrarily change graph every 15 seconds
+	}
+
+	if err := obj.Init(); err != nil {
+		return err
+	}
+
+	// install the exit signal handler
+	exit := make(chan struct{})
+	defer close(exit)
+	go func() {
+		signals := make(chan os.Signal, 1)
+		signal.Notify(signals, os.Interrupt) // catch ^C
+		//signal.Notify(signals, os.Kill) // catch signals
+		signal.Notify(signals, syscall.SIGTERM)
+
+		select {
+		case sig := <-signals: // any signal will do
+			if sig == os.Interrupt {
+				log.Println("Interrupted by ^C")
+				obj.Exit(nil)
+				return
+			}
+			log.Println("Interrupted by signal")
+			obj.Exit(fmt.Errorf("killed by %v", sig))
+			return
+		case <-exit:
+			return
+		}
+	}()
+
+	if err := obj.Run(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func main() {
+	log.Printf("Hello!")
+	var count uint = 1 // default
+	if len(os.Args) == 2 {
+		if i, err := strconv.Atoi(os.Args[1]); err == nil && i > 0 {
+			count = uint(i)
+		}
+	}
+	if err := Run(count); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+		return
+	}
+	log.Printf("Goodbye!")
+}

examples/lib/libmgmt3.go

@@ -0,0 +1,248 @@
+// libmgmt example of send->recv
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/purpleidea/mgmt/gapi"
+	mgmt "github.com/purpleidea/mgmt/lib"
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/resources"
+)
+
+// MyGAPI implements the main GAPI interface.
+type MyGAPI struct {
+	Name     string // graph name
+	Interval uint   // refresh interval, 0 to never refresh
+
+	data        gapi.Data
+	initialized bool
+	closeChan   chan struct{}
+	wg          sync.WaitGroup // sync group for tunnel go routines
+}
+
+// NewMyGAPI creates a new MyGAPI struct and calls Init().
+func NewMyGAPI(data gapi.Data, name string, interval uint) (*MyGAPI, error) {
+	obj := &MyGAPI{
+		Name:     name,
+		Interval: interval,
+	}
+	return obj, obj.Init(data)
+}
+
+// Init initializes the MyGAPI struct.
+func (obj *MyGAPI) Init(data gapi.Data) error {
+	if obj.initialized {
+		return fmt.Errorf("already initialized")
+	}
+	if obj.Name == "" {
+		return fmt.Errorf("the graph name must be specified")
+	}
+	obj.data = data // store for later
+	obj.closeChan = make(chan struct{})
+	obj.initialized = true
+	return nil
+}
+
+// Graph returns a current Graph.
+func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
+	if !obj.initialized {
+		return nil, fmt.Errorf("libmgmt: MyGAPI is not initialized")
+	}
+
+	g, err := pgraph.NewGraph(obj.Name)
+	if err != nil {
+		return nil, err
+	}
+
+	// FIXME: these are being specified temporarily until it's the default!
+	metaparams := resources.DefaultMetaParams
+
+	content := "Delete me to trigger a notification!\n"
+	f0 := &resources.FileRes{
+		BaseRes: resources.BaseRes{
+			Name:       "README",
+			MetaParams: metaparams,
+		},
+		Path:    "/tmp/mgmt/README",
+		Content: &content,
+		State:   "present",
+	}
+
+	g.AddVertex(f0)
+
+	p1 := &resources.PasswordRes{
+		BaseRes: resources.BaseRes{
+			Name:       "password1",
+			MetaParams: metaparams,
+		},
+		Length: 8,    // generated string will have this many characters
+		Saved:  true, // this causes passwords to be stored in plain text!
+	}
+	g.AddVertex(p1)
+
+	f1 := &resources.FileRes{
+		BaseRes: resources.BaseRes{
+			Name:       "file1",
+			MetaParams: metaparams,
+			// send->recv!
+			Recv: map[string]*resources.Send{
+				"Content": {Res: p1, Key: "Password"},
+			},
+		},
+		Path: "/tmp/mgmt/secret",
+		//Content:  p1.Password, // won't work
+		State: "present",
+	}
+
+	g.AddVertex(f1)
+
+	n1 := &resources.NoopRes{
+		BaseRes: resources.BaseRes{
+			Name:       "noop1",
+			MetaParams: metaparams,
+		},
+	}
+
+	g.AddVertex(n1)
+
+	e0 := &resources.Edge{Name: "e0"}
+	e0.Notify = true // send a notification from f0 to p1
+	g.AddEdge(f0, p1, e0)
+
+	g.AddEdge(p1, f1, &resources.Edge{Name: "e1"})
+
+	e2 := &resources.Edge{Name: "e2"}
+	e2.Notify = true // send a notification from f1 to n1
+	g.AddEdge(f1, n1, e2)
+
+	//g, err := config.NewGraphFromConfig(obj.data.Hostname, obj.data.World, obj.data.Noop)
+	return g, nil
+}
+
+// Next returns nil errors every time there could be a new graph.
+func (obj *MyGAPI) Next() chan gapi.Next {
+	ch := make(chan gapi.Next)
+	obj.wg.Add(1)
+	go func() {
+		defer obj.wg.Done()
+		defer close(ch) // this will run before the obj.wg.Done()
+		if !obj.initialized {
+			next := gapi.Next{
+				Err:  fmt.Errorf("libmgmt: MyGAPI is not initialized"),
+				Exit: true, // exit, b/c programming error?
+			}
+			ch <- next
+		}
+		startChan := make(chan struct{}) // start signal
+		close(startChan)                 // kick it off!
+
+		ticker := make(<-chan time.Time)
+		if obj.data.NoStreamWatch || obj.Interval <= 0 {
+			ticker = nil
+		} else {
+			// arbitrarily change graph every interval seconds
+			t := time.NewTicker(time.Duration(obj.Interval) * time.Second)
+			defer t.Stop()
+			ticker = t.C
+		}
+		for {
+			select {
+			case <-startChan: // kick the loop once at start
+				startChan = nil // disable
+				// pass
+			case <-ticker:
+				// pass
+			case <-obj.closeChan:
+				return
+			}
+
+			log.Printf("libmgmt: Generating new graph...")
+			select {
+			case ch <- gapi.Next{}: // trigger a run
+			case <-obj.closeChan:
+				return
+			}
+		}
+	}()
+	return ch
+}
+
+// Close shuts down the MyGAPI.
+func (obj *MyGAPI) Close() error {
+	if !obj.initialized {
+		return fmt.Errorf("libmgmt: MyGAPI is not initialized")
+	}
+	close(obj.closeChan)
+	obj.wg.Wait()
+	obj.initialized = false // closed = true
+	return nil
+}
+
+// Run runs an embedded mgmt server.
+func Run() error {
+
+	obj := &mgmt.Main{}
+	obj.Program = "libmgmt" // TODO: set on compilation
+	obj.Version = "0.0.1"   // TODO: set on compilation
+	obj.TmpPrefix = true    // disable for easy debugging
+	//prefix := "/tmp/testprefix/"
+	//obj.Prefix = &p // enable for easy debugging
+	obj.IdealClusterSize = -1
+	obj.ConvergedTimeout = -1
+	obj.Noop = false // FIXME: careful!
+
+	obj.GAPI = &MyGAPI{ // graph API
+		Name:     "libmgmt", // TODO: set on compilation
+		Interval: 60 * 10,   // arbitrarily change graph every 15 seconds
+	}
+
+	if err := obj.Init(); err != nil {
+		return err
+	}
+
+	// install the exit signal handler
+	exit := make(chan struct{})
+	defer close(exit)
+	go func() {
+		signals := make(chan os.Signal, 1)
+		signal.Notify(signals, os.Interrupt) // catch ^C
+		//signal.Notify(signals, os.Kill) // catch signals
+		signal.Notify(signals, syscall.SIGTERM)
+
+		select {
+		case sig := <-signals: // any signal will do
+			if sig == os.Interrupt {
+				log.Println("Interrupted by ^C")
+				obj.Exit(nil)
+				return
+			}
+			log.Println("Interrupted by signal")
+			obj.Exit(fmt.Errorf("killed by %v", sig))
+			return
+		case <-exit:
+			return
+		}
+	}()
+
+	if err := obj.Run(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func main() {
+	log.Printf("Hello!")
+	if err := Run(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+		return
+	}
+	log.Printf("Goodbye!")
+}

examples/limit1.yaml

@@ -0,0 +1,13 @@
+---
+graph: mygraph
+resources:
+  file:
+  - name: file1
+    meta:
+      limit: 0.2
+      burst: 5
+    path: "/tmp/mgmt/limit"
+    content: |
+      i am a normal file
+    state: exists
+edges: []

examples/noop0.yaml

@@ -0,0 +1,7 @@
+---
+graph: mygraph
+comment: simple noop example
+resources:
+  noop:
+  - name: noop0
+edges: []

examples/noop2.yaml

@@ -0,0 +1,30 @@
+---
+graph: mygraph
+comment: dangerous noop example
+resources:
+  noop:
+  - name: noop1
+    meta:
+      noop: true
+  file:
+  - name: file1
+    path: "/tmp/mgmt/hello-noop"
+    content: |
+      hello world from @purpleidea
+    state: exists
+    meta:
+      noop: true
+  exec:
+  - name: exec1
+    meta:
+      noop: true
+    cmd: 'rm -rf /'
+    shell: '/bin/bash'
+    timeout: 0
+    watchcmd: ''
+    watchshell: ''
+    ifcmd: ''
+    ifshell: ''
+    pollint: 0
+    state: present
+edges: []

examples/nspawn1.yaml

@@ -0,0 +1,7 @@
+---
+graph: mygraph
+resources:
+  nspawn:
+  - name: mgmt-nspawn1
+    state: running
+edges: []

examples/nspawn2.yaml

@@ -0,0 +1,7 @@
+---
+graph: mygraph
+resources:
+  nspawn:
+  - name: mgmt-nspawn2
+    state: stopped
+edges: []

examples/poll1.yaml

@@ -0,0 +1,24 @@
+---
+graph: mygraph
+resources:
+  file:
+  - name: file1
+    meta:
+      poll: 5
+    path: "/tmp/mgmt/f1"
+    content: |
+      i poll every 5 seconds
+    state: exists
+  - name: file2
+    path: "/tmp/mgmt/f2"
+    content: |
+      i use the event based watcher
+    state: exists
+  - name: file3
+    meta:
+      poll: 1
+    path: "/tmp/mgmt/f3"
+    content: |
+      i poll every second
+    state: exists
+edges: []

examples/remote2a.yaml

@@ -0,0 +1,20 @@
+---
+graph: mygraph
+comment: remote noop example
+resources:
+  file:
+  - name: file1a
+    path: "/tmp/file1a"
+    content: |
+      i am file1a
+    state: exists
+  - name: "@@file2a"
+    path: "/tmp/file2a"
+    content: |
+      i am file2a, exported from host a
+    state: exists
+collect:
+- kind: file
+  pattern: "/tmp/"
+edges: []
+remote: ssh://root:vagrant@192.168.121.201:22

examples/remote2b.yaml

@@ -0,0 +1,20 @@
+---
+graph: mygraph
+comment: remote noop example
+resources:
+  file:
+  - name: file1b
+    path: "/tmp/file1b"
+    content: |
+      i am file1b
+    state: exists
+  - name: "@@file2b"
+    path: "/tmp/file2b"
+    content: |
+      i am file2b, exported from host b
+    state: exists
+collect:
+- kind: file
+  pattern: "/tmp/"
+edges: []
+remote: ssh://root:vagrant@192.168.121.202:22

examples/retry1.yaml

@@ -0,0 +1,57 @@
+---
+graph: mygraph
+comment: You can test Watch and CheckApply failures with chmod ugo-r and chmod ugo-w.
+resources:
+  exec:
+  - name: exec1
+    cmd: 'touch /tmp/mgmt/no-read && chmod ugo-r /tmp/mgmt/no-read'
+    shell: '/bin/bash'
+    timeout: 0
+    watchcmd: ''
+    watchshell: ''
+    ifcmd: ''
+    ifshell: ''
+    pollint: 0
+    state: present
+  - name: exec2
+    cmd: 'touch /tmp/mgmt/no-write && chmod ugo-w /tmp/mgmt/no-write'
+    shell: '/bin/bash'
+    timeout: 0
+    watchcmd: ''
+    watchshell: ''
+    ifcmd: ''
+    ifshell: ''
+    pollint: 0
+    state: present
+  file:
+  - name: noread
+    path: "/tmp/mgmt/no-read"
+    meta:
+      retry: 3
+      delay: 5000
+    content: |
+      i am f1
+    state: exists
+  - name: nowrite
+    path: "/tmp/mgmt/no-write"
+    meta:
+      retry: 3
+      delay: 5000
+    content: |
+      i am f1
+    state: exists
+edges:
+- name: e1
+  from:
+    kind: exec
+    name: exec1
+  to:
+    kind: file
+    name: noread
+- name: e2
+  from:
+    kind: exec
+    name: exec2
+  to:
+    kind: file
+    name: nowrite

examples/svc2.yaml

@@ -0,0 +1,8 @@
+---
+graph: mygraph
+resources:
+  svc:
+  - name: purpleidea
+    state: running
+    session: true
+edges: []

examples/timer1.yaml

@@ -4,7 +4,7 @@ comment: timer example
 resources:
   timer:
   - name: timer1
-    interval: 30
+    interval: 3
   exec:
   - name: exec1
     cmd: echo hello world

examples/virt1.yaml

@@ -0,0 +1,11 @@
+---
+graph: mygraph
+resources:
+  virt:
+  - name: mgmt1
+    uri: 'qemu:///session'
+    cpus: 1
+    memory: 524288
+    state: shutoff
+    transient: true
+edges: []

examples/virt2.yaml

@@ -0,0 +1,11 @@
+---
+graph: mygraph
+resources:
+  virt:
+  - name: mgmt2
+    uri: 'qemu:///session'
+    cpus: 1
+    memory: 524288
+    state: shutoff
+    transient: false
+edges: []

examples/virt3.yaml

@@ -0,0 +1,11 @@
+---
+graph: mygraph
+resources:
+  virt:
+  - name: mgmt3
+    uri: 'qemu:///session'
+    cpus: 1
+    memory: 524288
+    state: running
+    transient: false
+edges: []

examples/virt4.yaml

@@ -0,0 +1,21 @@
+---
+graph: mygraph
+resources:
+  virt:
+  - name: mgmt4
+    meta:
+      limit: .inf
+      burst: 0
+    uri: 'qemu:///session'
+    cpus: 1
+    maxcpus: 4
+    memory: 524288
+    boot:
+    - hd
+    disk:
+    - type: qcow2
+      source: "~/.local/share/libvirt/images/fedora-23-scratch.qcow2"
+    state: running
+    transient: false
+edges: []
+comment: "qemu-img create -b fedora-23.qcow2 -f qcow2 fedora-23-scratch.qcow2"

gapi/gapi.go

@@ -0,0 +1,55 @@
+// Mgmt
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// Package gapi defines the interface that graph API generators must meet.
+package gapi
+
+import (
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/resources"
+)
+
+// Data is the set of input values passed into the GAPI structs via Init.
+type Data struct {
+	Hostname      string // uuid for the host, required for GAPI
+	World         resources.World
+	Noop          bool
+	NoConfigWatch bool
+	NoStreamWatch bool
+	// NOTE: we can add more fields here if needed by GAPI endpoints
+}
+
+// Next describes the particular response the GAPI implementer wishes to emit.
+type Next struct {
+	// FIXME: the Fast pause parameter should eventually get replaced with a
+	// "SwitchMethod" parameter or similar that instead lets the implementer
+	// choose between fast pause, slow pause, and interrupt. Interrupt could
+	// be a future extension to the Resource API that lets an Interrupt() be
+	// called if we want to exit immediately from the CheckApply part of the
+	// resource for some reason. For now we'll keep this simple with a bool.
+	Fast bool  // run a fast pause to switch?
+	Exit bool  // should we cause the program to exit? (specify err or not)
+	Err  error // if something goes wrong (use with or without exit!)
+}
+
+// GAPI is a Graph API that represents incoming graphs and change streams.
+type GAPI interface {
+	Init(Data) error               // initializes the GAPI and passes in useful data
+	Graph() (*pgraph.Graph, error) // returns the most recent pgraph
+	Next() chan Next               // returns a stream of switch events
+	Close() error                  // shutdown the GAPI
+}

lib/cli.go

@@ -0,0 +1,372 @@
+// Mgmt
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package lib
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/purpleidea/mgmt/puppet"
+	"github.com/purpleidea/mgmt/yamlgraph"
+	"github.com/purpleidea/mgmt/yamlgraph2"
+
+	"github.com/urfave/cli"
+)
+
+// run is the main run target.
+func run(c *cli.Context) error {
+
+	obj := &Main{}
+
+	obj.Program = c.App.Name
+	obj.Version = c.App.Version
+	if val, exists := c.App.Metadata["flags"]; exists {
+		if flags, ok := val.(Flags); ok {
+			obj.Flags = flags
+		}
+	}
+
+	if h := c.String("hostname"); c.IsSet("hostname") && h != "" {
+		obj.Hostname = &h
+	}
+
+	if s := c.String("prefix"); c.IsSet("prefix") && s != "" {
+		obj.Prefix = &s
+	}
+	obj.TmpPrefix = c.Bool("tmp-prefix")
+	obj.AllowTmpPrefix = c.Bool("allow-tmp-prefix")
+
+	if _ = c.String("code"); c.IsSet("code") {
+		if obj.GAPI != nil {
+			return fmt.Errorf("can't combine code GAPI with existing GAPI")
+		}
+		// TODO: implement DSL GAPI
+		//obj.GAPI = &dsl.GAPI{
+		//	Code: &s,
+		//}
+		return fmt.Errorf("the Code GAPI is not implemented yet") // TODO: DSL
+	}
+	if y := c.String("yaml"); c.IsSet("yaml") {
+		if obj.GAPI != nil {
+			return fmt.Errorf("can't combine YAML GAPI with existing GAPI")
+		}
+		obj.GAPI = &yamlgraph.GAPI{
+			File: &y,
+		}
+	}
+	if y := c.String("yaml2"); c.IsSet("yaml2") {
+		if obj.GAPI != nil {
+			return fmt.Errorf("can't combine YAMLv2 GAPI with existing GAPI")
+		}
+		obj.GAPI = &yamlgraph2.GAPI{
+			File: &y,
+		}
+	}
+	if p := c.String("puppet"); c.IsSet("puppet") {
+		if obj.GAPI != nil {
+			return fmt.Errorf("can't combine puppet GAPI with existing GAPI")
+		}
+		obj.GAPI = &puppet.GAPI{
+			PuppetParam: &p,
+			PuppetConf:  c.String("puppet-conf"),
+		}
+	}
+	obj.Remotes = c.StringSlice("remote") // FIXME: GAPI-ify somehow?
+
+	obj.NoWatch = c.Bool("no-watch")
+	obj.NoConfigWatch = c.Bool("no-config-watch")
+	obj.NoStreamWatch = c.Bool("no-stream-watch")
+
+	obj.Noop = c.Bool("noop")
+	obj.Sema = c.Int("sema")
+	obj.Graphviz = c.String("graphviz")
+	obj.GraphvizFilter = c.String("graphviz-filter")
+	obj.ConvergedTimeout = c.Int("converged-timeout")
+	obj.MaxRuntime = uint(c.Int("max-runtime"))
+
+	obj.Seeds = c.StringSlice("seeds")
+	obj.ClientURLs = c.StringSlice("client-urls")
+	obj.ServerURLs = c.StringSlice("server-urls")
+	obj.IdealClusterSize = c.Int("ideal-cluster-size")
+	obj.NoServer = c.Bool("no-server")
+
+	obj.CConns = uint16(c.Int("cconns"))
+	obj.AllowInteractive = c.Bool("allow-interactive")
+	obj.SSHPrivIDRsa = c.String("ssh-priv-id-rsa")
+	obj.NoCaching = c.Bool("no-caching")
+	obj.Depth = uint16(c.Int("depth"))
+
+	obj.NoPgp = c.Bool("no-pgp")
+
+	if kp := c.String("pgp-key-path"); c.IsSet("pgp-key-path") {
+		obj.PgpKeyPath = &kp
+	}
+
+	if us := c.String("pgp-identity"); c.IsSet("pgp-identity") {
+		obj.PgpIdentity = &us
+	}
+
+	if err := obj.Init(); err != nil {
+		return err
+	}
+
+	obj.Prometheus = c.Bool("prometheus")
+	obj.PrometheusListen = c.String("prometheus-listen")
+
+	// install the exit signal handler
+	exit := make(chan struct{})
+	defer close(exit)
+	go func() {
+		signals := make(chan os.Signal, 1)
+		signal.Notify(signals, os.Interrupt) // catch ^C
+		//signal.Notify(signals, os.Kill) // catch signals
+		signal.Notify(signals, syscall.SIGTERM)
+
+		select {
+		case sig := <-signals: // any signal will do
+			if sig == os.Interrupt {
+				log.Println("Interrupted by ^C")
+				obj.Exit(nil)
+				return
+			}
+			log.Println("Interrupted by signal")
+			obj.Exit(fmt.Errorf("killed by %v", sig))
+			return
+		case <-exit:
+			return
+		}
+	}()
+
+	if err := obj.Run(); err != nil {
+		return err
+		//return cli.NewExitError(err.Error(), 1) // TODO: ?
+		//return cli.NewExitError("", 1) // TODO: ?
+	}
+	return nil
+}
+
+// CLI is the entry point for using mgmt normally from the CLI.
+func CLI(program, version string, flags Flags) error {
+
+	// test for sanity
+	if program == "" || version == "" {
+		return fmt.Errorf("program was not compiled correctly, see Makefile")
+	}
+	app := cli.NewApp()
+	app.Name = program // App.name and App.version pass these values through
+	app.Version = version
+	app.Usage = "next generation config management"
+	app.Metadata = map[string]interface{}{ // additional flags
+		"flags": flags,
+	}
+	//app.Action = ... // without a default action, help runs
+
+	app.Commands = []cli.Command{
+		{
+			Name:    "run",
+			Aliases: []string{"r"},
+			Usage:   "run",
+			Action:  run,
+			Flags: []cli.Flag{
+				// useful for testing multiple instances on same machine
+				cli.StringFlag{
+					Name:  "hostname",
+					Value: "",
+					Usage: "hostname to use",
+				},
+
+				cli.StringFlag{
+					Name:   "prefix",
+					Usage:  "specify a path to the working prefix directory",
+					EnvVar: "MGMT_PREFIX",
+				},
+				cli.BoolFlag{
+					Name:  "tmp-prefix",
+					Usage: "request a pseudo-random, temporary prefix to be used",
+				},
+				cli.BoolFlag{
+					Name:  "allow-tmp-prefix",
+					Usage: "allow creation of a new temporary prefix if main prefix is unavailable",
+				},
+
+				cli.StringFlag{
+					Name:  "code, c",
+					Value: "",
+					Usage: "code definition to run",
+				},
+				cli.StringFlag{
+					Name:  "yaml",
+					Value: "",
+					Usage: "yaml graph definition to run",
+				},
+				cli.StringFlag{
+					Name:  "yaml2",
+					Value: "",
+					Usage: "yaml graph definition to run (parser v2)",
+				},
+				cli.StringFlag{
+					Name:  "puppet, p",
+					Value: "",
+					Usage: "load graph from puppet, optionally takes a manifest or path to manifest file",
+				},
+				cli.StringFlag{
+					Name:  "puppet-conf",
+					Value: "",
+					Usage: "the path to an alternate puppet.conf file",
+				},
+				cli.StringSliceFlag{
+					Name:  "remote",
+					Value: &cli.StringSlice{},
+					Usage: "list of remote graph definitions to run",
+				},
+
+				cli.BoolFlag{
+					Name:  "no-watch",
+					Usage: "do not update graph under any switch events",
+				},
+				cli.BoolFlag{
+					Name:  "no-config-watch",
+					Usage: "do not update graph on config switch events",
+				},
+				cli.BoolFlag{
+					Name:  "no-stream-watch",
+					Usage: "do not update graph on stream switch events",
+				},
+
+				cli.BoolFlag{
+					Name:  "noop",
+					Usage: "globally force all resources into no-op mode",
+				},
+				cli.IntFlag{
+					Name:  "sema",
+					Value: -1,
+					Usage: "globally add a semaphore to all resources with this lock count",
+				},
+				cli.StringFlag{
+					Name:  "graphviz, g",
+					Value: "",
+					Usage: "output file for graphviz data",
+				},
+				cli.StringFlag{
+					Name:  "graphviz-filter, gf",
+					Value: "",
+					Usage: "graphviz filter to use",
+				},
+				cli.IntFlag{
+					Name:   "converged-timeout, t",
+					Value:  -1,
+					Usage:  "exit after approximately this many seconds in a converged state",
+					EnvVar: "MGMT_CONVERGED_TIMEOUT",
+				},
+				cli.IntFlag{
+					Name:   "max-runtime",
+					Value:  0,
+					Usage:  "exit after a maximum of approximately this many seconds",
+					EnvVar: "MGMT_MAX_RUNTIME",
+				},
+
+				// if empty, it will startup a new server
+				cli.StringSliceFlag{
+					Name:   "seeds, s",
+					Value:  &cli.StringSlice{}, // empty slice
+					Usage:  "default etc client endpoint",
+					EnvVar: "MGMT_SEEDS",
+				},
+				// port 2379 and 4001 are common
+				cli.StringSliceFlag{
+					Name:   "client-urls",
+					Value:  &cli.StringSlice{},
+					Usage:  "list of URLs to listen on for client traffic",
+					EnvVar: "MGMT_CLIENT_URLS",
+				},
+				// port 2380 and 7001 are common
+				cli.StringSliceFlag{
+					Name:   "server-urls, peer-urls",
+					Value:  &cli.StringSlice{},
+					Usage:  "list of URLs to listen on for server (peer) traffic",
+					EnvVar: "MGMT_SERVER_URLS",
+				},
+				cli.IntFlag{
+					Name:   "ideal-cluster-size",
+					Value:  -1,
+					Usage:  "ideal number of server peers in cluster; only read by initial server",
+					EnvVar: "MGMT_IDEAL_CLUSTER_SIZE",
+				},
+				cli.BoolFlag{
+					Name:  "no-server",
+					Usage: "do not let other servers peer with me",
+				},
+
+				cli.IntFlag{
+					Name:   "cconns",
+					Value:  0,
+					Usage:  "number of maximum concurrent remote ssh connections to run; 0 for unlimited",
+					EnvVar: "MGMT_CCONNS",
+				},
+				cli.BoolFlag{
+					Name:  "allow-interactive",
+					Usage: "allow interactive prompting, such as for remote passwords",
+				},
+				cli.StringFlag{
+					Name:   "ssh-priv-id-rsa",
+					Value:  "~/.ssh/id_rsa",
+					Usage:  "default path to ssh key file, set empty to never touch",
+					EnvVar: "MGMT_SSH_PRIV_ID_RSA",
+				},
+				cli.BoolFlag{
+					Name:  "no-caching",
+					Usage: "don't allow remote caching of remote execution binary",
+				},
+				cli.IntFlag{
+					Name:   "depth",
+					Hidden: true, // internal use only
+					Value:  0,
+					Usage:  "specify depth in remote hierarchy",
+				},
+				cli.BoolFlag{
+					Name:  "no-pgp",
+					Usage: "don't create pgp keys",
+				},
+				cli.StringFlag{
+					Name:  "pgp-key-path",
+					Value: "",
+					Usage: "path for instance key pair",
+				},
+				cli.StringFlag{
+					Name:  "pgp-identity",
+					Value: "",
+					Usage: "default identity used for generation",
+				},
+				cli.BoolFlag{
+					Name:  "prometheus",
+					Usage: "start a prometheus instance",
+				},
+				cli.StringFlag{
+					Name:  "prometheus-listen",
+					Value: "",
+					Usage: "specify prometheus instance binding",
+				},
+			},
+		},
+	}
+	app.EnableBashCompletion = true
+	return app.Run(os.Args)
+}

lib/main.go

@@ -0,0 +1,671 @@
+// Mgmt
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package lib
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+	"time"
+
+	"github.com/purpleidea/mgmt/converger"
+	"github.com/purpleidea/mgmt/etcd"
+	"github.com/purpleidea/mgmt/gapi"
+	"github.com/purpleidea/mgmt/pgp"
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/prometheus"
+	"github.com/purpleidea/mgmt/recwatch"
+	"github.com/purpleidea/mgmt/remote"
+	"github.com/purpleidea/mgmt/resources"
+	"github.com/purpleidea/mgmt/util"
+
+	etcdtypes "github.com/coreos/etcd/pkg/types"
+	"github.com/coreos/pkg/capnslog"
+	multierr "github.com/hashicorp/go-multierror"
+	errwrap "github.com/pkg/errors"
+)
+
+// Flags are some constant flags which are used throughout the program.
+type Flags struct {
+	Debug   bool // add additional log messages
+	Trace   bool // add execution flow log messages
+	Verbose bool // add extra log message output
+}
+
+// Main is the main struct for running the mgmt logic.
+type Main struct {
+	Program string // the name of this program, usually set at compile time
+	Version string // the version of this program, usually set at compile time
+
+	Flags Flags // static global flags that are set at compile time
+
+	Hostname *string // hostname to use; nil if undefined
+
+	Prefix         *string // prefix passed in; nil if undefined
+	TmpPrefix      bool    // request a pseudo-random, temporary prefix to be used
+	AllowTmpPrefix bool    // allow creation of a new temporary prefix if main prefix is unavailable
+
+	GAPI    gapi.GAPI // graph API interface struct
+	Remotes []string  // list of remote graph definitions to run
+
+	NoWatch       bool // do not change graph under any circumstances
+	NoConfigWatch bool // do not update graph due to config changes
+	NoStreamWatch bool // do not update graph due to stream changes
+
+	Noop             bool   // globally force all resources into no-op mode
+	Sema             int    // add a semaphore with this lock count to each resource
+	Graphviz         string // output file for graphviz data
+	GraphvizFilter   string // graphviz filter to use
+	ConvergedTimeout int    // exit after approximately this many seconds in a converged state; -1 to disable
+	MaxRuntime       uint   // exit after a maximum of approximately this many seconds
+
+	Seeds            []string // default etc client endpoint
+	ClientURLs       []string // list of URLs to listen on for client traffic
+	ServerURLs       []string // list of URLs to listen on for server (peer) traffic
+	IdealClusterSize int      // ideal number of server peers in cluster; only read by initial server
+	NoServer         bool     // do not let other servers peer with me
+
+	CConns           uint16 // number of maximum concurrent remote ssh connections to run, 0 for unlimited
+	AllowInteractive bool   // allow interactive prompting, such as for remote passwords
+	SSHPrivIDRsa     string // default path to ssh key file, set empty to never touch
+	NoCaching        bool   // don't allow remote caching of remote execution binary
+	Depth            uint16 // depth in remote hierarchy; for internal use only
+
+	seeds            etcdtypes.URLs // processed seeds value
+	clientURLs       etcdtypes.URLs // processed client urls value
+	serverURLs       etcdtypes.URLs // processed server urls value
+	idealClusterSize uint16         // processed ideal cluster size value
+
+	NoPgp       bool    // disallow pgp functionality
+	PgpKeyPath  *string // import a pre-made key pair
+	PgpIdentity *string
+	pgpKeys     *pgp.PGP // agent key pair
+
+	Prometheus       bool   // enable prometheus metrics
+	PrometheusListen string // prometheus instance bind specification
+
+	exit chan error // exit signal
+}
+
+// Init initializes the main struct after it performs some validation.
+func (obj *Main) Init() error {
+
+	if obj.Program == "" || obj.Version == "" {
+		return fmt.Errorf("you must set the Program and Version strings")
+	}
+
+	if obj.Prefix != nil && obj.TmpPrefix {
+		return fmt.Errorf("choosing a prefix and the request for a tmp prefix is illogical")
+	}
+
+	// if we've turned off watching, then be explicit and disable them all!
+	// if all the watches are disabled, then it's equivalent to no watching
+	if obj.NoWatch {
+		obj.NoConfigWatch = true
+		obj.NoStreamWatch = true
+	} else if obj.NoConfigWatch && obj.NoStreamWatch {
+		obj.NoWatch = true
+	}
+
+	obj.idealClusterSize = uint16(obj.IdealClusterSize)
+	if obj.IdealClusterSize < 0 { // value is undefined, set to the default
+		obj.idealClusterSize = etcd.DefaultIdealClusterSize
+	}
+
+	if obj.idealClusterSize < 1 {
+		return fmt.Errorf("the IdealClusterSize should be at least one")
+	}
+
+	if obj.NoServer && len(obj.Remotes) > 0 {
+		// TODO: in this case, we won't be able to tunnel stuff back to
+		// here, so if we're okay with every remote graph running in an
+		// isolated mode, then this is okay. Improve on this if there's
+		// someone who really wants to be able to do this.
+		return fmt.Errorf("the Server is required when using Remotes")
+	}
+
+	if obj.CConns < 0 {
+		return fmt.Errorf("the CConns value should be at least zero")
+	}
+
+	if obj.ConvergedTimeout >= 0 && obj.CConns > 0 && len(obj.Remotes) > int(obj.CConns) {
+		return fmt.Errorf("you can't converge if you have more remotes than available connections")
+	}
+
+	if obj.Depth < 0 { // user should not be using this argument manually
+		return fmt.Errorf("negative values for Depth are not permitted")
+	}
+
+	// transform the url list inputs into etcd typed lists
+	var err error
+	obj.seeds, err = etcdtypes.NewURLs(
+		util.FlattenListWithSplit(obj.Seeds, []string{",", ";", " "}),
+	)
+	if err != nil && len(obj.Seeds) > 0 {
+		return fmt.Errorf("the Seeds didn't parse correctly")
+	}
+	obj.clientURLs, err = etcdtypes.NewURLs(
+		util.FlattenListWithSplit(obj.ClientURLs, []string{",", ";", " "}),
+	)
+	if err != nil && len(obj.ClientURLs) > 0 {
+		return fmt.Errorf("the ClientURLs didn't parse correctly")
+	}
+	obj.serverURLs, err = etcdtypes.NewURLs(
+		util.FlattenListWithSplit(obj.ServerURLs, []string{",", ";", " "}),
+	)
+	if err != nil && len(obj.ServerURLs) > 0 {
+		return fmt.Errorf("the ServerURLs didn't parse correctly")
+	}
+
+	obj.exit = make(chan error)
+	return nil
+}
+
+// Exit causes a safe shutdown. This is often attached to the ^C signal handler.
+func (obj *Main) Exit(err error) {
+	obj.exit <- err // trigger an exit!
+}
+
+// Run is the main execution entrypoint to run mgmt.
+func (obj *Main) Run() error {
+
+	var start = time.Now().UnixNano()
+
+	var flags int
+	if obj.Flags.Debug || true { // TODO: remove || true
+		flags = log.LstdFlags | log.Lshortfile
+	}
+	flags = (flags - log.Ldate) // remove the date for now
+	log.SetFlags(flags)
+
+	// un-hijack from capnslog...
+	log.SetOutput(os.Stderr)
+	if obj.Flags.Verbose {
+		capnslog.SetFormatter(capnslog.NewLogFormatter(os.Stderr, "(etcd) ", flags))
+	} else {
+		capnslog.SetFormatter(capnslog.NewNilFormatter())
+	}
+
+	log.Printf("This is: %s, version: %s", obj.Program, obj.Version)
+	log.Printf("Main: Start: %v", start)
+
+	hostname, err := os.Hostname() // a sensible default
+	// allow passing in the hostname, instead of using the system setting
+	if h := obj.Hostname; h != nil && *h != "" { // override by cli
+		hostname = *h
+	} else if err != nil {
+		return errwrap.Wrapf(err, "can't get default hostname")
+	}
+	if hostname == "" { // safety check
+		return fmt.Errorf("hostname cannot be empty")
+	}
+
+	var prefix = fmt.Sprintf("/var/lib/%s/", obj.Program) // default prefix
+	if p := obj.Prefix; p != nil {
+		prefix = *p
+	}
+	// make sure the working directory prefix exists
+	if obj.TmpPrefix || os.MkdirAll(prefix, 0770) != nil {
+		if obj.TmpPrefix || obj.AllowTmpPrefix {
+			var err error
+			if prefix, err = ioutil.TempDir("", obj.Program+"-"+hostname+"-"); err != nil {
+				return fmt.Errorf("can't create temporary prefix")
+			}
+			log.Println("Main: Warning: Working prefix directory is temporary!")
+
+		} else {
+			return fmt.Errorf("can't create prefix")
+		}
+	}
+	log.Printf("Main: Working prefix is: %s", prefix)
+	pgraphPrefix := fmt.Sprintf("%s/", path.Join(prefix, "pgraph")) // pgraph namespace
+	if err := os.MkdirAll(pgraphPrefix, 0770); err != nil {
+		return errwrap.Wrapf(err, "can't create pgraph prefix")
+	}
+
+	var prom *prometheus.Prometheus
+	if obj.Prometheus {
+		prom = &prometheus.Prometheus{
+			Listen: obj.PrometheusListen,
+		}
+		if err := prom.Init(); err != nil {
+			return errwrap.Wrapf(err, "can't create initiate Prometheus instance")
+		}
+
+		log.Printf("Main: Prometheus: Starting instance on %s", prom.Listen)
+		if err := prom.Start(); err != nil {
+			return errwrap.Wrapf(err, "can't start initiate Prometheus instance")
+		}
+	}
+
+	if !obj.NoPgp {
+		pgpPrefix := fmt.Sprintf("%s/", path.Join(prefix, "pgp"))
+		if err := os.MkdirAll(pgpPrefix, 0770); err != nil {
+			return errwrap.Wrapf(err, "can't create pgp prefix")
+		}
+
+		pgpKeyringPath := path.Join(pgpPrefix, pgp.DefaultKeyringFile) // default path
+
+		if p := obj.PgpKeyPath; p != nil {
+			pgpKeyringPath = *p
+		}
+
+		var err error
+		if obj.pgpKeys, err = pgp.Import(pgpKeyringPath); err != nil && !os.IsNotExist(err) {
+			return errwrap.Wrapf(err, "can't import pgp key")
+		}
+
+		if obj.pgpKeys == nil {
+
+			identity := fmt.Sprintf("%s <%s> %s", obj.Program, "root@"+hostname, "generated by "+obj.Program)
+			if p := obj.PgpIdentity; p != nil {
+				identity = *p
+			}
+
+			name, comment, email, err := pgp.ParseIdentity(identity)
+			if err != nil {
+				return errwrap.Wrapf(err, "can't parse user string")
+
+			}
+
+			// TODO: Make hash configurable
+			if obj.pgpKeys, err = pgp.Generate(name, comment, email, nil); err != nil {
+				return errwrap.Wrapf(err, "can't create pgp key")
+			}
+
+			if err := obj.pgpKeys.SaveKey(pgpKeyringPath); err != nil {
+				return errwrap.Wrapf(err, "can't save pgp key")
+			}
+		}
+
+		// TODO: Import admin key
+	}
+
+	oldGraph := &pgraph.Graph{}
+	graph := &resources.MGraph{}
+	// pass in the information we need
+	graph.Debug = obj.Flags.Debug
+	graph.Init()
+
+	// exit after `max-runtime` seconds for no reason at all...
+	if i := obj.MaxRuntime; i > 0 {
+		go func() {
+			time.Sleep(time.Duration(i) * time.Second)
+			obj.Exit(nil)
+		}()
+	}
+
+	// setup converger
+	converger := converger.NewConverger(
+		obj.ConvergedTimeout,
+		nil, // stateFn gets added in by EmbdEtcd
+	)
+	go converger.Loop(true) // main loop for converger, true to start paused
+
+	// embedded etcd
+	if len(obj.seeds) == 0 {
+		log.Printf("Main: Seeds: No seeds specified!")
+	} else {
+		log.Printf("Main: Seeds(%d): %v", len(obj.seeds), obj.seeds)
+	}
+	EmbdEtcd := etcd.NewEmbdEtcd(
+		hostname,
+		obj.seeds,
+		obj.clientURLs,
+		obj.serverURLs,
+		obj.NoServer,
+		obj.idealClusterSize,
+		etcd.Flags{
+			Debug:   obj.Flags.Debug,
+			Trace:   obj.Flags.Trace,
+			Verbose: obj.Flags.Verbose,
+		},
+		prefix,
+		converger,
+	)
+	if EmbdEtcd == nil {
+		// TODO: verify EmbdEtcd is not nil below...
+		obj.Exit(fmt.Errorf("Main: Etcd: Creation failed"))
+	} else if err := EmbdEtcd.Startup(); err != nil { // startup (returns when etcd main loop is running)
+		obj.Exit(fmt.Errorf("Main: Etcd: Startup failed: %v", err))
+	}
+
+	// wait for etcd server to be ready before continuing...
+	select {
+	case <-EmbdEtcd.ServerReady():
+		log.Printf("Main: Etcd: Server: Ready!")
+		// pass
+	case <-time.After(((etcd.MaxStartServerTimeout * etcd.MaxStartServerRetries) + 1) * time.Second):
+		obj.Exit(fmt.Errorf("Main: Etcd: Startup timeout"))
+	}
+
+	convergerStateFn := func(b bool) error {
+		// exit if we are using the converged timeout and we are the
+		// root node. otherwise, if we are a child node in a remote
+		// execution hierarchy, we should only notify our converged
+		// state and wait for the parent to trigger the exit.
+		if t := obj.ConvergedTimeout; obj.Depth == 0 && t >= 0 {
+			if b {
+				log.Printf("Main: Converged for %d seconds, exiting!", t)
+				obj.Exit(nil) // trigger an exit!
+			}
+			return nil
+		}
+		// send our individual state into etcd for others to see
+		return etcd.SetHostnameConverged(EmbdEtcd, hostname, b) // TODO: what should happen on error?
+	}
+	if EmbdEtcd != nil {
+		converger.SetStateFn(convergerStateFn)
+	}
+
+	// implementation of the World API (alternates can be substituted in)
+	world := &etcd.World{
+		Hostname: hostname,
+		EmbdEtcd: EmbdEtcd,
+	}
+
+	graph.Data = &resources.ResData{
+		Hostname:   hostname,
+		Converger:  converger,
+		Prometheus: prom,
+		World:      world,
+		Prefix:     pgraphPrefix,
+		Debug:      obj.Flags.Debug,
+	}
+
+	var gapiChan chan gapi.Next // stream events contain some instructions!
+	if obj.GAPI != nil {
+		data := gapi.Data{
+			Hostname: hostname,
+			World:    world,
+			Noop:     obj.Noop,
+			//NoWatch:  obj.NoWatch,
+			NoConfigWatch: obj.NoConfigWatch,
+			NoStreamWatch: obj.NoStreamWatch,
+		}
+		if err := obj.GAPI.Init(data); err != nil {
+			obj.Exit(fmt.Errorf("Main: GAPI: Init failed: %v", err))
+		} else {
+			// this must generate at least one event for it to work
+			gapiChan = obj.GAPI.Next() // stream of graph switch events!
+		}
+	}
+
+	exitchan := make(chan struct{}) // exit on close
+	go func() {
+		first := true // first loop or not
+		for {
+			log.Println("Main: Waiting...")
+			// The GAPI should always kick off an event on Next() at
+			// startup when (and if) it indeed has a graph to share!
+			fastPause := false
+			select {
+			case next, ok := <-gapiChan:
+				if !ok { // channel closed
+					if obj.Flags.Debug {
+						log.Printf("Main: GAPI exited")
+					}
+					gapiChan = nil // disable it
+					continue
+				}
+
+				// if we've been asked to exit...
+				if next.Exit {
+					obj.Exit(next.Err) // trigger exit
+					continue           // wait for exitchan
+				}
+
+				// the gapi lets us send an error to the channel
+				// this means there was a failure, but not fatal
+				if err := next.Err; err != nil {
+					log.Printf("Main: Error with graph stream: %v", err)
+					continue // wait for another event
+				}
+				// everything else passes through to cause a compile!
+
+				fastPause = next.Fast // should we pause fast?
+
+			case <-exitchan:
+				return
+			}
+
+			if obj.GAPI == nil {
+				log.Printf("Main: GAPI is empty!")
+				continue
+			}
+
+			// we need the vertices to be paused to work on them, so
+			// run graph vertex LOCK...
+			if !first { // TODO: we can flatten this check out I think
+				converger.Pause()      // FIXME: add sync wait?
+				graph.Pause(fastPause) // sync
+
+				//graph.UnGroup() // FIXME: implement me if needed!
+			}
+
+			// make the graph from yaml, lib, puppet->yaml, or dsl!
+			newGraph, err := obj.GAPI.Graph() // generate graph!
+			if err != nil {
+				log.Printf("Main: Error creating new graph: %v", err)
+				// unpause!
+				if !first {
+					graph.Start(first) // sync
+					converger.Start()  // after Start()
+				}
+				continue
+			}
+			if obj.Flags.Debug {
+				log.Printf("Main: New Graph: %v", newGraph)
+			}
+
+			// this edits the paused vertices, but it is safe to do
+			// so even if we don't use this new graph, since those
+			// value should be the same for existing vertices...
+			for _, v := range newGraph.Vertices() {
+				m := resources.VtoR(v).Meta()
+				// apply the global noop parameter if requested
+				if obj.Noop {
+					m.Noop = obj.Noop
+				}
+
+				// append the semaphore to each resource
+				if obj.Sema > 0 { // NOTE: size == 0 would block
+					// a semaphore with an empty id is valid
+					m.Sema = append(m.Sema, fmt.Sprintf(":%d", obj.Sema))
+				}
+			}
+
+			// We don't have to "UnGroup()" to compare, since we
+			// save the old graph to use when we compare.
+			// TODO: Does this hurt performance or graph changes ?
+			log.Printf("Main: GraphSync...")
+			vertexCmpFn := func(v1, v2 pgraph.Vertex) (bool, error) {
+				return resources.VtoR(v1).Compare(resources.VtoR(v2)), nil
+			}
+			vertexAddFn := func(v pgraph.Vertex) error {
+				err := resources.VtoR(v).Validate()
+				return errwrap.Wrapf(err, "could not Validate() resource")
+			}
+			vertexRemoveFn := func(v pgraph.Vertex) error {
+				// wait for exit before starting new graph!
+				resources.VtoR(v).Exit() // sync
+				return nil
+			}
+			edgeCmpFn := func(e1, e2 pgraph.Edge) (bool, error) {
+				edge1 := e1.(*resources.Edge) // panic if wrong
+				edge2 := e2.(*resources.Edge) // panic if wrong
+				return edge1.Compare(edge2), nil
+			}
+			// on success, this updates the receiver graph...
+			if err := oldGraph.GraphSync(newGraph, vertexCmpFn, vertexAddFn, vertexRemoveFn, edgeCmpFn); err != nil {
+				log.Printf("Main: Error running graph sync: %v", err)
+				// unpause!
+				if !first {
+					graph.Start(first) // sync
+					converger.Start()  // after Start()
+				}
+				continue
+			}
+
+			// TODO: should we call each Res.Setup() here instead?
+
+			// add autoedges; modifies the graph only if no error
+			if err := resources.AutoEdges(oldGraph); err != nil {
+				log.Printf("Main: Error running auto edges: %v", err)
+				// unpause!
+				if !first {
+					graph.Start(first) // sync
+					converger.Start()  // after Start()
+				}
+				continue
+			}
+
+			graph.Update(oldGraph) // copy in structure of new graph
+
+			resources.AutoGroup(graph.Graph, &resources.NonReachabilityGrouper{}) // run autogroup; modifies the graph
+			// TODO: do we want to do a transitive reduction?
+			// FIXME: run a type checker that verifies all the send->recv relationships
+
+			// Call this here because at this point the graph does
+			// not know anything about the prometheus instance.
+			if err := prom.UpdatePgraphStartTime(); err != nil {
+				log.Printf("Main: Prometheus.UpdatePgraphStartTime() errored: %v", err)
+			}
+			// Start() needs to be synchronous or wait,
+			// because if half of the nodes are started and
+			// some are not ready yet and the EtcdWatch
+			// loops, we'll cause Pause() before we
+			// even got going, thus causing nil pointer errors
+			graph.Start(first) // sync
+			converger.Start()  // after Start()
+
+			log.Printf("Main: Graph: %v", graph) // show graph
+			if obj.Graphviz != "" {
+				filter := obj.GraphvizFilter
+				if filter == "" {
+					filter = "dot" // directed graph default
+				}
+				if err := graph.ExecGraphviz(filter, obj.Graphviz, hostname); err != nil {
+					log.Printf("Main: Graphviz: %v", err)
+				} else {
+					log.Printf("Main: Graphviz: Successfully generated graph!")
+				}
+			}
+			first = false
+		}
+	}()
+
+	configWatcher := recwatch.NewConfigWatcher()
+	configWatcher.Flags = recwatch.Flags{Debug: obj.Flags.Debug}
+	events := configWatcher.Events()
+	if !obj.NoWatch { // FIXME: fit this into a clean GAPI?
+		configWatcher.Add(obj.Remotes...) // add all the files...
+	} else {
+		events = nil // signal that no-watch is true
+	}
+	go func() {
+		select {
+		case err := <-configWatcher.Error():
+			obj.Exit(err) // trigger an exit!
+
+		case <-exitchan:
+			return
+		}
+	}()
+
+	// initialize the add watcher, which calls the f callback on map changes
+	convergerCb := func(f func(map[string]bool) error) (func(), error) {
+		return etcd.AddHostnameConvergedWatcher(EmbdEtcd, f)
+	}
+
+	// build remotes struct for remote ssh
+	remotes := remote.NewRemotes(
+		EmbdEtcd.LocalhostClientURLs().StringSlice(),
+		[]string{etcd.DefaultClientURL},
+		obj.Noop,
+		obj.Remotes, // list of files
+		events,      // watch for file changes
+		obj.CConns,
+		obj.AllowInteractive,
+		obj.SSHPrivIDRsa,
+		!obj.NoCaching,
+		obj.Depth,
+		prefix,
+		converger,
+		convergerCb,
+		remote.Flags{
+			Program: obj.Program,
+			Debug:   obj.Flags.Debug,
+		},
+	)
+
+	// TODO: is there any benefit to running the remotes above in the loop?
+	// wait for etcd to be running before we remote in, which we do above!
+	go remotes.Run()
+
+	if obj.GAPI == nil {
+		converger.Start() // better start this for empty graphs
+	}
+	log.Println("Main: Running...")
+
+	reterr := <-obj.exit // wait for exit signal
+
+	log.Println("Main: Destroy...")
+
+	if obj.GAPI != nil {
+		if err := obj.GAPI.Close(); err != nil {
+			err = errwrap.Wrapf(err, "the GAPI closed poorly")
+			reterr = multierr.Append(reterr, err) // list of errors
+		}
+	}
+
+	configWatcher.Close()                  // stop sending file changes to remotes
+	if err := remotes.Exit(); err != nil { // tell all the remote connections to shutdown; waits!
+		err = errwrap.Wrapf(err, "the Remote exited poorly")
+		reterr = multierr.Append(reterr, err) // list of errors
+	}
+
+	// tell inner main loop to exit
+	close(exitchan)
+
+	graph.Exit() // tells all the children to exit, and waits for them to do so
+
+	// cleanup etcd main loop last so it can process everything first
+	if err := EmbdEtcd.Destroy(); err != nil { // shutdown and cleanup etcd
+		err = errwrap.Wrapf(err, "embedded Etcd exited poorly")
+		reterr = multierr.Append(reterr, err) // list of errors
+	}
+
+	if obj.Prometheus {
+		log.Printf("Main: Prometheus: Stopping instance")
+		if err := prom.Stop(); err != nil {
+			err = errwrap.Wrapf(err, "the Prometheus instance exited poorly")
+			reterr = multierr.Append(reterr, err)
+		}
+	}
+
+	if obj.Flags.Debug {
+		log.Printf("Main: Graph: %v", graph)
+	}
+
+	// TODO: wait for each vertex to exit...
+	log.Println("Goodbye!")
+	return reterr
+}

main.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -19,576 +19,33 @@ package main
 
 import (
 	"fmt"
-	"io/ioutil"
-	"log"
 	"os"
-	"os/signal"
-	"sync"
-	"syscall"
-	"time"
 
-	"github.com/purpleidea/mgmt/converger"
-	"github.com/purpleidea/mgmt/etcd"
-	"github.com/purpleidea/mgmt/gconfig"
-	"github.com/purpleidea/mgmt/global"
-	"github.com/purpleidea/mgmt/pgraph"
-	"github.com/purpleidea/mgmt/puppet"
-	"github.com/purpleidea/mgmt/recwatch"
-	"github.com/purpleidea/mgmt/remote"
-	"github.com/purpleidea/mgmt/util"
+	mgmt "github.com/purpleidea/mgmt/lib"
+)
 
-	etcdtypes "github.com/coreos/etcd/pkg/types"
-	"github.com/coreos/pkg/capnslog"
-	"github.com/urfave/cli"
+// These constants are some global variables that are used throughout the code.
+const (
+	DEBUG   = false // add additional log messages
+	TRACE   = false // add execution flow log messages
+	VERBOSE = false // add extra log message output
 )
 
 // set at compile time
 var (
 	program string
 	version string
-	prefix  = fmt.Sprintf("/var/lib/%s/", program)
 )
 
-// signal handler
-func waitForSignal(exit chan error) error {
-	signals := make(chan os.Signal, 1)
-	signal.Notify(signals, os.Interrupt) // catch ^C
-	//signal.Notify(signals, os.Kill) // catch signals
-	signal.Notify(signals, syscall.SIGTERM)
-
-	select {
-	case sig := <-signals: // any signal will do
-		if sig == os.Interrupt {
-			log.Println("Interrupted by ^C")
-			return nil
-		} else {
-			log.Println("Interrupted by signal")
-			return fmt.Errorf("Killed by %v", sig)
-		}
-	case err := <-exit: // or a manual signal
-		log.Println("Interrupted by exit signal")
-		return err
-	}
-}
-
-// run is the main run target.
-func run(c *cli.Context) error {
-	var start = time.Now().UnixNano()
-	log.Printf("This is: %v, version: %v", program, version)
-	log.Printf("Main: Start: %v", start)
-
-	hostname, _ := os.Hostname()
-	// allow passing in the hostname, instead of using --hostname
-	if c.IsSet("file") {
-		if config := gconfig.ParseConfigFromFile(c.String("file")); config != nil {
-			if h := config.Hostname; h != "" {
-				hostname = h
-			}
-		}
-	}
-	if c.IsSet("hostname") { // override by cli
-		if h := c.String("hostname"); h != "" {
-			hostname = h
-		}
-	}
-	noop := c.Bool("noop")
-
-	seeds, err := etcdtypes.NewURLs(
-		util.FlattenListWithSplit(c.StringSlice("seeds"), []string{",", ";", " "}),
-	)
-	if err != nil && len(c.StringSlice("seeds")) > 0 {
-		log.Printf("Main: Error: seeds didn't parse correctly!")
-		return cli.NewExitError("", 1)
-	}
-	clientURLs, err := etcdtypes.NewURLs(
-		util.FlattenListWithSplit(c.StringSlice("client-urls"), []string{",", ";", " "}),
-	)
-	if err != nil && len(c.StringSlice("client-urls")) > 0 {
-		log.Printf("Main: Error: clientURLs didn't parse correctly!")
-		return cli.NewExitError("", 1)
-	}
-	serverURLs, err := etcdtypes.NewURLs(
-		util.FlattenListWithSplit(c.StringSlice("server-urls"), []string{",", ";", " "}),
-	)
-	if err != nil && len(c.StringSlice("server-urls")) > 0 {
-		log.Printf("Main: Error: serverURLs didn't parse correctly!")
-		return cli.NewExitError("", 1)
-	}
-
-	idealClusterSize := uint16(c.Int("ideal-cluster-size"))
-	if idealClusterSize < 1 {
-		log.Printf("Main: Error: idealClusterSize should be at least one!")
-		return cli.NewExitError("", 1)
-	}
-
-	if c.IsSet("file") && c.IsSet("puppet") {
-		log.Println("Main: Error: the --file and --puppet parameters cannot be used together!")
-		return cli.NewExitError("", 1)
-	}
-
-	if c.Bool("no-server") && len(c.StringSlice("remote")) > 0 {
-		// TODO: in this case, we won't be able to tunnel stuff back to
-		// here, so if we're okay with every remote graph running in an
-		// isolated mode, then this is okay. Improve on this if there's
-		// someone who really wants to be able to do this.
-		log.Println("Main: Error: the --no-server and --remote parameters cannot be used together!")
-		return cli.NewExitError("", 1)
-	}
-
-	cConns := uint16(c.Int("cconns"))
-	if cConns < 0 {
-		log.Printf("Main: Error: --cconns should be at least zero!")
-		return cli.NewExitError("", 1)
-	}
-
-	if c.IsSet("converged-timeout") && cConns > 0 && len(c.StringSlice("remote")) > c.Int("cconns") {
-		log.Printf("Main: Error: combining --converged-timeout with more remotes than available connections will never converge!")
-		return cli.NewExitError("", 1)
-	}
-
-	depth := uint16(c.Int("depth"))
-	if depth < 0 { // user should not be using this argument manually
-		log.Printf("Main: Error: negative values for --depth are not permitted!")
-		return cli.NewExitError("", 1)
-	}
-
-	if c.IsSet("prefix") && c.Bool("tmp-prefix") {
-		log.Println("Main: Error: combining --prefix and the request for a tmp prefix is illogical!")
-		return cli.NewExitError("", 1)
-	}
-	if s := c.String("prefix"); c.IsSet("prefix") && s != "" {
-		prefix = s
-	}
-
-	// make sure the working directory prefix exists
-	if c.Bool("tmp-prefix") || os.MkdirAll(prefix, 0770) != nil {
-		if c.Bool("tmp-prefix") || c.Bool("allow-tmp-prefix") {
-			if prefix, err = ioutil.TempDir("", program+"-"); err != nil {
-				log.Printf("Main: Error: Can't create temporary prefix!")
-				return cli.NewExitError("", 1)
-			}
-			log.Println("Main: Warning: Working prefix directory is temporary!")
-
-		} else {
-			log.Printf("Main: Error: Can't create prefix!")
-			return cli.NewExitError("", 1)
-		}
-	}
-	log.Printf("Main: Working prefix is: %s", prefix)
-
-	var wg sync.WaitGroup
-	exit := make(chan error) // exit signal
-	var G, fullGraph *pgraph.Graph
-
-	// exit after `max-runtime` seconds for no reason at all...
-	if i := c.Int("max-runtime"); i > 0 {
-		go func() {
-			time.Sleep(time.Duration(i) * time.Second)
-			exit <- nil
-		}()
-	}
-
-	// setup converger
-	converger := converger.NewConverger(
-		c.Int("converged-timeout"),
-		nil, // stateFn gets added in by EmbdEtcd
-	)
-	go converger.Loop(true) // main loop for converger, true to start paused
-
-	// embedded etcd
-	if len(seeds) == 0 {
-		log.Printf("Main: Seeds: No seeds specified!")
-	} else {
-		log.Printf("Main: Seeds(%v): %v", len(seeds), seeds)
-	}
-	EmbdEtcd := etcd.NewEmbdEtcd(
-		hostname,
-		seeds,
-		clientURLs,
-		serverURLs,
-		c.Bool("no-server"),
-		idealClusterSize,
-		prefix,
-		converger,
-	)
-	if EmbdEtcd == nil {
-		// TODO: verify EmbdEtcd is not nil below...
-		exit <- fmt.Errorf("Main: Etcd: Creation failed!")
-	} else if err := EmbdEtcd.Startup(); err != nil { // startup (returns when etcd main loop is running)
-		exit <- fmt.Errorf("Main: Etcd: Startup failed: %v", err)
-	}
-	convergerStateFn := func(b bool) error {
-		// exit if we are using the converged-timeout and we are the
-		// root node. otherwise, if we are a child node in a remote
-		// execution hierarchy, we should only notify our converged
-		// state and wait for the parent to trigger the exit.
-		if depth == 0 && c.Int("converged-timeout") >= 0 {
-			if b {
-				log.Printf("Converged for %d seconds, exiting!", c.Int("converged-timeout"))
-				exit <- nil // trigger an exit!
-			}
-			return nil
-		}
-		// send our individual state into etcd for others to see
-		return etcd.EtcdSetHostnameConverged(EmbdEtcd, hostname, b) // TODO: what should happen on error?
-	}
-	if EmbdEtcd != nil {
-		converger.SetStateFn(convergerStateFn)
-	}
-
-	exitchan := make(chan struct{}) // exit on close
-	go func() {
-		startchan := make(chan struct{}) // start signal
-		go func() { startchan <- struct{}{} }()
-		file := c.String("file")
-		var configchan chan error
-		var puppetchan <-chan time.Time
-		if !c.Bool("no-watch") && c.IsSet("file") {
-			configchan = recwatch.ConfigWatch(file)
-		} else if c.IsSet("puppet") {
-			interval := puppet.PuppetInterval(c.String("puppet-conf"))
-			puppetchan = time.Tick(time.Duration(interval) * time.Second)
-		}
-		log.Println("Etcd: Starting...")
-		etcdchan := etcd.EtcdWatch(EmbdEtcd)
-		first := true // first loop or not
-		for {
-			log.Println("Main: Waiting...")
-			select {
-			case <-startchan: // kick the loop once at start
-				// pass
-
-			case b := <-etcdchan:
-				if !b { // ignore the message
-					continue
-				}
-				// everything else passes through to cause a compile!
-
-			case <-puppetchan:
-				// nothing, just go on
-
-			case e := <-configchan:
-				if c.Bool("no-watch") {
-					continue // not ready to read config
-				}
-				if e != nil {
-					exit <- e // trigger exit
-					continue
-					//return // TODO: return or wait for exitchan?
-				}
-			// XXX: case compile_event: ...
-			// ...
-			case <-exitchan:
-				return
-			}
-
-			var config *gconfig.GraphConfig
-			if c.IsSet("file") {
-				config = gconfig.ParseConfigFromFile(file)
-			} else if c.IsSet("puppet") {
-				config = puppet.ParseConfigFromPuppet(c.String("puppet"), c.String("puppet-conf"))
-			}
-			if config == nil {
-				log.Printf("Config: Parse failure")
-				continue
-			}
-
-			if config.Hostname != "" && config.Hostname != hostname {
-				log.Printf("Config: Hostname changed, ignoring config!")
-				continue
-			}
-			config.Hostname = hostname // set it in case it was ""
-
-			// run graph vertex LOCK...
-			if !first { // TODO: we can flatten this check out I think
-				converger.Pause() // FIXME: add sync wait?
-				G.Pause()         // sync
-			}
-
-			// build graph from yaml file on events (eg: from etcd)
-			// we need the vertices to be paused to work on them
-			if newFullgraph, err := config.NewGraphFromConfig(fullGraph, EmbdEtcd, noop); err == nil { // keep references to all original elements
-				fullGraph = newFullgraph
-			} else {
-				log.Printf("Config: Error making new graph from config: %v", err)
-				// unpause!
-				if !first {
-					G.Start(&wg, first) // sync
-					converger.Start()   // after G.Start()
-				}
-				continue
-			}
-
-			G = fullGraph.Copy() // copy to active graph
-			// XXX: do etcd transaction out here...
-			G.AutoEdges() // add autoedges; modifies the graph
-			G.AutoGroup() // run autogroup; modifies the graph
-			// TODO: do we want to do a transitive reduction?
-
-			log.Printf("Graph: %v", G) // show graph
-			err := G.ExecGraphviz(c.String("graphviz-filter"), c.String("graphviz"))
-			if err != nil {
-				log.Printf("Graphviz: %v", err)
-			} else {
-				log.Printf("Graphviz: Successfully generated graph!")
-			}
-			G.AssociateData(converger)
-			// G.Start(...) needs to be synchronous or wait,
-			// because if half of the nodes are started and
-			// some are not ready yet and the EtcdWatch
-			// loops, we'll cause G.Pause(...) before we
-			// even got going, thus causing nil pointer errors
-			G.Start(&wg, first) // sync
-			converger.Start()   // after G.Start()
-			first = false
-		}
-	}()
-
-	configWatcher := recwatch.NewConfigWatcher()
-	events := configWatcher.Events()
-	if !c.Bool("no-watch") {
-		configWatcher.Add(c.StringSlice("remote")...) // add all the files...
-	} else {
-		events = nil // signal that no-watch is true
-	}
-	go func() {
-		select {
-		case err := <-configWatcher.Error():
-			exit <- err // trigger an exit!
-
-		case <-exitchan:
-			return
-		}
-	}()
-
-	// initialize the add watcher, which calls the f callback on map changes
-	convergerCb := func(f func(map[string]bool) error) (func(), error) {
-		return etcd.EtcdAddHostnameConvergedWatcher(EmbdEtcd, f)
-	}
-
-	// build remotes struct for remote ssh
-	remotes := remote.NewRemotes(
-		EmbdEtcd.LocalhostClientURLs().StringSlice(),
-		[]string{etcd.DefaultClientURL},
-		noop,
-		c.StringSlice("remote"), // list of files
-		events,                  // watch for file changes
-		cConns,
-		c.Bool("allow-interactive"),
-		c.String("ssh-priv-id-rsa"),
-		!c.Bool("no-caching"),
-		depth,
-		prefix,
-		converger,
-		convergerCb,
-		program,
-	)
-
-	// TODO: is there any benefit to running the remotes above in the loop?
-	// wait for etcd to be running before we remote in, which we do above!
-	go remotes.Run()
-
-	if !c.IsSet("file") && !c.IsSet("puppet") {
-		converger.Start() // better start this for empty graphs
-	}
-	log.Println("Main: Running...")
-
-	err = waitForSignal(exit) // pass in exit channel to watch
-
-	log.Println("Destroy...")
-
-	configWatcher.Close() // stop sending file changes to remotes
-	remotes.Exit()        // tell all the remote connections to shutdown; waits!
-
-	G.Exit() // tell all the children to exit
-
-	// tell inner main loop to exit
-	close(exitchan)
-
-	// cleanup etcd main loop last so it can process everything first
-	if err := EmbdEtcd.Destroy(); err != nil { // shutdown and cleanup etcd
-		log.Printf("Etcd exited poorly with: %v", err)
-	}
-
-	if global.DEBUG {
-		log.Printf("Graph: %v", G)
-	}
-
-	wg.Wait() // wait for primary go routines to exit
-
-	// TODO: wait for each vertex to exit...
-	log.Println("Goodbye!")
-	return err
-}
-
 func main() {
-	var flags int
-	if global.DEBUG || true { // TODO: remove || true
-		flags = log.LstdFlags | log.Lshortfile
+	flags := mgmt.Flags{
+		Debug:   DEBUG,
+		Trace:   TRACE,
+		Verbose: VERBOSE,
 	}
-	flags = (flags - log.Ldate) // remove the date for now
-	log.SetFlags(flags)
-
-	// un-hijack from capnslog...
-	log.SetOutput(os.Stderr)
-	if global.VERBOSE {
-		capnslog.SetFormatter(capnslog.NewLogFormatter(os.Stderr, "(etcd) ", flags))
-	} else {
-		capnslog.SetFormatter(capnslog.NewNilFormatter())
+	if err := mgmt.CLI(program, version, flags); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+		return
 	}
-
-	// test for sanity
-	if program == "" || version == "" {
-		log.Fatal("Program was not compiled correctly. Please see Makefile.")
-	}
-	app := cli.NewApp()
-	app.Name = program
-	app.Usage = "next generation config management"
-	app.Version = version
-	//app.Action = ... // without a default action, help runs
-
-	app.Commands = []cli.Command{
-		{
-			Name:    "run",
-			Aliases: []string{"r"},
-			Usage:   "run",
-			Action:  run,
-			Flags: []cli.Flag{
-				cli.StringFlag{
-					Name:   "file, f",
-					Value:  "",
-					Usage:  "graph definition to run",
-					EnvVar: "MGMT_FILE",
-				},
-				cli.BoolFlag{
-					Name:  "no-watch",
-					Usage: "do not update graph on watched graph definition file changes",
-				},
-				cli.StringFlag{
-					Name:  "code, c",
-					Value: "",
-					Usage: "code definition to run",
-				},
-				cli.StringFlag{
-					Name:  "graphviz, g",
-					Value: "",
-					Usage: "output file for graphviz data",
-				},
-				cli.StringFlag{
-					Name:  "graphviz-filter, gf",
-					Value: "dot", // directed graph default
-					Usage: "graphviz filter to use",
-				},
-				// useful for testing multiple instances on same machine
-				cli.StringFlag{
-					Name:  "hostname",
-					Value: "",
-					Usage: "hostname to use",
-				},
-				// if empty, it will startup a new server
-				cli.StringSliceFlag{
-					Name:   "seeds, s",
-					Value:  &cli.StringSlice{}, // empty slice
-					Usage:  "default etc client endpoint",
-					EnvVar: "MGMT_SEEDS",
-				},
-				// port 2379 and 4001 are common
-				cli.StringSliceFlag{
-					Name:   "client-urls",
-					Value:  &cli.StringSlice{},
-					Usage:  "list of URLs to listen on for client traffic",
-					EnvVar: "MGMT_CLIENT_URLS",
-				},
-				// port 2380 and 7001 are common
-				cli.StringSliceFlag{
-					Name:   "server-urls, peer-urls",
-					Value:  &cli.StringSlice{},
-					Usage:  "list of URLs to listen on for server (peer) traffic",
-					EnvVar: "MGMT_SERVER_URLS",
-				},
-				cli.BoolFlag{
-					Name:  "no-server",
-					Usage: "do not let other servers peer with me",
-				},
-				cli.IntFlag{
-					Name:   "ideal-cluster-size",
-					Value:  etcd.DefaultIdealClusterSize,
-					Usage:  "ideal number of server peers in cluster, only read by initial server",
-					EnvVar: "MGMT_IDEAL_CLUSTER_SIZE",
-				},
-				cli.IntFlag{
-					Name:   "converged-timeout, t",
-					Value:  -1,
-					Usage:  "exit after approximately this many seconds in a converged state",
-					EnvVar: "MGMT_CONVERGED_TIMEOUT",
-				},
-				cli.IntFlag{
-					Name:   "max-runtime",
-					Value:  0,
-					Usage:  "exit after a maximum of approximately this many seconds",
-					EnvVar: "MGMT_MAX_RUNTIME",
-				},
-				cli.BoolFlag{
-					Name:  "noop",
-					Usage: "globally force all resources into no-op mode",
-				},
-				cli.StringFlag{
-					Name:  "puppet, p",
-					Value: "",
-					Usage: "load graph from puppet, optionally takes a manifest or path to manifest file",
-				},
-				cli.StringFlag{
-					Name:  "puppet-conf",
-					Value: "",
-					Usage: "supply the path to an alternate puppet.conf file to use",
-				},
-				cli.StringSliceFlag{
-					Name:  "remote",
-					Value: &cli.StringSlice{},
-					Usage: "list of remote graph definitions to run",
-				},
-				cli.BoolFlag{
-					Name:  "allow-interactive",
-					Usage: "allow interactive prompting, such as for remote passwords",
-				},
-				cli.StringFlag{
-					Name:   "ssh-priv-id-rsa",
-					Value:  "~/.ssh/id_rsa",
-					Usage:  "default path to ssh key file, set empty to never touch",
-					EnvVar: "MGMT_SSH_PRIV_ID_RSA",
-				},
-				cli.IntFlag{
-					Name:   "cconns",
-					Value:  0,
-					Usage:  "number of maximum concurrent remote ssh connections to run, 0 for unlimited",
-					EnvVar: "MGMT_CCONNS",
-				},
-				cli.BoolFlag{
-					Name:  "no-caching",
-					Usage: "don't allow remote caching of remote execution binary",
-				},
-				cli.IntFlag{
-					Name:   "depth",
-					Hidden: true, // internal use only
-					Value:  0,
-					Usage:  "specify depth in remote hierarchy",
-				},
-				cli.StringFlag{
-					Name:   "prefix",
-					Usage:  "specify a path to the working prefix directory",
-					EnvVar: "MGMT_PREFIX",
-				},
-				cli.BoolFlag{
-					Name:  "tmp-prefix",
-					Usage: "request a pseudo-random, temporary prefix to be used",
-				},
-				cli.BoolFlag{
-					Name:  "allow-tmp-prefix",
-					Usage: "allow creation of a new temporary prefix if main prefix is unavailable",
-				},
-			},
-		},
-	}
-	app.EnableBashCompletion = true
-	app.Run(os.Args)
 }

misc/delta-cpu.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+# shitty cpu count control, useful for live demos
+
+minimum=1	# don't decrease below this number of cpus
+maximum=8	# don't increase above this number of cpus
+count=1		# initial count
+factor=3
+function output() {
+count=$1	# arg!
+cat << EOF > ~/code/mgmt/examples/virt4.yaml
+---
+graph: mygraph
+resources:
+  virt:
+  - name: mgmt4
+    meta:
+      limit: .inf
+      burst: 0
+    uri: 'qemu:///session'
+    cpus: $count
+    maxcpus: $maximum
+    memory: 524288
+    boot:
+    - hd
+    disk:
+    - type: qcow2
+      source: "~/.local/share/libvirt/images/fedora-23-scratch.qcow2"
+    state: running
+    transient: false
+edges: []
+comment: "qemu-img create -b fedora-23.qcow2 -f qcow2 fedora-23-scratch.qcow2"
+EOF
+}
+#tput cuu 1 && tput el	# remove last line
+str=''
+tnuoc=$((maximum-count))	# backwards count
+count2=$((count * factor))
+tnuoc2=$((tnuoc * factor))
+left=`yes '>' | head -$count2 | paste -s -d '' -`
+right=`yes ' ' | head -$tnuoc2 | paste -s -d '' -`
+str="${left}${right}"
+_min=$((minimum-1))
+_max=$((maximum+1))
+reset	# clean up once...
+output $count	# call function
+while true; do
+
+	read -n1 -r -s -p "CPUs count is: $count; ${str} Press +/- key to adjust." key
+	if [ "$key" = "q" ] || [ "$key" = "Q" ]; then
+		echo	# newline
+		exit
+	fi
+	if [ ! "$key" = "+" ] && [ ! "$key" = "-" ] && [ ! "$key" = "=" ] && [ ! "$key" = "_" ]; then	# wrong key
+		reset	# woops, reset it all...
+		continue
+	fi
+	if [ "$key" == "+" ] || [ "$key" == "=" ]; then
+		count=$((count+1))
+	fi
+	if [ "$key" == "-" ] || [ "$key" == "_" ]; then
+		count=$((count-1))
+	fi
+	if [ $count -eq $_min ]; then	# min
+		count=$minimum
+	fi
+	if [ $count -eq $_max ]; then	# max
+		count=$maximum
+	fi
+
+	tnuoc=$((maximum-count))	# backwards count
+	#echo "count is: $count"
+	#echo "tnuoc is: $tnuoc"
+	count2=$((count * factor))
+	tnuoc2=$((tnuoc * factor))
+	left=`yes '>' | head -$count2 | paste -s -d '' -`
+	right=`yes ' ' | head -$tnuoc2 | paste -s -d '' -`
+	str="${left}${right}"
+	#echo "str is: $str"
+	echo -ne '\r'	# backup
+	output $count	# call function
+done

misc/make-deps.sh

@@ -11,13 +11,42 @@ fi
 
 sudo_command=$(which sudo)
 
+YUM=`which yum 2>/dev/null`
+DNF=`which dnf 2>/dev/null`
+APT=`which apt-get 2>/dev/null`
+BREW=`which brew 2>/dev/null`
+PACMAN=`which pacman 2>/dev/null`
+
+# if DNF is available use it
+if [ -x "$DNF" ]; then
+	YUM=$DNF
+fi
+
+if [ -z "$YUM" -a -z "$APT" -a -z "$BREW" -a -z "$PACMAN" ]; then
+	echo "The package managers can't be found."
+	exit 1
+fi
+
+if [ ! -z "$YUM" ]; then
+	$sudo_command $YUM install -y libvirt-devel
+	$sudo_command $YUM install -y augeas-devel
+
+fi
+if [ ! -z "$APT" ]; then
+	$sudo_command $APT install -y libvirt-dev || true
+	$sudo_command $APT install -y libaugeas-dev || true
+	$sudo_command $APT install -y libpcap0.8-dev || true
+fi
+
+if [ ! -z "$BREW" ]; then
+	$BREW install libvirt || true
+fi
+
+if [ ! -z "$PACMAN" ]; then
+	$sudo_command $PACMAN -S --noconfirm libvirt augeas libpcap
+fi
+
 if [ $travis -eq 0 ]; then
-	YUM=`which yum 2>/dev/null`
-	APT=`which apt-get 2>/dev/null`
-	if [ -z "$YUM" -a -z "$APT" ]; then
-		echo "The package managers can't be found."
-		exit 1
-	fi
 	if [ ! -z "$YUM" ]; then
 		# some go dependencies are stored in mercurial
 		$sudo_command $YUM install -y golang golang-googlecode-tools-stringer hg
@@ -29,17 +58,19 @@ if [ $travis -eq 0 ]; then
 		# one of these two golang tools packages should work on debian
 		$sudo_command $APT install -y golang-golang-x-tools || true
 		$sudo_command $APT install -y golang-go.tools || true
-		$sudo_command $APT install -y libpcap0.8-dev || true
+	fi
+	if [ ! -z "$PACMAN" ]; then
+		$sudo_command $PACMAN -S --noconfirm go
 	fi
 fi
 
 # if golang is too old, we don't want to fail with an obscure error later
-if go version | grep 'go1\.[0123]\.'; then
-	echo "mgmt requires go1.4 or higher."
+if go version | grep 'go1\.[012345]\.'; then
+	echo "mgmt requires go1.6 or higher."
 	exit 1
 fi
 
-go get ./...	# get all the go dependencies
+go get -d ./...	# get all the go dependencies
 [ -e "$GOBIN/mgmt" ] && rm -f "$GOBIN/mgmt"	# the `go get` version has no -X
 # vet is built-in in go 1.6 - we check for go vet command
 go vet 1> /dev/null 2>&1
@@ -49,4 +80,5 @@ if [[ $ret != 0 ]]; then
 fi
 go get golang.org/x/tools/cmd/stringer	# for automatic stringer-ing
 go get github.com/golang/lint/golint	# for `golint`-ing
+go get github.com/alecthomas/gometalinter && gometalinter --install	# bonus
 cd "$XPWD" >/dev/null

misc/mgmt.service

@@ -5,7 +5,7 @@ After=systemd-networkd.service
 Requires=systemd-networkd.service
 
 [Service]
-ExecStart=/usr/bin/mgmt run ${OPTS}
+ExecStart=/usr/bin/mgmt run $OPTS
 RestartSec=5s
 Restart=always
 

pgp/pgp.go

@@ -0,0 +1,230 @@
+// Mgmt
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgp
+
+import (
+	"bufio"
+	"bytes"
+	"crypto"
+	"encoding/base64"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+
+	errwrap "github.com/pkg/errors"
+	"golang.org/x/crypto/openpgp"
+	"golang.org/x/crypto/openpgp/packet"
+)
+
+// DefaultKeyringFile is the default file name for keyrings.
+const DefaultKeyringFile = "keyring.pgp"
+
+// CONFIG set default Hash.
+var CONFIG packet.Config
+
+func init() {
+	CONFIG.DefaultHash = crypto.SHA256
+}
+
+// PGP contains base entity.
+type PGP struct {
+	Entity *openpgp.Entity
+}
+
+// Import private key from defined path.
+func Import(privKeyPath string) (*PGP, error) {
+
+	privKeyFile, err := os.Open(privKeyPath)
+	if err != nil {
+		return nil, err
+	}
+	defer privKeyFile.Close()
+
+	file := packet.NewReader(bufio.NewReader(privKeyFile))
+	entity, err := openpgp.ReadEntity(file)
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "can't read entity from path")
+	}
+
+	obj := &PGP{
+		Entity: entity,
+	}
+
+	log.Printf("PGP: Imported key: %s", obj.Entity.PrivateKey.KeyIdShortString())
+	return obj, nil
+}
+
+// Generate creates new key pair. This key pair must be saved or it will be lost.
+func Generate(name, comment, email string, hash *crypto.Hash) (*PGP, error) {
+	if hash != nil {
+		CONFIG.DefaultHash = *hash
+	}
+	// generate a new public/private key pair
+	entity, err := openpgp.NewEntity(name, comment, email, &CONFIG)
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "can't generate entity")
+	}
+
+	obj := &PGP{
+		Entity: entity,
+	}
+
+	log.Printf("PGP: Created key: %s", obj.Entity.PrivateKey.KeyIdShortString())
+	return obj, nil
+}
+
+// SaveKey writes the whole entity (including private key!) to a .gpg file.
+func (obj *PGP) SaveKey(path string) error {
+	f, err := os.Create(path)
+	if err != nil {
+		return errwrap.Wrapf(err, "can't create file from given path")
+	}
+
+	w := bufio.NewWriter(f)
+	if err != nil {
+		return errwrap.Wrapf(err, "can't create writer")
+	}
+
+	if err := obj.Entity.SerializePrivate(w, &CONFIG); err != nil {
+		return errwrap.Wrapf(err, "can't serialize private key")
+	}
+
+	for _, ident := range obj.Entity.Identities {
+		for _, sig := range ident.Signatures {
+			if err := sig.Serialize(w); err != nil {
+				return errwrap.Wrapf(err, "can't serialize signature")
+			}
+		}
+	}
+
+	if err := w.Flush(); err != nil {
+		return errwrap.Wrapf(err, "enable to flush writer")
+	}
+
+	return nil
+}
+
+// WriteFile from given buffer in specified path.
+func (obj *PGP) WriteFile(path string, buff *bytes.Buffer) error {
+	w, err := createWriter(path)
+	if err != nil {
+		return errwrap.Wrapf(err, "can't create writer")
+	}
+	buff.WriteTo(w)
+
+	if err := w.Flush(); err != nil {
+		return errwrap.Wrapf(err, "can't flush buffered data")
+	}
+	return nil
+}
+
+// CreateWriter remove duplicate function.
+func createWriter(path string) (*bufio.Writer, error) {
+	f, err := os.Create(path)
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "can't create file from given path")
+	}
+	return bufio.NewWriter(f), nil
+}
+
+// Encrypt message for specified entity.
+func (obj *PGP) Encrypt(to *openpgp.Entity, msg string) (string, error) {
+	buf, err := obj.EncryptMsg(to, msg)
+	if err != nil {
+		return "", errwrap.Wrapf(err, "can't encrypt message")
+	}
+
+	// encode to base64
+	bytes, err := ioutil.ReadAll(buf)
+	if err != nil {
+		return "", errwrap.Wrapf(err, "can't read unverified body")
+	}
+	return base64.StdEncoding.EncodeToString(bytes), nil
+}
+
+// EncryptMsg encrypts the message.
+func (obj *PGP) EncryptMsg(to *openpgp.Entity, msg string) (*bytes.Buffer, error) {
+	ents := []*openpgp.Entity{to}
+
+	buf := new(bytes.Buffer)
+	w, err := openpgp.Encrypt(buf, ents, obj.Entity, nil, nil)
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "can't encrypt message")
+	}
+
+	_, err = w.Write([]byte(msg))
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "can't write to buffer")
+	}
+
+	if err = w.Close(); err != nil {
+		return nil, errwrap.Wrapf(err, "can't close writer")
+	}
+	return buf, nil
+}
+
+// Decrypt an encrypted msg.
+func (obj *PGP) Decrypt(encString string) (string, error) {
+	entityList := openpgp.EntityList{obj.Entity}
+
+	// decode the base64 string
+	dec, err := base64.StdEncoding.DecodeString(encString)
+	if err != nil {
+		return "", errwrap.Wrapf(err, "fail at decoding encrypted string")
+	}
+
+	// decrypt it with the contents of the private key
+	md, err := openpgp.ReadMessage(bytes.NewBuffer(dec), entityList, nil, nil)
+	if err != nil {
+		return "", errwrap.Wrapf(err, "can't read message")
+	}
+
+	bytes, err := ioutil.ReadAll(md.UnverifiedBody)
+	if err != nil {
+		return "", errwrap.Wrapf(err, "can't read unverified body")
+	}
+	return string(bytes), nil
+}
+
+// GetIdentities return the first identities from current object.
+func (obj *PGP) GetIdentities() (string, error) {
+	identities := []*openpgp.Identity{}
+
+	for _, v := range obj.Entity.Identities {
+		identities = append(identities, v)
+	}
+	return identities[0].Name, nil
+}
+
+// ParseIdentity parses an identity into name, comment and email components.
+func ParseIdentity(identity string) (name, comment, email string, err error) {
+	// get name
+	n := strings.Split(identity, " <")
+	if len(n) != 2 {
+		return "", "", "", errwrap.Wrap(err, "user string mal formated")
+	}
+
+	// get email and comment
+	ec := strings.Split(n[1], "> ")
+	if len(ec) != 2 {
+		return "", "", "", errwrap.Wrap(err, "user string mal formated")
+	}
+
+	return n[0], ec[1], ec[0], nil
+}

pgraph/autoedge.go

@@ -1,104 +0,0 @@
-// Mgmt
-// Copyright (C) 2013-2016+ James Shubin and the project contributors
-// Written by James Shubin <james@shubin.ca> and the project contributors
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-// Package pgraph represents the internal "pointer graph" that we use.
-package pgraph
-
-import (
-	"fmt"
-	"log"
-
-	"github.com/purpleidea/mgmt/global"
-	"github.com/purpleidea/mgmt/resources"
-)
-
-// add edges to the vertex in a graph based on if it matches a uuid list
-func (g *Graph) addEdgesByMatchingUUIDS(v *Vertex, uuids []resources.ResUUID) []bool {
-	// search for edges and see what matches!
-	var result []bool
-
-	// loop through each uuid, and see if it matches any vertex
-	for _, uuid := range uuids {
-		var found = false
-		// uuid is a ResUUID object
-		for _, vv := range g.GetVertices() { // search
-			if v == vv { // skip self
-				continue
-			}
-			if global.DEBUG {
-				log.Printf("Compile: AutoEdge: Match: %v[%v] with UUID: %v[%v]", vv.Kind(), vv.GetName(), uuid.Kind(), uuid.GetName())
-			}
-			// we must match to an effective UUID for the resource,
-			// that is to say, the name value of a res is a helpful
-			// handle, but it is not necessarily a unique identity!
-			// remember, resources can return multiple UUID's each!
-			if resources.UUIDExistsInUUIDs(uuid, vv.GetUUIDs()) {
-				// add edge from: vv -> v
-				if uuid.Reversed() {
-					txt := fmt.Sprintf("AutoEdge: %v[%v] -> %v[%v]", vv.Kind(), vv.GetName(), v.Kind(), v.GetName())
-					log.Printf("Compile: Adding %v", txt)
-					g.AddEdge(vv, v, NewEdge(txt))
-				} else { // edges go the "normal" way, eg: pkg resource
-					txt := fmt.Sprintf("AutoEdge: %v[%v] -> %v[%v]", v.Kind(), v.GetName(), vv.Kind(), vv.GetName())
-					log.Printf("Compile: Adding %v", txt)
-					g.AddEdge(v, vv, NewEdge(txt))
-				}
-				found = true
-				break
-			}
-		}
-		result = append(result, found)
-	}
-	return result
-}
-
-// AutoEdges adds the automatic edges to the graph.
-func (g *Graph) AutoEdges() {
-	log.Println("Compile: Adding AutoEdges...")
-	for _, v := range g.GetVertices() { // for each vertexes autoedges
-		if !v.Meta().AutoEdge { // is the metaparam true?
-			continue
-		}
-		autoEdgeObj := v.AutoEdges()
-		if autoEdgeObj == nil {
-			log.Printf("%v[%v]: Config: No auto edges were found!", v.Kind(), v.GetName())
-			continue // next vertex
-		}
-
-		for { // while the autoEdgeObj has more uuids to add...
-			uuids := autoEdgeObj.Next() // get some!
-			if uuids == nil {
-				log.Printf("%v[%v]: Config: The auto edge list is empty!", v.Kind(), v.GetName())
-				break // inner loop
-			}
-			if global.DEBUG {
-				log.Println("Compile: AutoEdge: UUIDS:")
-				for i, u := range uuids {
-					log.Printf("Compile: AutoEdge: UUID%d: %v", i, u)
-				}
-			}
-
-			// match and add edges
-			result := g.addEdgesByMatchingUUIDS(v, uuids)
-
-			// report back, and find out if we should continue
-			if !autoEdgeObj.Test(result) {
-				break
-			}
-		}
-	}
-}

pgraph/graphsync.go

@@ -0,0 +1,129 @@
+// Mgmt
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgraph
+
+import (
+	"fmt"
+
+	errwrap "github.com/pkg/errors"
+)
+
+// GraphSync updates the Graph so that it matches the newGraph. It leaves
+// identical elements alone so that they don't need to be refreshed.
+// It tries to mutate existing elements into new ones, if they support this.
+// This updates the Graph on success only.
+// FIXME: should we do this with copies of the vertex resources?
+// FIXME: add test cases
+func (obj *Graph) GraphSync(newGraph *Graph, vertexCmpFn func(Vertex, Vertex) (bool, error), vertexAddFn func(Vertex) error, vertexRemoveFn func(Vertex) error, edgeCmpFn func(Edge, Edge) (bool, error)) error {
+
+	oldGraph := obj.Copy() // work on a copy of the old graph
+	if oldGraph == nil {
+		var err error
+		oldGraph, err = NewGraph(newGraph.GetName()) // copy over the name
+		if err != nil {
+			return errwrap.Wrapf(err, "GraphSync failed")
+		}
+	}
+	oldGraph.SetName(newGraph.GetName()) // overwrite the name
+
+	var lookup = make(map[Vertex]Vertex)
+	var vertexKeep []Vertex // list of vertices which are the same in new graph
+	var edgeKeep []Edge     // list of vertices which are the same in new graph
+
+	for v := range newGraph.Adjacency() { // loop through the vertices (resources)
+		var vertex Vertex
+		// step one, direct compare with res.Compare
+		if vertex == nil { // redundant guard for consistency
+			fn := func(vv Vertex) (bool, error) {
+				b, err := vertexCmpFn(vv, v)
+				return b, errwrap.Wrapf(err, "vertexCmpFn failed")
+			}
+			var err error
+			vertex, err = oldGraph.VertexMatchFn(fn)
+			if err != nil {
+				return errwrap.Wrapf(err, "VertexMatchFn failed")
+			}
+		}
+
+		// TODO: consider adding a mutate API.
+		// step two, try and mutate with res.Mutate
+		//if vertex == nil { // not found yet...
+		//	vertex = oldGraph.MutateMatch(res)
+		//}
+
+		if vertex == nil { // no match found yet
+			if err := vertexAddFn(v); err != nil {
+				return errwrap.Wrapf(err, "vertexAddFn failed")
+			}
+			vertex = v
+			oldGraph.AddVertex(vertex) // call standalone in case not part of an edge
+		}
+		lookup[v] = vertex                      // used for constructing edges
+		vertexKeep = append(vertexKeep, vertex) // append
+	}
+
+	// get rid of any vertices we shouldn't keep (that aren't in new graph)
+	for v := range oldGraph.Adjacency() {
+		if !VertexContains(v, vertexKeep) {
+			if err := vertexRemoveFn(v); err != nil {
+				return errwrap.Wrapf(err, "vertexRemoveFn failed")
+			}
+			oldGraph.DeleteVertex(v)
+		}
+	}
+
+	// compare edges
+	for v1 := range newGraph.Adjacency() { // loop through the vertices (resources)
+		for v2, e := range newGraph.Adjacency()[v1] {
+			// we have an edge!
+			// lookup vertices (these should exist now)
+			vertex1, exists1 := lookup[v1]
+			vertex2, exists2 := lookup[v2]
+			if !exists1 || !exists2 { // no match found, bug?
+				//if vertex1 == nil || vertex2 == nil { // no match found
+				return fmt.Errorf("new vertices weren't found") // programming error
+			}
+
+			edge, exists := oldGraph.Adjacency()[vertex1][vertex2]
+			if !exists {
+				edge = e // use edge
+			} else if b, err := edgeCmpFn(edge, e); err != nil {
+				return errwrap.Wrapf(err, "edgeCmpFn failed")
+			} else if !b {
+				edge = e // overwrite edge
+			}
+
+			oldGraph.Adjacency()[vertex1][vertex2] = edge // store it (AddEdge)
+			edgeKeep = append(edgeKeep, edge)             // mark as saved
+		}
+	}
+
+	// delete unused edges
+	for v1 := range oldGraph.Adjacency() {
+		for _, e := range oldGraph.Adjacency()[v1] {
+			// we have an edge!
+			if !EdgeContains(e, edgeKeep) {
+				oldGraph.DeleteEdge(e)
+			}
+		}
+	}
+
+	// success
+	*obj = *oldGraph // save old graph
+	return nil
+}

pgraph/graphviz.go

@@ -0,0 +1,118 @@
+// Mgmt
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgraph // TODO: this should be a subpackage
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"strconv"
+	"syscall"
+)
+
+// Graphviz outputs the graph in graphviz format.
+// https://en.wikipedia.org/wiki/DOT_%28graph_description_language%29
+func (g *Graph) Graphviz() (out string) {
+	//digraph g {
+	//	label="hello world";
+	//	node [shape=box];
+	//	A [label="A"];
+	//	B [label="B"];
+	//	C [label="C"];
+	//	D [label="D"];
+	//	E [label="E"];
+	//	A -> B [label=f];
+	//	B -> C [label=g];
+	//	D -> E [label=h];
+	//}
+	out += fmt.Sprintf("digraph %s {\n", g.GetName())
+	out += fmt.Sprintf("\tlabel=\"%s\";\n", g.GetName())
+	//out += "\tnode [shape=box];\n"
+	str := ""
+	for i := range g.Adjacency() { // reverse paths
+		out += fmt.Sprintf("\t\"%s\" [label=\"%s\"];\n", i, i)
+		for j := range g.Adjacency()[i] {
+			k := g.Adjacency()[i][j]
+			// use str for clearer output ordering
+			//if fmtBoldFn(k) { // TODO: add this sort of formatting
+			//	str += fmt.Sprintf("\t\"%s\" -> \"%s\" [label=\"%s\",style=bold];\n", i, j, k)
+			//} else {
+			str += fmt.Sprintf("\t\"%s\" -> \"%s\" [label=\"%s\"];\n", i, j, k)
+			//}
+		}
+	}
+	out += str
+	out += "}\n"
+	return
+}
+
+// ExecGraphviz writes out the graphviz data and runs the correct graphviz
+// filter command.
+func (g *Graph) ExecGraphviz(program, filename, hostname string) error {
+
+	switch program {
+	case "dot", "neato", "twopi", "circo", "fdp":
+	default:
+		return fmt.Errorf("invalid graphviz program selected")
+	}
+
+	if filename == "" {
+		return fmt.Errorf("no filename given")
+	}
+
+	if hostname != "" {
+		filename = fmt.Sprintf("%s@%s", filename, hostname)
+	}
+
+	// run as a normal user if possible when run with sudo
+	uid, err1 := strconv.Atoi(os.Getenv("SUDO_UID"))
+	gid, err2 := strconv.Atoi(os.Getenv("SUDO_GID"))
+
+	err := ioutil.WriteFile(filename, []byte(g.Graphviz()), 0644)
+	if err != nil {
+		return fmt.Errorf("error writing to filename")
+	}
+
+	if err1 == nil && err2 == nil {
+		if err := os.Chown(filename, uid, gid); err != nil {
+			return fmt.Errorf("error changing file owner")
+		}
+	}
+
+	path, err := exec.LookPath(program)
+	if err != nil {
+		return fmt.Errorf("the Graphviz program is missing")
+	}
+
+	out := fmt.Sprintf("%s.png", filename)
+	cmd := exec.Command(path, "-Tpng", fmt.Sprintf("-o%s", out), filename)
+
+	if err1 == nil && err2 == nil {
+		cmd.SysProcAttr = &syscall.SysProcAttr{}
+		cmd.SysProcAttr.Credential = &syscall.Credential{
+			Uid: uint32(uid),
+			Gid: uint32(gid),
+		}
+	}
+	_, err = cmd.Output()
+	if err != nil {
+		return fmt.Errorf("error writing to image")
+	}
+	return nil
+}

pgraph/subgraph.go

@@ -0,0 +1,106 @@
+// Mgmt
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgraph
+
+// AddGraph adds the set of edges and vertices of a graph to the existing graph.
+func (g *Graph) AddGraph(graph *Graph) {
+	g.addEdgeVertexGraphHelper(nil, graph, nil, false, false)
+}
+
+// AddEdgeVertexGraph adds a directed edge to the graph from a vertex.
+// This is useful for flattening the relationship between a subgraph and an
+// existing graph, without having to run the subgraph recursively. It adds the
+// maximum number of edges, creating a relationship to every vertex.
+func (g *Graph) AddEdgeVertexGraph(vertex Vertex, graph *Graph, edgeGenFn func(v1, v2 Vertex) Edge) {
+	g.addEdgeVertexGraphHelper(vertex, graph, edgeGenFn, false, false)
+}
+
+// AddEdgeVertexGraphLight adds a directed edge to the graph from a vertex.
+// This is useful for flattening the relationship between a subgraph and an
+// existing graph, without having to run the subgraph recursively. It adds the
+// minimum number of edges, creating a relationship to the vertices with
+// indegree equal to zero.
+func (g *Graph) AddEdgeVertexGraphLight(vertex Vertex, graph *Graph, edgeGenFn func(v1, v2 Vertex) Edge) {
+	g.addEdgeVertexGraphHelper(vertex, graph, edgeGenFn, false, true)
+}
+
+// AddEdgeGraphVertex adds a directed edge to the vertex from a graph.
+// This is useful for flattening the relationship between a subgraph and an
+// existing graph, without having to run the subgraph recursively. It adds the
+// maximum number of edges, creating a relationship from every vertex.
+func (g *Graph) AddEdgeGraphVertex(graph *Graph, vertex Vertex, edgeGenFn func(v1, v2 Vertex) Edge) {
+	g.addEdgeVertexGraphHelper(vertex, graph, edgeGenFn, true, false)
+}
+
+// AddEdgeGraphVertexLight adds a directed edge to the vertex from a graph.
+// This is useful for flattening the relationship between a subgraph and an
+// existing graph, without having to run the subgraph recursively. It adds the
+// minimum number of edges, creating a relationship from the vertices with
+// outdegree equal to zero.
+func (g *Graph) AddEdgeGraphVertexLight(graph *Graph, vertex Vertex, edgeGenFn func(v1, v2 Vertex) Edge) {
+	g.addEdgeVertexGraphHelper(vertex, graph, edgeGenFn, true, true)
+}
+
+// addEdgeVertexGraphHelper is a helper function to add a directed edges to the
+// graph from a vertex, or vice-versa. It operates in this reverse direction by
+// specifying the reverse argument as true. It is useful for flattening the
+// relationship between a subgraph and an existing graph, without having to run
+// the subgraph recursively. It adds the maximum number of edges, creating a
+// relationship to or from every vertex if the light argument is false, and if
+// it is true, it adds the minimum number of edges, creating a relationship to
+// or from the vertices with an indegree or outdegree equal to zero depending on
+// if we specified reverse or not.
+func (g *Graph) addEdgeVertexGraphHelper(vertex Vertex, graph *Graph, edgeGenFn func(v1, v2 Vertex) Edge, reverse, light bool) {
+
+	var degree map[Vertex]int // compute all of the in/outdegree's if needed
+	if light && reverse {
+		degree = graph.OutDegree()
+	} else if light { // && !reverse
+		degree = graph.InDegree()
+	}
+	for _, v := range graph.VerticesSorted() { // sort to help out edgeGenFn
+
+		// forward:
+		// we only want to add edges to indegree == 0, because every
+		// other vertex is a dependency of at least one of those
+
+		// reverse:
+		// we only want to add edges to outdegree == 0, because every
+		// other vertex is a pre-requisite to at least one of these
+		if light && degree[v] != 0 {
+			continue
+		}
+
+		g.AddVertex(v) // ensure vertex is part of the graph
+
+		if vertex != nil && reverse {
+			edge := edgeGenFn(v, vertex) // generate a new unique edge
+			g.AddEdge(v, vertex, edge)
+		} else if vertex != nil { // && !reverse
+			edge := edgeGenFn(vertex, v)
+			g.AddEdge(vertex, v, edge)
+		}
+	}
+
+	// also remember to suck in all of the graph's edges too!
+	for v1 := range graph.Adjacency() {
+		for v2, e := range graph.Adjacency()[v1] {
+			g.AddEdge(v1, v2, e)
+		}
+	}
+}

pgraph/subgraph_test.go

@@ -0,0 +1,210 @@
+// Mgmt
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgraph
+
+import (
+	"fmt"
+	"testing"
+)
+
+// TODO: unify with the other function like this...
+// TODO: where should we put our test helpers?
+func runGraphCmp(t *testing.T, g1, g2 *Graph) {
+	err := g1.GraphCmp(g2, vertexCmpFn, edgeCmpFn)
+	if err != nil {
+		t.Logf("  actual (g1): %v%v", g1, fullPrint(g1))
+		t.Logf("expected (g2): %v%v", g2, fullPrint(g2))
+		t.Logf("Cmp error:")
+		t.Errorf("%v", err)
+	}
+}
+
+// TODO: unify with the other function like this...
+func fullPrint(g *Graph) (str string) {
+	str += "\n"
+	for v := range g.Adjacency() {
+		str += fmt.Sprintf("* v: %s\n", v)
+	}
+	for v1 := range g.Adjacency() {
+		for v2, e := range g.Adjacency()[v1] {
+			str += fmt.Sprintf("* e: %s -> %s # %s\n", v1, v2, e)
+		}
+	}
+	return
+}
+
+// edgeGenFn generates unique edges for each vertex pair, assuming unique
+// vertices.
+func edgeGenFn(v1, v2 Vertex) Edge {
+	return NE(fmt.Sprintf("%s,%s", v1, v2))
+}
+
+func TestPgraphAddEdgeGraph1(t *testing.T) {
+	v1 := NV("v1")
+	v2 := NV("v2")
+	v3 := NV("v3")
+	v4 := NV("v4")
+	v5 := NV("v5")
+	e1 := NE("e1")
+	e2 := NE("e2")
+	e3 := NE("e3")
+
+	g := &Graph{}
+	g.AddEdge(v1, v3, e1)
+	g.AddEdge(v2, v3, e2)
+
+	sub := &Graph{}
+	sub.AddEdge(v4, v5, e3)
+
+	g.AddGraph(sub)
+
+	// expected (can re-use the same vertices)
+	expected := &Graph{}
+	expected.AddEdge(v1, v3, e1)
+	expected.AddEdge(v2, v3, e2)
+	expected.AddEdge(v4, v5, e3)
+
+	//expected.AddEdge(v3, v4, NE("v3,v4"))
+	//expected.AddEdge(v3, v5, NE("v3,v5"))
+
+	runGraphCmp(t, g, expected)
+}
+
+func TestPgraphAddEdgeVertexGraph1(t *testing.T) {
+	v1 := NV("v1")
+	v2 := NV("v2")
+	v3 := NV("v3")
+	v4 := NV("v4")
+	v5 := NV("v5")
+	e1 := NE("e1")
+	e2 := NE("e2")
+	e3 := NE("e3")
+
+	g := &Graph{}
+	g.AddEdge(v1, v3, e1)
+	g.AddEdge(v2, v3, e2)
+
+	sub := &Graph{}
+	sub.AddEdge(v4, v5, e3)
+
+	g.AddEdgeVertexGraph(v3, sub, edgeGenFn)
+
+	// expected (can re-use the same vertices)
+	expected := &Graph{}
+	expected.AddEdge(v1, v3, e1)
+	expected.AddEdge(v2, v3, e2)
+	expected.AddEdge(v4, v5, e3)
+
+	expected.AddEdge(v3, v4, NE("v3,v4"))
+	expected.AddEdge(v3, v5, NE("v3,v5"))
+
+	runGraphCmp(t, g, expected)
+}
+
+func TestPgraphAddEdgeGraphVertex1(t *testing.T) {
+	v1 := NV("v1")
+	v2 := NV("v2")
+	v3 := NV("v3")
+	v4 := NV("v4")
+	v5 := NV("v5")
+	e1 := NE("e1")
+	e2 := NE("e2")
+	e3 := NE("e3")
+
+	g := &Graph{}
+	g.AddEdge(v1, v3, e1)
+	g.AddEdge(v2, v3, e2)
+
+	sub := &Graph{}
+	sub.AddEdge(v4, v5, e3)
+
+	g.AddEdgeGraphVertex(sub, v3, edgeGenFn)
+
+	// expected (can re-use the same vertices)
+	expected := &Graph{}
+	expected.AddEdge(v1, v3, e1)
+	expected.AddEdge(v2, v3, e2)
+	expected.AddEdge(v4, v5, e3)
+
+	expected.AddEdge(v4, v3, NE("v4,v3"))
+	expected.AddEdge(v5, v3, NE("v5,v3"))
+
+	runGraphCmp(t, g, expected)
+}
+
+func TestPgraphAddEdgeVertexGraphLight1(t *testing.T) {
+	v1 := NV("v1")
+	v2 := NV("v2")
+	v3 := NV("v3")
+	v4 := NV("v4")
+	v5 := NV("v5")
+	e1 := NE("e1")
+	e2 := NE("e2")
+	e3 := NE("e3")
+
+	g := &Graph{}
+	g.AddEdge(v1, v3, e1)
+	g.AddEdge(v2, v3, e2)
+
+	sub := &Graph{}
+	sub.AddEdge(v4, v5, e3)
+
+	g.AddEdgeVertexGraphLight(v3, sub, edgeGenFn)
+
+	// expected (can re-use the same vertices)
+	expected := &Graph{}
+	expected.AddEdge(v1, v3, e1)
+	expected.AddEdge(v2, v3, e2)
+	expected.AddEdge(v4, v5, e3)
+
+	expected.AddEdge(v3, v4, NE("v3,v4"))
+	//expected.AddEdge(v3, v5, NE("v3,v5")) // not needed with light
+
+	runGraphCmp(t, g, expected)
+}
+
+func TestPgraphAddEdgeGraphVertexLight1(t *testing.T) {
+	v1 := NV("v1")
+	v2 := NV("v2")
+	v3 := NV("v3")
+	v4 := NV("v4")
+	v5 := NV("v5")
+	e1 := NE("e1")
+	e2 := NE("e2")
+	e3 := NE("e3")
+
+	g := &Graph{}
+	g.AddEdge(v1, v3, e1)
+	g.AddEdge(v2, v3, e2)
+
+	sub := &Graph{}
+	sub.AddEdge(v4, v5, e3)
+
+	g.AddEdgeGraphVertexLight(sub, v3, edgeGenFn)
+
+	// expected (can re-use the same vertices)
+	expected := &Graph{}
+	expected.AddEdge(v1, v3, e1)
+	expected.AddEdge(v2, v3, e2)
+	expected.AddEdge(v4, v5, e3)
+
+	//expected.AddEdge(v4, v3, NE("v4,v3")) // not needed with light
+	expected.AddEdge(v5, v3, NE("v5,v3"))
+
+	runGraphCmp(t, g, expected)
+}

prometheus/prometheus.go

@@ -0,0 +1,263 @@
+// Mgmt
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// Package prometheus provides functions that are useful to control and manage
+// the build-in prometheus instance.
+package prometheus
+
+import (
+	"errors"
+	"net/http"
+	"strconv"
+	"sync"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
+	errwrap "github.com/pkg/errors"
+)
+
+// DefaultPrometheusListen is registered in
+// https://github.com/prometheus/prometheus/wiki/Default-port-allocations
+const DefaultPrometheusListen = "127.0.0.1:9233"
+
+// ResState represents the status of a resource.
+type ResState int
+
+const (
+	// ResStateOK represents a working resource
+	ResStateOK ResState = iota
+	// ResStateSoftFail represents a resource in soft fail (will be retried)
+	ResStateSoftFail
+	// ResStateHardFail represents a resource in hard fail (will NOT be retried)
+	ResStateHardFail
+)
+
+// Prometheus is the struct that contains information about the
+// prometheus instance. Run Init() on it.
+type Prometheus struct {
+	Listen string // the listen specification for the net/http server
+
+	checkApplyTotal        *prometheus.CounterVec // total of CheckApplies that have been triggered
+	pgraphStartTimeSeconds prometheus.Gauge       // process start time in seconds since unix epoch
+	managedResources       *prometheus.GaugeVec   // Resources we manage now
+	failedResourcesTotal   *prometheus.CounterVec // Total of failures since mgmt has started
+	failedResources        *prometheus.GaugeVec   // Number of current resources
+
+	resourcesState map[string]resStateWithKind // Maps the resources with their current kind/state
+	mutex          *sync.Mutex                 // Mutex used to update resourcesState
+}
+
+// resStateWithKind is used to count the failures by kind
+type resStateWithKind struct {
+	state ResState
+	kind  string
+}
+
+// Init some parameters - currently the Listen address.
+func (obj *Prometheus) Init() error {
+	if len(obj.Listen) == 0 {
+		obj.Listen = DefaultPrometheusListen
+	}
+
+	obj.mutex = &sync.Mutex{}
+	obj.resourcesState = make(map[string]resStateWithKind)
+
+	obj.checkApplyTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "mgmt_checkapply_total",
+			Help: "Number of CheckApply that have run.",
+		},
+		// Labels for this metric.
+		// kind: resource type: Svc, File, ...
+		// apply: if the CheckApply happened in "apply" mode
+		// eventful: did the CheckApply generate an event
+		// errorful: did the CheckApply generate an error
+		[]string{"kind", "apply", "eventful", "errorful"},
+	)
+	prometheus.MustRegister(obj.checkApplyTotal)
+
+	obj.pgraphStartTimeSeconds = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "mgmt_graph_start_time_seconds",
+			Help: "Start time of the current graph since unix epoch in seconds.",
+		},
+	)
+	prometheus.MustRegister(obj.pgraphStartTimeSeconds)
+
+	obj.managedResources = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "mgmt_resources",
+			Help: "Number of managed resources.",
+		},
+		// kind: resource type: Svc, File, ...
+		[]string{"kind"},
+	)
+	prometheus.MustRegister(obj.managedResources)
+
+	obj.failedResourcesTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "mgmt_failures_total",
+			Help: "Total of failed resources.",
+		},
+		// kind: resource type: Svc, File, ...
+		// failure: soft or hard
+		[]string{"kind", "failure"},
+	)
+	prometheus.MustRegister(obj.failedResourcesTotal)
+
+	obj.failedResources = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "mgmt_failures",
+			Help: "Number of failing resources.",
+		},
+		// kind: resource type: Svc, File, ...
+		// failure: soft or hard
+		[]string{"kind", "failure"},
+	)
+	prometheus.MustRegister(obj.failedResources)
+
+	return nil
+}
+
+// Start runs a http server in a go routine, that responds to /metrics
+// as prometheus would expect.
+func (obj *Prometheus) Start() error {
+	http.Handle("/metrics", promhttp.Handler())
+	go http.ListenAndServe(obj.Listen, nil)
+	return nil
+}
+
+// Stop the http server.
+func (obj *Prometheus) Stop() error {
+	// TODO: There is no way in go < 1.8 to stop a http server.
+	// https://stackoverflow.com/questions/39320025/go-how-to-stop-http-listenandserve/41433555#41433555
+	return nil
+}
+
+// UpdateCheckApplyTotal refreshes the failing gauge by parsing the internal
+// state map.
+func (obj *Prometheus) UpdateCheckApplyTotal(kind string, apply, eventful, errorful bool) error {
+	if obj == nil {
+		return nil // happens when mgmt is launched without --prometheus
+	}
+	labels := prometheus.Labels{"kind": kind, "apply": strconv.FormatBool(apply), "eventful": strconv.FormatBool(eventful), "errorful": strconv.FormatBool(errorful)}
+	metric := obj.checkApplyTotal.With(labels)
+	metric.Inc()
+	return nil
+}
+
+// UpdatePgraphStartTime updates the mgmt_graph_start_time_seconds metric
+// to the current timestamp.
+func (obj *Prometheus) UpdatePgraphStartTime() error {
+	if obj == nil {
+		return nil // happens when mgmt is launched without --prometheus
+	}
+	obj.pgraphStartTimeSeconds.SetToCurrentTime()
+	return nil
+}
+
+// AddManagedResource increments the Managed Resource counter and updates the resource status.
+func (obj *Prometheus) AddManagedResource(resUUID string, rtype string) error {
+	if obj == nil {
+		return nil // happens when mgmt is launched without --prometheus
+	}
+	obj.managedResources.With(prometheus.Labels{"kind": rtype}).Inc()
+	if err := obj.UpdateState(resUUID, rtype, ResStateOK); err != nil {
+		return errwrap.Wrapf(err, "can't update the resource status in the map")
+	}
+	return nil
+}
+
+// RemoveManagedResource decrements the Managed Resource counter and updates the resource status.
+func (obj *Prometheus) RemoveManagedResource(resUUID string, rtype string) error {
+	if obj == nil {
+		return nil // happens when mgmt is launched without --prometheus
+	}
+	obj.managedResources.With(prometheus.Labels{"kind": rtype}).Dec()
+	if err := obj.deleteState(resUUID); err != nil {
+		return errwrap.Wrapf(err, "can't remove the resource status from the map")
+	}
+	return nil
+}
+
+// deleteState removes the resources for the state map and re-populates the failing gauge.
+func (obj *Prometheus) deleteState(resUUID string) error {
+	if obj == nil {
+		return nil // happens when mgmt is launched without --prometheus
+	}
+	obj.mutex.Lock()
+	delete(obj.resourcesState, resUUID)
+	obj.mutex.Unlock()
+	if err := obj.updateFailingGauge(); err != nil {
+		return errwrap.Wrapf(err, "can't update the failing gauge")
+	}
+	return nil
+}
+
+// UpdateState updates the state of the resources in our internal state map
+// then triggers a refresh of the failing gauge.
+func (obj *Prometheus) UpdateState(resUUID string, rtype string, newState ResState) error {
+	defer obj.updateFailingGauge()
+	if obj == nil {
+		return nil // happens when mgmt is launched without --prometheus
+	}
+	obj.mutex.Lock()
+	obj.resourcesState[resUUID] = resStateWithKind{state: newState, kind: rtype}
+	obj.mutex.Unlock()
+	if newState != ResStateOK {
+		var strState string
+		if newState == ResStateSoftFail {
+			strState = "soft"
+		} else if newState == ResStateHardFail {
+			strState = "hard"
+		} else {
+			return errors.New("state should be soft or hard failure")
+		}
+		obj.failedResourcesTotal.With(prometheus.Labels{"kind": rtype, "failure": strState}).Inc()
+	}
+	return nil
+}
+
+// updateFailingGauge refreshes the failing gauge by parsking the internal
+// state map.
+func (obj *Prometheus) updateFailingGauge() error {
+	if obj == nil {
+		return nil // happens when mgmt is launched without --prometheus
+	}
+	var softFails, hardFails map[string]float64
+	softFails = make(map[string]float64)
+	hardFails = make(map[string]float64)
+	for _, v := range obj.resourcesState {
+		if v.state == ResStateSoftFail {
+			softFails[v.kind]++
+		} else if v.state == ResStateHardFail {
+			hardFails[v.kind]++
+		}
+	}
+	// TODO: we might want to Zero the metrics we are not using
+	// because in prometheus design the metrics keep living for some time
+	// even after they are removed.
+	obj.failedResources.Reset()
+	for k, v := range softFails {
+		obj.failedResources.With(prometheus.Labels{"kind": k, "failure": "soft"}).Set(v)
+	}
+	for k, v := range hardFails {
+		obj.failedResources.With(prometheus.Labels{"kind": k, "failure": "hard"}).Set(v)
+	}
+	return nil
+}

puppet/gapi.go

@@ -0,0 +1,150 @@
+// Mgmt
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package puppet
+
+import (
+	"fmt"
+	"log"
+	"sync"
+	"time"
+
+	"github.com/purpleidea/mgmt/gapi"
+	"github.com/purpleidea/mgmt/pgraph"
+)
+
+// GAPI implements the main puppet GAPI interface.
+type GAPI struct {
+	PuppetParam *string // puppet mode to run; nil if undefined
+	PuppetConf  string  // the path to an alternate puppet.conf file
+
+	data        gapi.Data
+	initialized bool
+	closeChan   chan struct{}
+	wg          sync.WaitGroup // sync group for tunnel go routines
+}
+
+// NewGAPI creates a new puppet GAPI struct and calls Init().
+func NewGAPI(data gapi.Data, puppetParam *string, puppetConf string) (*GAPI, error) {
+	obj := &GAPI{
+		PuppetParam: puppetParam,
+		PuppetConf:  puppetConf,
+	}
+	return obj, obj.Init(data)
+}
+
+// Init initializes the puppet GAPI struct.
+func (obj *GAPI) Init(data gapi.Data) error {
+	if obj.initialized {
+		return fmt.Errorf("already initialized")
+	}
+	if obj.PuppetParam == nil {
+		return fmt.Errorf("the PuppetParam param must be specified")
+	}
+	obj.data = data // store for later
+	obj.closeChan = make(chan struct{})
+	obj.initialized = true
+	return nil
+}
+
+// Graph returns a current Graph.
+func (obj *GAPI) Graph() (*pgraph.Graph, error) {
+	if !obj.initialized {
+		return nil, fmt.Errorf("the puppet GAPI is not initialized")
+	}
+	config := ParseConfigFromPuppet(*obj.PuppetParam, obj.PuppetConf)
+	if config == nil {
+		return nil, fmt.Errorf("function ParseConfigFromPuppet returned nil")
+	}
+	g, err := config.NewGraphFromConfig(obj.data.Hostname, obj.data.World, obj.data.Noop)
+	return g, err
+}
+
+// Next returns nil errors every time there could be a new graph.
+func (obj *GAPI) Next() chan gapi.Next {
+	puppetChan := func() <-chan time.Time { // helper function
+		return time.Tick(time.Duration(RefreshInterval(obj.PuppetConf)) * time.Second)
+	}
+	ch := make(chan gapi.Next)
+	obj.wg.Add(1)
+	go func() {
+		defer obj.wg.Done()
+		defer close(ch) // this will run before the obj.wg.Done()
+		if !obj.initialized {
+			next := gapi.Next{
+				Err:  fmt.Errorf("the puppet GAPI is not initialized"),
+				Exit: true, // exit, b/c programming error?
+			}
+			ch <- next
+			return
+		}
+		startChan := make(chan struct{}) // start signal
+		close(startChan)                 // kick it off!
+
+		pChan := make(<-chan time.Time)
+		// NOTE: we don't look at obj.data.NoConfigWatch since emulating
+		// puppet means we do not switch graphs on code changes anyways.
+		if obj.data.NoStreamWatch {
+			pChan = nil
+		} else {
+			pChan = puppetChan()
+		}
+
+		for {
+			select {
+			case <-startChan: // kick the loop once at start
+				startChan = nil // disable
+				// pass
+			case _, ok := <-pChan:
+				if !ok { // the channel closed!
+					return
+				}
+			case <-obj.closeChan:
+				return
+			}
+
+			log.Printf("Puppet: Generating new graph...")
+			if obj.data.NoStreamWatch {
+				pChan = nil
+			} else {
+				pChan = puppetChan() // TODO: okay to update interval in case it changed?
+			}
+			next := gapi.Next{
+				//Exit: true, // TODO: for permanent shutdown!
+				Err: nil,
+			}
+			select {
+			case ch <- next: // trigger a run (send a msg)
+			// unblock if we exit while waiting to send!
+			case <-obj.closeChan:
+				return
+			}
+		}
+	}()
+	return ch
+}
+
+// Close shuts down the Puppet GAPI.
+func (obj *GAPI) Close() error {
+	if !obj.initialized {
+		return fmt.Errorf("the puppet GAPI is not initialized")
+	}
+	close(obj.closeChan)
+	obj.wg.Wait()
+	obj.initialized = false // closed = true
+	return nil
+}

puppet/puppet.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -26,17 +26,18 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/purpleidea/mgmt/gconfig"
-	"github.com/purpleidea/mgmt/global"
+	"github.com/purpleidea/mgmt/yamlgraph"
 )
 
 const (
 	// PuppetYAMLBufferSize is the maximum buffer size for the yaml input data
 	PuppetYAMLBufferSize = 65535
+	// Debug is a local debug constant used in this module
+	Debug = false // FIXME: integrate with global debug flag
 )
 
 func runPuppetCommand(cmd *exec.Cmd) ([]byte, error) {
-	if global.DEBUG {
+	if Debug {
 		log.Printf("Puppet: running command: %v", cmd)
 	}
 
@@ -71,7 +72,7 @@ func runPuppetCommand(cmd *exec.Cmd) ([]byte, error) {
 		// will choke on an oversized slice. http://stackoverflow.com/a/33726617/3356612
 		result = append(result, data[0:count]...)
 	}
-	if global.DEBUG {
+	if Debug {
 		log.Printf("Puppet: read %v bytes of data from puppet", len(result))
 	}
 	for scanner := bufio.NewScanner(stderr); scanner.Scan(); {
@@ -87,7 +88,7 @@ func runPuppetCommand(cmd *exec.Cmd) ([]byte, error) {
 
 // ParseConfigFromPuppet takes a special puppet param string and config and
 // returns the graph configuration structure.
-func ParseConfigFromPuppet(puppetParam, puppetConf string) *gconfig.GraphConfig {
+func ParseConfigFromPuppet(puppetParam, puppetConf string) *yamlgraph.GraphConfig {
 	var puppetConfArg string
 	if puppetConf != "" {
 		puppetConfArg = "--config=" + puppetConf
@@ -104,7 +105,7 @@ func ParseConfigFromPuppet(puppetParam, puppetConf string) *gconfig.GraphConfig
 
 	log.Println("Puppet: launching translator")
 
-	var config gconfig.GraphConfig
+	var config yamlgraph.GraphConfig
 	if data, err := runPuppetCommand(cmd); err != nil {
 		return nil
 	} else if err := config.Parse(data); err != nil {
@@ -115,9 +116,9 @@ func ParseConfigFromPuppet(puppetParam, puppetConf string) *gconfig.GraphConfig
 	return &config
 }
 
-// PuppetInterval returns the graph refresh interval from the puppet configuration.
-func PuppetInterval(puppetConf string) int {
-	if global.DEBUG {
+// RefreshInterval returns the graph refresh interval from the puppet configuration.
+func RefreshInterval(puppetConf string) int {
+	if Debug {
 		log.Printf("Puppet: determining graph refresh interval")
 	}
 	var cmd *exec.Cmd

recwatch/configwatch.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -20,12 +20,12 @@ package recwatch
 import (
 	"log"
 	"sync"
-
-	"github.com/purpleidea/mgmt/global"
 )
 
 // ConfigWatcher returns events on a channel anytime one of its files events.
 type ConfigWatcher struct {
+	Flags Flags
+
 	ch        chan string
 	wg        sync.WaitGroup
 	closechan chan struct{}
@@ -56,18 +56,26 @@ func (obj *ConfigWatcher) Add(file ...string) {
 	obj.wg.Add(1)
 	go func() {
 		defer obj.wg.Done()
-		ch := ConfigWatch(file[0])
+		ch := obj.ConfigWatch(file[0])
 		for {
 			select {
-			case e := <-ch:
+			case e, ok := <-ch:
+				if !ok { // channel closed
+					return
+				}
 				if e != nil {
 					obj.errorchan <- e
 					return
 				}
-				obj.ch <- file[0]
+				select {
+				case obj.ch <- file[0]: // send on channel
+				case <-obj.closechan:
+					return // never mind, close early!
+				}
 				continue
-			case <-obj.closechan:
-				return
+				// not needed, closes via ConfigWatch() chan close
+				//case <-obj.closechan:
+				//	return
 			}
 		}
 	}()
@@ -100,7 +108,7 @@ func (obj *ConfigWatcher) Close() {
 }
 
 // ConfigWatch writes on the channel every time an event is seen for the path.
-func ConfigWatch(file string) chan error {
+func (obj *ConfigWatcher) ConfigWatch(file string) chan error {
 	ch := make(chan error)
 	go func() {
 		recWatcher, err := NewRecWatcher(file, false)
@@ -109,9 +117,10 @@ func ConfigWatch(file string) chan error {
 			close(ch)
 			return
 		}
+		recWatcher.Flags = obj.Flags
 		defer recWatcher.Close()
 		for {
-			if global.DEBUG {
+			if obj.Flags.Debug {
 				log.Printf("Watching: %v", file)
 			}
 			select {
@@ -125,7 +134,12 @@ func ConfigWatch(file string) chan error {
 					close(ch)
 					return
 				}
-				ch <- nil // send event!
+				select {
+				case ch <- nil: // send event!
+				case <-obj.closechan:
+					close(ch)
+					return
+				}
 			}
 		}
 		//close(ch)

recwatch/flags.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -15,12 +15,9 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package main
+package recwatch
 
-import (
-//"testing"
-)
-
-//func TestT1(t *testing.T) {
-
-//}
+// Flags contains all the constant flags that recwatch needs.
+type Flags struct {
+	Debug bool
+}

recwatch/recwatch.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -29,7 +29,6 @@ import (
 	"sync"
 	"syscall"
 
-	"github.com/purpleidea/mgmt/global" // XXX: package mgmtmain instead?
 	"github.com/purpleidea/mgmt/util"
 
 	"gopkg.in/fsnotify.v1"
@@ -46,15 +45,16 @@ type Event struct {
 type RecWatcher struct {
 	Path     string // computed path
 	Recurse  bool   // should we watch recursively?
+	Flags    Flags
 	isDir    bool   // computed isDir
 	safename string // safe path
 	watcher  *fsnotify.Watcher
 	watches  map[string]struct{}
 	events   chan Event // one channel for events and err...
-	once     sync.Once
+	closed   bool       // is the events channel closed?
+	mutex    sync.Mutex // lock guarding the channel closing
 	wg       sync.WaitGroup
 	exit     chan struct{}
-	closeErr error
 }
 
 // NewRecWatcher creates an initializes a new recursive watcher.
@@ -87,45 +87,48 @@ func (obj *RecWatcher) Init() error {
 		}
 	}
 
+	obj.wg.Add(1)
 	go func() {
+		defer obj.wg.Done()
 		if err := obj.Watch(); err != nil {
-			obj.events <- Event{Error: err}
+			// we need this mutex, because if we Init and then Close
+			// immediately, this can send after closed which panics!
+			obj.mutex.Lock()
+			if !obj.closed {
+				select {
+				case obj.events <- Event{Error: err}:
+				case <-obj.exit:
+					// pass
+				}
+			}
+			obj.mutex.Unlock()
 		}
-		obj.Close()
 	}()
 	return nil
 }
 
-//func (obj *RecWatcher) Add(path string) error { // XXX implement me or not?
+//func (obj *RecWatcher) Add(path string) error { // XXX: implement me or not?
 //
 //}
 //
-//func (obj *RecWatcher) Remove(path string) error { // XXX implement me or not?
+//func (obj *RecWatcher) Remove(path string) error { // XXX: implement me or not?
 //
 //}
 
 // Close shuts down the watcher.
 func (obj *RecWatcher) Close() error {
-	obj.once.Do(obj.close) // don't cause the channel to close twice
-	return obj.closeErr
-}
-
-// This close function is the function that actually does the close work. Don't
-// call it more than once!
-func (obj *RecWatcher) close() {
 	var err error
 	close(obj.exit) // send exit signal
 	obj.wg.Wait()
 	if obj.watcher != nil {
 		err = obj.watcher.Close()
 		obj.watcher = nil
-		// TODO: should we send the close error?
-		//if err != nil {
-		//	obj.events <- Event{Error: err}
-		//}
 	}
+	obj.mutex.Lock() // FIXME: I don't think this mutex is needed anymore...
+	obj.closed = true
 	close(obj.events)
-	obj.closeErr = err // set the error
+	obj.mutex.Unlock()
+	return err
 }
 
 // Events returns a channel of events. These include events for errors.
@@ -134,10 +137,8 @@ func (obj *RecWatcher) Events() chan Event { return obj.events }
 // Watch is the primary listener for this resource and it outputs events.
 func (obj *RecWatcher) Watch() error {
 	if obj.watcher == nil {
-		return fmt.Errorf("Watcher is not initialized!")
+		return fmt.Errorf("the watcher is not initialized")
 	}
-	obj.wg.Add(1)
-	defer obj.wg.Done()
 
 	patharray := util.PathSplit(obj.safename) // tokenize the path
 	var index = len(patharray)                // starting index
@@ -150,12 +151,12 @@ func (obj *RecWatcher) Watch() error {
 		if current == "" { // the empty string top is the root dir ("/")
 			current = "/"
 		}
-		if global.DEBUG {
+		if obj.Flags.Debug {
 			log.Printf("Watching: %s", current) // attempting to watch...
 		}
 		// initialize in the loop so that we can reset on rm-ed handles
 		if err := obj.watcher.Add(current); err != nil {
-			if global.DEBUG {
+			if obj.Flags.Debug {
 				log.Printf("watcher.Add(%s): Error: %v", current, err)
 			}
 
@@ -169,16 +170,16 @@ func (obj *RecWatcher) Watch() error {
 				// no space left on device, out of inotify watches
 				// TODO: consider letting the user fall back to
 				// polling if they hit this error very often...
-				return fmt.Errorf("Out of inotify watches: %v", err)
+				return fmt.Errorf("out of inotify watches: %v", err)
 			} else if os.IsPermission(err) {
-				return fmt.Errorf("Permission denied adding a watch: %v", err)
+				return fmt.Errorf("permission denied adding a watch: %v", err)
 			}
-			return fmt.Errorf("Unknown error: %v", err)
+			return fmt.Errorf("unknown error: %v", err)
 		}
 
 		select {
 		case event := <-obj.watcher.Events:
-			if global.DEBUG {
+			if obj.Flags.Debug {
 				log.Printf("Watch(%s), Event(%s): %v", current, event.Name, event.Op)
 			}
 			// the deeper you go, the bigger the deltaDepth is...
@@ -209,7 +210,7 @@ func (obj *RecWatcher) Watch() error {
 				}
 
 			} else {
-				// TODO different watchers get each others events!
+				// TODO: different watchers get each others events!
 				// https://github.com/go-fsnotify/fsnotify/issues/95
 				// this happened with two values such as:
 				// event.Name: /tmp/mgmt/f3 and current: /tmp/mgmt/f2
@@ -236,6 +237,13 @@ func (obj *RecWatcher) Watch() error {
 					index--
 				}
 
+				// when the file is moved, remove the watcher and add a new one,
+				// so we stop tracking the old inode.
+				if deltaDepth >= 0 && (event.Op&fsnotify.Rename == fsnotify.Rename) {
+					obj.watcher.Remove(current)
+					obj.watcher.Add(current)
+				}
+
 				// we must be a parent watcher, so descend in
 				if deltaDepth < 0 {
 					// XXX: we can block here due to: https://github.com/fsnotify/fsnotify/issues/123
@@ -272,11 +280,16 @@ func (obj *RecWatcher) Watch() error {
 			if send {
 				send = false
 				// only invalid state on certain types of events
-				obj.events <- Event{Error: nil, Body: &event}
+				select {
+				// exit even when we're blocked on event sending
+				case obj.events <- Event{Error: nil, Body: &event}:
+				case <-obj.exit:
+					return fmt.Errorf("pending event not sent")
+				}
 			}
 
 		case err := <-obj.watcher.Errors:
-			return fmt.Errorf("Unknown watcher error: %v", err)
+			return fmt.Errorf("unknown watcher error: %v", err)
 
 		case <-obj.exit:
 			return nil
@@ -291,7 +304,7 @@ func (obj *RecWatcher) addSubFolders(p string) error {
 	}
 	// look at all subfolders...
 	walkFn := func(path string, info os.FileInfo, err error) error {
-		if global.DEBUG {
+		if obj.Flags.Debug {
 			log.Printf("Walk: %s (%v): %v", path, info, err)
 		}
 		if err != nil {

remote/remote.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -62,12 +62,14 @@ import (
 	"time"
 
 	cv "github.com/purpleidea/mgmt/converger"
-	"github.com/purpleidea/mgmt/gconfig"
-	"github.com/purpleidea/mgmt/global"
 	"github.com/purpleidea/mgmt/util"
+	"github.com/purpleidea/mgmt/util/semaphore"
+	"github.com/purpleidea/mgmt/yamlgraph"
 
+	multierr "github.com/hashicorp/go-multierror"
 	"github.com/howeyc/gopass"
 	"github.com/kardianos/osext"
+	errwrap "github.com/pkg/errors"
 	"github.com/pkg/sftp"
 	"golang.org/x/crypto/ssh"
 )
@@ -83,6 +85,12 @@ const (
 	nonInteractivePasswordTimeout        = 5 * 2                                  // five minutes
 )
 
+// Flags are constants required by the remote lib.
+type Flags struct {
+	Program string
+	Debug   bool
+}
+
 // The SSH struct is the unit building block for a single remote SSH connection.
 type SSH struct {
 	hostname string // uuid of the host, as used by the --hostname argument
@@ -114,7 +122,7 @@ type SSH struct {
 	lock    sync.Mutex     // mutex to avoid exit races
 	exiting bool           // flag to let us know if we're exiting
 
-	program  string // name of the binary
+	flags    Flags  // constant runtime values
 	remotewd string // path to remote working directory
 	execpath string // path to remote mgmt binary
 	filepath string // path to remote file config
@@ -149,11 +157,11 @@ func (obj *SSH) Sftp() error {
 	var err error
 
 	if obj.client == nil {
-		return fmt.Errorf("Not dialed!")
+		return fmt.Errorf("not dialed")
 	}
 	// this check is needed because the golang path.Base function is weird!
 	if strings.HasSuffix(obj.file, "/") {
-		return fmt.Errorf("File must not be a directory.")
+		return fmt.Errorf("file must not be a directory")
 	}
 
 	// we run local operations first so that remote clean up is easier...
@@ -171,7 +179,7 @@ func (obj *SSH) Sftp() error {
 
 	// TODO: make the path configurable to deal with /tmp/ mounted noexec?
 	tmpdir := func() string {
-		return fmt.Sprintf(formatPattern, fmtUUID(10)) // eg: /tmp/mgmt.abcdefghij/
+		return fmt.Sprintf(formatPattern, fmtUID(10)) // eg: /tmp/mgmt.abcdefghij/
 	}
 	var ready bool
 	obj.remotewd = ""
@@ -189,7 +197,7 @@ func (obj *SSH) Sftp() error {
 	}
 
 	for i := 0; true; {
-		// NOTE: since fmtUUID is deterministic, if we don't clean up
+		// NOTE: since fmtUID is deterministic, if we don't clean up
 		// previous runs, we may get the same paths generated, and here
 		// they will conflict.
 		if err := obj.sftp.Mkdir(obj.remotewd); err != nil {
@@ -222,7 +230,7 @@ func (obj *SSH) Sftp() error {
 		break
 	}
 
-	obj.execpath = path.Join(obj.remotewd, obj.program) // program is a compile time string
+	obj.execpath = path.Join(obj.remotewd, obj.flags.Program) // program is a compile time string
 	log.Printf("Remote: Remote path is: %s", obj.execpath)
 
 	var same bool
@@ -247,7 +255,7 @@ func (obj *SSH) Sftp() error {
 	// make file executable; don't cache this in case it didn't ever happen
 	// TODO: do we want the group or other bits set?
 	if err := obj.sftp.Chmod(obj.execpath, 0770); err != nil {
-		return fmt.Errorf("Can't set file mode bits!")
+		return fmt.Errorf("can't set file mode bits")
 	}
 
 	// copy graph file
@@ -266,7 +274,7 @@ func (obj *SSH) Sftp() error {
 // SftpGraphCopy is a helper function used for re-copying the graph definition.
 func (obj *SSH) SftpGraphCopy() (int64, error) {
 	if obj.filepath == "" {
-		return -1, fmt.Errorf("Sftp session isn't ready yet!")
+		return -1, fmt.Errorf("sftp session isn't ready yet")
 	}
 	return obj.SftpCopy(obj.file, obj.filepath)
 }
@@ -274,7 +282,7 @@ func (obj *SSH) SftpGraphCopy() (int64, error) {
 // SftpCopy is a simple helper function that runs a local -> remote sftp copy.
 func (obj *SSH) SftpCopy(src, dst string) (int64, error) {
 	if obj.sftp == nil {
-		return -1, fmt.Errorf("Sftp session is not active!")
+		return -1, fmt.Errorf("sftp session is not active")
 	}
 	var err error
 	// TODO: add a check to make sure we don't run two copies of this
@@ -306,7 +314,7 @@ func (obj *SSH) SftpCopy(src, dst string) (int64, error) {
 		return n, fmt.Errorf("Can't copy to remote path: %v", err)
 	}
 	if n <= 0 {
-		return n, fmt.Errorf("Zero bytes copied!")
+		return n, fmt.Errorf("zero bytes copied")
 	}
 	return n, nil
 }
@@ -384,10 +392,10 @@ func (obj *SSH) Tunnel() error {
 	var err error
 
 	if len(obj.clientURLs) < 1 {
-		return fmt.Errorf("Need at least one client URL to tunnel!")
+		return fmt.Errorf("need at least one client URL to tunnel")
 	}
 	if len(obj.remoteURLs) < 1 {
-		return fmt.Errorf("Need at least one remote URL to tunnel!")
+		return fmt.Errorf("need at least one remote URL to tunnel")
 	}
 
 	// TODO: do something less arbitrary about which one we pick?
@@ -446,7 +454,7 @@ func (obj *SSH) forward(remoteConn net.Conn) net.Conn {
 			log.Printf("Remote: io.Copy error: %s", err)
 			// FIXME: what should we do here???
 		}
-		if global.DEBUG {
+		if obj.flags.Debug {
 			log.Printf("Remote: io.Copy finished: %d", n)
 		}
 	}
@@ -470,10 +478,10 @@ func (obj *SSH) TunnelClose() error {
 // Exec runs the binary on the remote server.
 func (obj *SSH) Exec() error {
 	if obj.execpath == "" {
-		return fmt.Errorf("Must have a binary path to execute!")
+		return fmt.Errorf("must have a binary path to execute")
 	}
 	if obj.filepath == "" {
-		return fmt.Errorf("Must have a graph definition to run!")
+		return fmt.Errorf("must have a graph definition to run")
 	}
 
 	var err error
@@ -491,7 +499,7 @@ func (obj *SSH) Exec() error {
 	// TODO: do something less arbitrary about which one we pick?
 	url := cleanURL(obj.remoteURLs[0])                           // arbitrarily pick the first one
 	seeds := fmt.Sprintf("--no-server --seeds 'http://%s'", url) // XXX: escape untrusted input? (or check if url is valid)
-	file := fmt.Sprintf("--file '%s'", obj.filepath)             // XXX: escape untrusted input! (or check if file path exists)
+	file := fmt.Sprintf("--yaml '%s'", obj.filepath)             // XXX: escape untrusted input! (or check if file path exists)
 	depth := fmt.Sprintf("--depth %d", obj.depth+1)              // child is +1 distance
 	args := []string{hostname, seeds, file, depth}
 	if obj.noop {
@@ -561,7 +569,7 @@ func (obj *SSH) ExecExit() error {
 	}
 
 	// FIXME: workaround: force a signal!
-	if _, err := obj.simpleRun(fmt.Sprintf("killall -SIGINT %s", obj.program)); err != nil { // FIXME: low specificity
+	if _, err := obj.simpleRun(fmt.Sprintf("killall -SIGINT %s", obj.flags.Program)); err != nil { // FIXME: low specificity
 		log.Printf("Remote: Failed to send SIGINT: %s", err.Error())
 	}
 
@@ -570,12 +578,12 @@ func (obj *SSH) ExecExit() error {
 		// try killing the process more violently
 		time.Sleep(10 * time.Second)
 		//obj.session.Signal(ssh.SIGKILL)
-		cmd := fmt.Sprintf("killall -SIGKILL %s", obj.program) // FIXME: low specificity
+		cmd := fmt.Sprintf("killall -SIGKILL %s", obj.flags.Program) // FIXME: low specificity
 		obj.simpleRun(cmd)
 	}()
 
 	// FIXME: workaround: wait (spin lock) until process quits cleanly...
-	cmd := fmt.Sprintf("while killall -0 %s 2> /dev/null; do sleep 1s; done", obj.program) // FIXME: low specificity
+	cmd := fmt.Sprintf("while killall -0 %s 2> /dev/null; do sleep 1s; done", obj.flags.Program) // FIXME: low specificity
 	if _, err := obj.simpleRun(cmd); err != nil {
 		return fmt.Errorf("Error waiting: %s", err)
 	}
@@ -691,22 +699,22 @@ type Remotes struct {
 	converger    cv.Converger
 	convergerCb  func(func(map[string]bool) error) (func(), error)
 
-	wg                 sync.WaitGroup              // keep track of each running SSH connection
-	lock               sync.Mutex                  // mutex for access to sshmap
-	sshmap             map[string]*SSH             // map to each SSH struct with the remote as the key
-	exiting            bool                        // flag to let us know if we're exiting
-	exitChan           chan struct{}               // closes when we should exit
-	semaphore          Semaphore                   // counting semaphore to limit concurrent connections
-	hostnames          []string                    // list of hostnames we've seen so far
-	cuuid              cv.ConvergerUUID            // convergerUUID for the remote itself
-	cuuids             map[string]cv.ConvergerUUID // map to each SSH struct with the remote as the key
-	callbackCancelFunc func()                      // stored callback function cancel function
+	wg                 sync.WaitGroup       // keep track of each running SSH connection
+	lock               sync.Mutex           // mutex for access to sshmap
+	sshmap             map[string]*SSH      // map to each SSH struct with the remote as the key
+	exiting            bool                 // flag to let us know if we're exiting
+	exitChan           chan struct{}        // closes when we should exit
+	semaphore          *semaphore.Semaphore // counting semaphore to limit concurrent connections
+	hostnames          []string             // list of hostnames we've seen so far
+	cuid               cv.UID               // convergerUID for the remote itself
+	cuids              map[string]cv.UID    // map to each SSH struct with the remote as the key
+	callbackCancelFunc func()               // stored callback function cancel function
 
-	program string // name of the program
+	flags Flags // constant runtime values
 }
 
 // NewRemotes builds a Remotes struct.
-func NewRemotes(clientURLs, remoteURLs []string, noop bool, remotes []string, fileWatch chan string, cConns uint16, interactive bool, sshPrivIdRsa string, caching bool, depth uint16, prefix string, converger cv.Converger, convergerCb func(func(map[string]bool) error) (func(), error), program string) *Remotes {
+func NewRemotes(clientURLs, remoteURLs []string, noop bool, remotes []string, fileWatch chan string, cConns uint16, interactive bool, sshPrivIdRsa string, caching bool, depth uint16, prefix string, converger cv.Converger, convergerCb func(func(map[string]bool) error) (func(), error), flags Flags) *Remotes {
 	return &Remotes{
 		clientURLs:   clientURLs,
 		remoteURLs:   remoteURLs,
@@ -723,10 +731,10 @@ func NewRemotes(clientURLs, remoteURLs []string, noop bool, remotes []string, fi
 		convergerCb:  convergerCb,
 		sshmap:       make(map[string]*SSH),
 		exitChan:     make(chan struct{}),
-		semaphore:    NewSemaphore(int(cConns)),
+		semaphore:    semaphore.NewSemaphore(int(cConns)),
 		hostnames:    make([]string, len(remotes)),
-		cuuids:       make(map[string]cv.ConvergerUUID),
-		program:      program,
+		cuids:        make(map[string]cv.UID),
+		flags:        flags,
 	}
 }
 
@@ -734,7 +742,7 @@ func NewRemotes(clientURLs, remoteURLs []string, noop bool, remotes []string, fi
 // It takes as input the path to a graph definition file.
 func (obj *Remotes) NewSSH(file string) (*SSH, error) {
 	// first do the parsing...
-	config := gconfig.ParseConfigFromFile(file)
+	config := yamlgraph.ParseConfigFromFile(file) // FIXME: GAPI-ify somehow?
 	if config == nil {
 		return nil, fmt.Errorf("Remote: Error parsing remote graph: %s", file)
 	}
@@ -765,7 +773,7 @@ func (obj *Remotes) NewSSH(file string) (*SSH, error) {
 	}
 	host = x[0]
 	if host == "" {
-		return nil, fmt.Errorf("Empty hostname!")
+		return nil, fmt.Errorf("empty hostname")
 	}
 
 	user := defaultUser // default
@@ -788,15 +796,16 @@ func (obj *Remotes) NewSSH(file string) (*SSH, error) {
 	}
 
 	if len(auth) == 0 {
-		return nil, fmt.Errorf("No authentication methods available!")
+		return nil, fmt.Errorf("no authentication methods available")
 	}
 
-	hostname := config.Hostname
+	//hostname := config.Hostname // TODO: optionally specify local hostname somehow
+	hostname := ""
 	if hostname == "" {
 		hostname = host // default to above
 	}
 	if util.StrInList(hostname, obj.hostnames) {
-		return nil, fmt.Errorf("Remote: Hostname `%s` already exists!", hostname)
+		return nil, fmt.Errorf("Remote: Hostname `%s` already exists", hostname)
 	}
 	obj.hostnames = append(obj.hostnames, hostname)
 
@@ -815,14 +824,14 @@ func (obj *Remotes) NewSSH(file string) (*SSH, error) {
 		caching:    obj.caching,
 		converger:  obj.converger,
 		prefix:     obj.prefix,
-		program:    obj.program,
+		flags:      obj.flags,
 	}, nil
 }
 
 // sshKeyAuth is a helper function to get the ssh key auth struct needed
 func (obj *Remotes) sshKeyAuth() (ssh.AuthMethod, error) {
 	if obj.sshPrivIdRsa == "" {
-		return nil, fmt.Errorf("Empty path specified!")
+		return nil, fmt.Errorf("empty path specified")
 	}
 	p := ""
 	// TODO: this doesn't match strings of the form: ~james/.ssh/id_rsa
@@ -835,7 +844,7 @@ func (obj *Remotes) sshKeyAuth() (ssh.AuthMethod, error) {
 		p = path.Join(usr.HomeDir, obj.sshPrivIdRsa[len("~/"):])
 	}
 	if p == "" {
-		return nil, fmt.Errorf("Empty path specified!")
+		return nil, fmt.Errorf("empty path specified")
 	}
 	// A public key may be used to authenticate against the server by using
 	// an unencrypted PEM-encoded private key file. If you have an encrypted
@@ -884,7 +893,7 @@ func (obj *Remotes) passwordCallback(user, host string) func() (string, error) {
 		case e := <-failchan:
 			return "", e
 		case <-util.TimeAfterOrBlock(timeout):
-			return "", fmt.Errorf("Interactive timeout reached!")
+			return "", fmt.Errorf("interactive timeout reached")
 		}
 	}
 	return cb
@@ -894,12 +903,12 @@ func (obj *Remotes) passwordCallback(user, host string) func() (string, error) {
 func (obj *Remotes) Run() {
 	// TODO: we can disable a lot of this if we're not using --converged-timeout
 	// link in all the converged timeout checking and callbacks...
-	obj.cuuid = obj.converger.Register() // one for me!
-	obj.cuuid.SetName("Remote: Run")
+	obj.cuid = obj.converger.Register() // one for me!
+	obj.cuid.SetName("Remote: Run")
 	for _, f := range obj.remotes { // one for each remote...
-		obj.cuuids[f] = obj.converger.Register() // save a reference
-		obj.cuuids[f].SetName(fmt.Sprintf("Remote: %s", f))
-		//obj.cuuids[f].SetConverged(false) // everyone starts off false
+		obj.cuids[f] = obj.converger.Register() // save a reference
+		obj.cuids[f].SetName(fmt.Sprintf("Remote: %s", f))
+		//obj.cuids[f].SetConverged(false) // everyone starts off false
 	}
 
 	// watch for converged state in the group of remotes...
@@ -921,12 +930,12 @@ func (obj *Remotes) Run() {
 			if !ok { // no status on hostname means unconverged!
 				continue
 			}
-			if global.DEBUG {
+			if obj.flags.Debug {
 				log.Printf("Remote: Converged: Status: %+v", obj.converger.Status())
 			}
 			// if exiting, don't update, it will be unregistered...
 			if !sshobj.exiting { // this is actually racy, but safe
-				obj.cuuids[f].SetConverged(b) // ignore errors!
+				obj.cuids[f].SetConverged(b) // ignore errors!
 			}
 		}
 
@@ -953,10 +962,10 @@ func (obj *Remotes) Run() {
 					if !more {
 						return
 					}
-					obj.cuuid.SetConverged(false) // activity!
+					obj.cuid.SetConverged(false) // activity!
 
-				case <-obj.cuuid.ConvergedTimer():
-					obj.cuuid.SetConverged(true) // converged!
+				case <-obj.cuid.ConvergedTimer():
+					obj.cuid.SetConverged(true) // converged!
 					continue
 				}
 				obj.lock.Lock()
@@ -975,7 +984,7 @@ func (obj *Remotes) Run() {
 			}
 		}()
 	} else {
-		obj.cuuid.SetConverged(true) // if no watches, we're converged!
+		obj.cuid.SetConverged(true) // if no watches, we're converged!
 	}
 
 	// the semaphore provides the max simultaneous connection limit
@@ -993,7 +1002,7 @@ func (obj *Remotes) Run() {
 			if obj.cConns != 0 {
 				obj.semaphore.V(1) // don't lock the loop
 			}
-			obj.cuuids[f].Unregister() // don't stall the converge!
+			obj.cuids[f].Unregister() // don't stall the converge!
 			continue
 		}
 		obj.sshmap[f] = sshobj // save a reference
@@ -1004,7 +1013,7 @@ func (obj *Remotes) Run() {
 				defer obj.semaphore.V(1)
 			}
 			defer obj.wg.Done()
-			defer obj.cuuids[f].Unregister()
+			defer obj.cuids[f].Unregister()
 
 			if err := sshobj.Go(); err != nil {
 				log.Printf("Remote: Error: %s", err)
@@ -1017,11 +1026,12 @@ func (obj *Remotes) Run() {
 
 // Exit causes as much of the Remotes struct to shutdown as quickly and as
 // cleanly as possible. It only returns once everything is shutdown.
-func (obj *Remotes) Exit() {
+func (obj *Remotes) Exit() error {
 	obj.lock.Lock()
 	obj.exiting = true // don't spawn new ones once this flag is set!
 	obj.lock.Unlock()
 	close(obj.exitChan)
+	var reterr error
 	for _, f := range obj.remotes {
 		sshobj, exists := obj.sshmap[f]
 		if !exists || sshobj == nil {
@@ -1030,7 +1040,8 @@ func (obj *Remotes) Exit() {
 
 		// TODO: should we run these as go routines?
 		if err := sshobj.Stop(); err != nil {
-			log.Printf("Remote: Error stopping: %s", err)
+			err = errwrap.Wrapf(err, "Remote: Error stopping!")
+			reterr = multierr.Append(reterr, err) // list of errors
 		}
 	}
 
@@ -1038,14 +1049,15 @@ func (obj *Remotes) Exit() {
 		obj.callbackCancelFunc() // cancel our callback
 	}
 
-	defer obj.cuuid.Unregister()
+	defer obj.cuid.Unregister()
 	obj.wg.Wait() // wait for everyone to exit
+	return reterr
 }
 
-// fmtUUID makes a random string of length n, it is not cryptographically safe.
+// fmtUID makes a random string of length n, it is not cryptographically safe.
 // This function actually usually generates the same sequence of random strings
 // each time the program is run, which makes repeatability of this code easier.
-func fmtUUID(n int) string {
+func fmtUID(n int) string {
 	b := make([]byte, n)
 	for i := range b {
 		b[i] = formatChars[rand.Intn(len(formatChars))]
@@ -1067,29 +1079,6 @@ func cleanURL(s string) string {
 	return u.Host
 }
 
-// Semaphore is a counting semaphore.
-type Semaphore chan struct{}
-
-// NewSemaphore creates a new semaphore.
-func NewSemaphore(size int) Semaphore {
-	return make(Semaphore, size)
-}
-
-// P acquires n resources.
-func (s Semaphore) P(n int) {
-	e := struct{}{}
-	for i := 0; i < n; i++ {
-		s <- e // acquire one
-	}
-}
-
-// V releases n resources.
-func (s Semaphore) V(n int) {
-	for i := 0; i < n; i++ {
-		<-s // release one
-	}
-}
-
 // combinedWriter mimics what the ssh.CombinedOutput command does.
 type combinedWriter struct {
 	b  bytes.Buffer

resources/actions.go

@@ -0,0 +1,649 @@
+// Mgmt
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"fmt"
+	"log"
+	"math"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/purpleidea/mgmt/event"
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/prometheus"
+	"github.com/purpleidea/mgmt/util"
+
+	multierr "github.com/hashicorp/go-multierror"
+	errwrap "github.com/pkg/errors"
+	"golang.org/x/time/rate"
+)
+
+// SentinelErr is a sentinal as an error type that wraps an arbitrary error.
+type SentinelErr struct {
+	err error
+}
+
+// Error is the required method to fulfill the error type.
+func (obj *SentinelErr) Error() string {
+	return obj.err.Error()
+}
+
+// OKTimestamp returns true if this element can run right now?
+func (obj *BaseRes) OKTimestamp() bool {
+	// these are all the vertices pointing TO v, eg: ??? -> v
+	for _, n := range obj.Graph.IncomingGraphVertices(obj.Vertex) {
+		// if the vertex has a greater timestamp than any pre-req (n)
+		// then we can't run right now...
+		// if they're equal (eg: on init of 0) then we also can't run
+		// b/c we should let our pre-req's go first...
+		x, y := obj.Timestamp(), VtoR(n).Timestamp()
+		if b, ok := obj.Graph.Value("debug"); ok && util.Bool(b) {
+			log.Printf("%s: OKTimestamp: (%v) >= %s(%v): !%v", obj, x, n, y, x >= y)
+		}
+		if x >= y {
+			return false
+		}
+	}
+	return true
+}
+
+// Poke tells nodes after me in the dependency graph that they need to refresh.
+func (obj *BaseRes) Poke() error {
+	// if we're pausing (or exiting) then we should suspend poke's so that
+	// the graph doesn't go on running forever until it's completely done!
+	// this is an optional feature which we can do by default on user exit
+	if obj.Graph.FastPause {
+		return nil // TODO: should this be an error instead?
+	}
+
+	var wg sync.WaitGroup
+	// these are all the vertices pointing AWAY FROM v, eg: v -> ???
+	for _, n := range obj.Graph.OutgoingGraphVertices(obj.Vertex) {
+		// we can skip this poke if resource hasn't done work yet... it
+		// needs to be poked if already running, or not running though!
+		// TODO: does this need an || activity flag?
+		if VtoR(n).GetState() != ResStateProcess {
+			if b, ok := obj.Graph.Value("debug"); ok && util.Bool(b) {
+				log.Printf("%s: Poke: %s", obj, n)
+			}
+			wg.Add(1)
+			go func(nn pgraph.Vertex) error {
+				defer wg.Done()
+				//edge := obj.Graph.adjacency[v][nn] // lookup
+				//notify := edge.Notify && edge.Refresh()
+				return VtoR(nn).SendEvent(event.EventPoke, nil)
+			}(n)
+
+		} else {
+			if b, ok := obj.Graph.Value("debug"); ok && util.Bool(b) {
+				log.Printf("%s: Poke: %s: Skipped!", obj, n)
+			}
+		}
+	}
+	// TODO: do something with return values?
+	wg.Wait() // wait for all the pokes to complete
+	return nil
+}
+
+// BackPoke pokes the pre-requisites that are stale and need to run before I can run.
+func (obj *BaseRes) BackPoke() {
+	var wg sync.WaitGroup
+	// these are all the vertices pointing TO v, eg: ??? -> v
+	for _, n := range obj.Graph.IncomingGraphVertices(obj.Vertex) {
+		x, y, s := obj.Timestamp(), VtoR(n).Timestamp(), VtoR(n).GetState()
+		// If the parent timestamp needs poking AND it's not running
+		// Process, then poke it. If the parent is in ResStateProcess it
+		// means that an event is pending, so we'll be expecting a poke
+		// back soon, so we can safely discard the extra parent poke...
+		// TODO: implement a stateLT (less than) to tell if something
+		// happens earlier in the state cycle and that doesn't wrap nil
+		if x >= y && (s != ResStateProcess && s != ResStateCheckApply) {
+			if b, ok := obj.Graph.Value("debug"); ok && util.Bool(b) {
+				log.Printf("%s: BackPoke: %s", obj, n)
+			}
+			wg.Add(1)
+			go func(nn pgraph.Vertex) error {
+				defer wg.Done()
+				return VtoR(nn).SendEvent(event.EventBackPoke, nil)
+			}(n)
+
+		} else {
+			if b, ok := obj.Graph.Value("debug"); ok && util.Bool(b) {
+				log.Printf("%s: BackPoke: %s: Skipped!", obj, n)
+			}
+		}
+	}
+	// TODO: do something with return values?
+	wg.Wait() // wait for all the pokes to complete
+}
+
+// RefreshPending determines if any previous nodes have a refresh pending here.
+// If this is true, it means I am expected to apply a refresh when I next run.
+func (obj *BaseRes) RefreshPending() bool {
+	var refresh bool
+	for _, edge := range obj.Graph.IncomingGraphEdges(obj.Vertex) {
+		// if we asked for a notify *and* if one is pending!
+		edge := edge.(*Edge) // panic if wrong
+		if edge.Notify && edge.Refresh() {
+			refresh = true
+			break
+		}
+	}
+	return refresh
+}
+
+// SetUpstreamRefresh sets the refresh value to any upstream vertices.
+func (obj *BaseRes) SetUpstreamRefresh(b bool) {
+	for _, edge := range obj.Graph.IncomingGraphEdges(obj.Vertex) {
+		edge := edge.(*Edge) // panic if wrong
+		if edge.Notify {
+			edge.SetRefresh(b)
+		}
+	}
+}
+
+// SetDownstreamRefresh sets the refresh value to any downstream vertices.
+func (obj *BaseRes) SetDownstreamRefresh(b bool) {
+	for _, edge := range obj.Graph.OutgoingGraphEdges(obj.Vertex) {
+		edge := edge.(*Edge) // panic if wrong
+		// if we asked for a notify *and* if one is pending!
+		if edge.Notify {
+			edge.SetRefresh(b)
+		}
+	}
+}
+
+// Process is the primary function to execute for a particular vertex in the graph.
+func (obj *BaseRes) Process() error {
+	if obj.debug {
+		log.Printf("%s: Process()", obj)
+	}
+	// FIXME: should these SetState methods be here or after the sema code?
+	defer obj.SetState(ResStateNil) // reset state when finished
+	obj.SetState(ResStateProcess)
+
+	// is it okay to run dependency wise right now?
+	// if not, that's okay because when the dependency runs, it will poke
+	// us back and we will run if needed then!
+	if !obj.OKTimestamp() {
+		go obj.BackPoke()
+		return nil
+	}
+	// timestamp must be okay...
+	if b, ok := obj.Graph.Value("debug"); ok && util.Bool(b) {
+		log.Printf("%s: OKTimestamp(%v)", obj, obj.Timestamp())
+	}
+
+	// semaphores!
+	// These shouldn't ever block an exit, since the graph should eventually
+	// converge causing their them to unlock. More interestingly, since they
+	// run in a DAG alphabetically, there is no way to permanently deadlock,
+	// assuming that resources individually don't ever block from finishing!
+	// The exception is that semaphores with a zero count will always block!
+	// TODO: Add a close mechanism to close/unblock zero count semaphores...
+	semas := obj.Meta().Sema
+	if obj.debug && len(semas) > 0 {
+		log.Printf("%s: Sema: P(%s)", obj, strings.Join(semas, ", "))
+	}
+	if err := obj.Graph.SemaLock(semas); err != nil { // lock
+		// NOTE: in practice, this might not ever be truly necessary...
+		return fmt.Errorf("shutdown of semaphores")
+	}
+	defer obj.Graph.SemaUnlock(semas) // unlock
+	if obj.debug && len(semas) > 0 {
+		defer log.Printf("%s: Sema: V(%s)", obj, strings.Join(semas, ", "))
+	}
+
+	var ok = true
+	var applied = false // did we run an apply?
+
+	// connect any senders to receivers and detect if values changed
+	if updated, err := obj.SendRecv(obj.Res); err != nil {
+		return errwrap.Wrapf(err, "could not SendRecv in Process")
+	} else if len(updated) > 0 {
+		for _, changed := range updated {
+			if changed { // at least one was updated
+				obj.StateOK(false) // invalidate cache, mark as dirty
+				break
+			}
+		}
+	}
+
+	var noop = obj.Meta().Noop // lookup the noop value
+	var refresh bool
+	var checkOK bool
+	var err error
+
+	if b, ok := obj.Graph.Value("debug"); ok && util.Bool(b) {
+		log.Printf("%s: CheckApply(%t)", obj, !noop)
+	}
+
+	// lookup the refresh (notification) variable
+	refresh = obj.RefreshPending() // do i need to perform a refresh?
+	obj.SetRefresh(refresh)        // tell the resource
+
+	// changes can occur after this...
+	obj.SetState(ResStateCheckApply)
+
+	// check cached state, to skip CheckApply; can't skip if refreshing
+	if !refresh && obj.IsStateOK() {
+		checkOK, err = true, nil
+
+		// NOTE: technically this block is wrong because we don't know
+		// if the resource implements refresh! If it doesn't, we could
+		// skip this, but it doesn't make a big difference under noop!
+	} else if noop && refresh { // had a refresh to do w/ noop!
+		checkOK, err = false, nil // therefore the state is wrong
+
+		// run the CheckApply!
+	} else {
+		// if this fails, don't UpdateTimestamp()
+		checkOK, err = obj.Res.CheckApply(!noop)
+
+		if promErr := obj.Data().Prometheus.UpdateCheckApplyTotal(obj.GetKind(), !noop, !checkOK, err != nil); promErr != nil {
+			// TODO: how to error correctly
+			log.Printf("%s: Prometheus.UpdateCheckApplyTotal() errored: %v", obj, err)
+		}
+		// TODO: Can the `Poll` converged timeout tracking be a
+		// more general method for all converged timeouts? this
+		// would simplify the resources by removing boilerplate
+		if obj.Meta().Poll > 0 {
+			if !checkOK { // something changed, restart timer
+				obj.cuid.ResetTimer() // activity!
+				if b, ok := obj.Graph.Value("debug"); ok && util.Bool(b) {
+					log.Printf("%s: Converger: ResetTimer", obj)
+				}
+			}
+		}
+	}
+
+	if checkOK && err != nil { // should never return this way
+		log.Fatalf("%s: CheckApply(): %t, %+v", obj, checkOK, err)
+	}
+	if b, ok := obj.Graph.Value("debug"); ok && util.Bool(b) {
+		log.Printf("%s: CheckApply(): %t, %v", obj, checkOK, err)
+	}
+
+	// if CheckApply ran without noop and without error, state should be good
+	if !noop && err == nil { // aka !noop || checkOK
+		obj.StateOK(true) // reset
+		if refresh {
+			obj.SetUpstreamRefresh(false) // refresh happened, clear the request
+			obj.SetRefresh(false)
+		}
+	}
+
+	if !checkOK { // if state *was* not ok, we had to have apply'ed
+		if err != nil { // error during check or apply
+			ok = false
+		} else {
+			applied = true
+		}
+	}
+
+	// when noop is true we always want to update timestamp
+	if noop && err == nil {
+		ok = true
+	}
+
+	if ok {
+		// did we actually do work?
+		activity := applied
+		if noop {
+			activity = false // no we didn't do work...
+		}
+
+		if activity { // add refresh flag to downstream edges...
+			obj.SetDownstreamRefresh(true)
+		}
+
+		// update this timestamp *before* we poke or the poked
+		// nodes might fail due to having a too old timestamp!
+		obj.UpdateTimestamp()        // this was touched...
+		obj.SetState(ResStatePoking) // can't cancel parent poke
+		if err := obj.Poke(); err != nil {
+			return errwrap.Wrapf(err, "the Poke() failed")
+		}
+	}
+	// poke at our pre-req's instead since they need to refresh/run...
+	return errwrap.Wrapf(err, "could not Process() successfully")
+}
+
+// innerWorker is the CheckApply runner that reads from processChan.
+func (obj *BaseRes) innerWorker() {
+	running := false
+	done := make(chan struct{})
+	playback := false // do we need to run another one?
+
+	waiting := false
+	var timer = time.NewTimer(time.Duration(math.MaxInt64)) // longest duration
+	if !timer.Stop() {
+		<-timer.C // unnecessary, shouldn't happen
+	}
+
+	var delay = time.Duration(obj.Meta().Delay) * time.Millisecond
+	var retry = obj.Meta().Retry // number of tries left, -1 for infinite
+	var limiter = rate.NewLimiter(obj.Meta().Limit, obj.Meta().Burst)
+	limited := false
+
+	wg := &sync.WaitGroup{} // wait for Process routine to exit
+
+Loop:
+	for {
+		select {
+		case ev, ok := <-obj.processChan: // must use like this
+			if !ok { // processChan closed, let's exit
+				break Loop // no event, so no ack!
+			}
+			if obj.Meta().Poll == 0 { // skip for polling
+				obj.wcuid.SetConverged(false)
+			}
+
+			// if process started, but no action yet, skip!
+			if obj.GetState() == ResStateProcess {
+				if b, ok := obj.Graph.Value("debug"); ok && util.Bool(b) {
+					log.Printf("%s: Skipped event!", obj)
+				}
+				ev.ACK() // ready for next message
+				obj.quiesceGroup.Done()
+				continue
+			}
+
+			// if running, we skip running a new execution!
+			// if waiting, we skip running a new execution!
+			if running || waiting {
+				if b, ok := obj.Graph.Value("debug"); ok && util.Bool(b) {
+					log.Printf("%s: Playback added!", obj)
+				}
+				playback = true
+				ev.ACK() // ready for next message
+				obj.quiesceGroup.Done()
+				continue
+			}
+
+			// catch invalid rates
+			if obj.Meta().Burst == 0 && !(obj.Meta().Limit == rate.Inf) { // blocked
+				e := fmt.Errorf("%s: Permanently limited (rate != Inf, burst: 0)", obj)
+				ev.ACK() // ready for next message
+				obj.quiesceGroup.Done()
+				obj.SendEvent(event.EventExit, &SentinelErr{e})
+				continue
+			}
+
+			// rate limit
+			// FIXME: consider skipping rate limit check if
+			// the event is a poke instead of a watch event
+			if !limited && !(obj.Meta().Limit == rate.Inf) { // skip over the playback event...
+				now := time.Now()
+				r := limiter.ReserveN(now, 1) // one event
+				// r.OK() seems to always be true here!
+				d := r.DelayFrom(now)
+				if d > 0 { // delay
+					limited = true
+					playback = true
+					log.Printf("%s: Limited (rate: %v/sec, burst: %d, next: %v)", obj, obj.Meta().Limit, obj.Meta().Burst, d)
+					// start the timer...
+					timer.Reset(d)
+					waiting = true // waiting for retry timer
+					ev.ACK()
+					obj.quiesceGroup.Done()
+					continue
+				} // otherwise, we run directly!
+			}
+			limited = false // let one through
+
+			wg.Add(1)
+			running = true
+			go func(ev *event.Event) {
+				obj.pcuid.SetConverged(false) // "block" Process
+				defer wg.Done()
+				if e := obj.Process(); e != nil {
+					playback = true
+					log.Printf("%s: CheckApply errored: %v", obj, e)
+					if retry == 0 {
+						if err := obj.Data().Prometheus.UpdateState(obj.String(), obj.GetKind(), prometheus.ResStateHardFail); err != nil {
+							// TODO: how to error this?
+							log.Printf("%s: Prometheus.UpdateState() errored: %v", obj, err)
+						}
+
+						// wrap the error in the sentinel
+						obj.quiesceGroup.Done() // before the Wait that happens in SendEvent!
+						obj.SendEvent(event.EventExit, &SentinelErr{e})
+						return
+					}
+					if retry > 0 { // don't decrement the -1
+						retry--
+					}
+					if err := obj.Data().Prometheus.UpdateState(obj.String(), obj.GetKind(), prometheus.ResStateSoftFail); err != nil {
+						// TODO: how to error this?
+						log.Printf("%s: Prometheus.UpdateState() errored: %v", obj, err)
+					}
+					log.Printf("%s: CheckApply: Retrying after %.4f seconds (%d left)", obj, delay.Seconds(), retry)
+					// start the timer...
+					timer.Reset(delay)
+					waiting = true // waiting for retry timer
+					// don't obj.quiesceGroup.Done() b/c
+					// the timer is running and it can exit!
+					return
+				}
+				retry = obj.Meta().Retry // reset on success
+				close(done)              // trigger
+			}(ev)
+			ev.ACK() // sync (now mostly useless)
+
+		case <-timer.C:
+			if obj.Meta().Poll == 0 { // skip for polling
+				obj.wcuid.SetConverged(false)
+			}
+			waiting = false
+			if !timer.Stop() {
+				//<-timer.C // blocks, docs are wrong!
+			}
+			log.Printf("%s: CheckApply delay expired!", obj)
+			close(done)
+
+		// a CheckApply run (with possibly retry pause) finished
+		case <-done:
+			if obj.Meta().Poll == 0 { // skip for polling
+				obj.wcuid.SetConverged(false)
+			}
+			if b, ok := obj.Graph.Value("debug"); ok && util.Bool(b) {
+				log.Printf("%s: CheckApply finished!", obj)
+			}
+			done = make(chan struct{}) // reset
+			// re-send this event, to trigger a CheckApply()
+			if playback {
+				// this lock avoids us sending to
+				// channel after we've closed it!
+				// TODO: can this experience indefinite postponement ?
+				// see: https://github.com/golang/go/issues/11506
+				// pause or exit is in process if not quiescing!
+				if !obj.quiescing {
+					playback = false
+					obj.quiesceGroup.Add(1) // lock around it, b/c still running...
+					go func() {
+						obj.Event() // replay a new event
+						obj.quiesceGroup.Done()
+					}()
+				}
+			}
+			running = false
+			obj.pcuid.SetConverged(true) // "unblock" Process
+			obj.quiesceGroup.Done()
+
+		case <-obj.wcuid.ConvergedTimer():
+			obj.wcuid.SetConverged(true) // converged!
+			continue
+		}
+	}
+	wg.Wait()
+	return
+}
+
+// Worker is the common run frontend of the vertex. It handles all of the retry
+// and retry delay common code, and ultimately returns the final status of this
+// vertex execution.
+func (obj *BaseRes) Worker() error {
+	// listen for chan events from Watch() and run
+	// the Process() function when they're received
+	// this avoids us having to pass the data into
+	// the Watch() function about which graph it is
+	// running on, which isolates things nicely...
+	if obj.debug {
+		log.Printf("%s: Worker: Running", obj)
+		defer log.Printf("%s: Worker: Stopped", obj)
+	}
+	// run the init (should match 1-1 with Close function)
+	if err := obj.Res.Init(); err != nil {
+		obj.ProcessExit()
+		// always exit the worker function by finishing with Close()
+		if e := obj.Res.Close(); e != nil {
+			err = multierr.Append(err, e) // list of errors
+		}
+		return errwrap.Wrapf(err, "could not Init() resource")
+	}
+
+	// if the CheckApply run takes longer than the converged
+	// timeout, we could inappropriately converge mid-apply!
+	// avoid this by blocking convergence with a fake report
+	// we also add a similar blocker around the worker loop!
+	// XXX: put these in Init() ?
+	// get extra cuids (worker, process)
+	obj.wcuid.SetConverged(true) // starts off false, and waits for loop timeout
+	obj.pcuid.SetConverged(true) // starts off true, because it's not running...
+
+	obj.processSync.Add(1)
+	go func() {
+		defer obj.processSync.Done()
+		obj.innerWorker()
+	}()
+
+	var err error // propagate the error up (this is a permanent BAD error!)
+	// the watch delay runs inside of the Watch resource loop, so that it
+	// can still process signals and exit if needed. It shouldn't run any
+	// resource specific code since this is supposed to be a retry delay.
+	// NOTE: we're using the same retry and delay metaparams that CheckApply
+	// uses. This is for practicality. We can separate them later if needed!
+	var watchDelay time.Duration
+	var watchRetry = obj.Meta().Retry // number of tries left, -1 for infinite
+	// watch blocks until it ends, & errors to retry
+	for {
+		// TODO: do we have to stop the converged-timeout when in this block (perhaps we're in the delay block!)
+		// TODO: should we setup/manage some of the converged timeout stuff in here anyways?
+
+		// if a retry-delay was requested, wait, but don't block our events!
+		if watchDelay > 0 {
+			//var pendingSendEvent bool
+			timer := time.NewTimer(watchDelay)
+		Loop:
+			for {
+				select {
+				case <-timer.C: // the wait is over
+					break Loop // critical
+
+				// TODO: resources could have a separate exit channel to avoid this complexity!?
+				case event := <-obj.Events():
+					// NOTE: this code should match the similar Res code!
+					//cuid.SetConverged(false) // TODO: ?
+					if exit, send := obj.ReadEvent(event); exit != nil {
+						obj.ProcessExit()
+						err := *exit // exit err
+						if e := obj.Res.Close(); err == nil {
+							err = e
+						} else if e != nil {
+							err = multierr.Append(err, e) // list of errors
+						}
+						return err // exit
+					} else if send {
+						// if we dive down this rabbit hole, our
+						// timer.C won't get seen until we get out!
+						// in this situation, the Watch() is blocked
+						// from performing until CheckApply returns
+						// successfully, or errors out. This isn't
+						// so bad, but we should document it. Is it
+						// possible that some resource *needs* Watch
+						// to run to be able to execute a CheckApply?
+						// That situation shouldn't be common, and
+						// should probably not be allowed. Can we
+						// avoid it though?
+						//if exit, err := doSend(); exit || err != nil {
+						//	return err // we exit or bubble up a NACK...
+						//}
+						// Instead of doing the above, we can
+						// add events to a pending list, and
+						// when we finish the delay, we can run
+						// them.
+						//pendingSendEvent = true // all events are identical for now...
+					}
+				}
+			}
+			timer.Stop() // it's nice to cleanup
+			log.Printf("%s: Watch delay expired!", obj)
+			// NOTE: we can avoid the send if running Watch guarantees
+			// one CheckApply event on startup!
+			//if pendingSendEvent { // TODO: should this become a list in the future?
+			//	if err := obj.Event() err != nil {
+			//		return err // we exit or bubble up a NACK...
+			//	}
+			//}
+		}
+
+		// TODO: reset the watch retry count after some amount of success
+		var e error
+		if obj.Meta().Poll > 0 { // poll instead of watching :(
+			obj.cuid.StartTimer()
+			e = obj.Poll()
+			obj.cuid.StopTimer() // clean up nicely
+		} else {
+			e = obj.Res.Watch() // run the watch normally
+		}
+		if e == nil { // exit signal
+			err = nil // clean exit
+			break
+		}
+		if sentinelErr, ok := e.(*SentinelErr); ok { // unwrap the sentinel
+			err = sentinelErr.err
+			break // sentinel means, perma-exit
+		}
+		log.Printf("%s: Watch errored: %v", obj, e)
+		if watchRetry == 0 {
+			err = fmt.Errorf("Permanent watch error: %v", e)
+			break
+		}
+		if watchRetry > 0 { // don't decrement the -1
+			watchRetry--
+		}
+		watchDelay = time.Duration(obj.Meta().Delay) * time.Millisecond
+		log.Printf("%s: Watch: Retrying after %.4f seconds (%d left)", obj, watchDelay.Seconds(), watchRetry)
+		// We need to trigger a CheckApply after Watch restarts, so that
+		// we catch any lost events that happened while down. We do this
+		// by getting the Watch resource to send one event once it's up!
+		//v.SendEvent(eventPoke, false, false)
+	}
+
+	obj.ProcessExit()
+	// close resource and return possible errors if any
+	if e := obj.Res.Close(); err == nil {
+		err = e
+	} else if e != nil {
+		err = multierr.Append(err, e) // list of errors
+	}
+	return err
+}

resources/augeas.go

@@ -0,0 +1,300 @@
+// Mgmt
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// +build !noaugeas
+
+package resources
+
+import (
+	"encoding/gob"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+
+	"github.com/purpleidea/mgmt/recwatch"
+
+	errwrap "github.com/pkg/errors"
+	// FIXME: we vendor go/augeas because master requires augeas 1.6.0
+	// and libaugeas-dev-1.6.0 is not yet available in a PPA.
+	"honnef.co/go/augeas"
+)
+
+const (
+	// NS is a namespace for augeas operations
+	NS = "Xmgmt"
+)
+
+func init() {
+	gob.Register(&AugeasRes{})
+	RegisterResource("augeas", func() Res { return &AugeasRes{} })
+}
+
+// AugeasRes is a resource that enables you to use the augeas resource.
+// Currently only allows you to change simple files (e.g sshd_config).
+type AugeasRes struct {
+	BaseRes `yaml:",inline"`
+
+	// File is the path to the file targeted by this resource.
+	File string `yaml:"file"`
+
+	// Lens is the lens used by this resource. If specified, mgmt
+	// will lower the augeas overhead by only loading that lens.
+	Lens string `yaml:"lens"`
+
+	// Sets is a list of changes that will be applied to the file, in the form of
+	// ["path", "value"]. mgmt will run augeas.Get() before augeas.Set(), to
+	// prevent changing the file when it is not needed.
+	Sets []AugeasSet `yaml:"sets"`
+
+	recWatcher *recwatch.RecWatcher // used to watch the changed files
+}
+
+// AugeasSet represents a key/value pair of settings to be applied.
+type AugeasSet struct {
+	Path  string `yaml:"path"`  // The relative path to the value to be changed.
+	Value string `yaml:"value"` // The value to be set on the given Path.
+}
+
+// Default returns some sensible defaults for this resource.
+func (obj *AugeasRes) Default() Res {
+	return &AugeasRes{
+		BaseRes: BaseRes{
+			MetaParams: DefaultMetaParams, // force a default
+		},
+	}
+}
+
+// Validate if the params passed in are valid data.
+func (obj *AugeasRes) Validate() error {
+	if !strings.HasPrefix(obj.File, "/") {
+		return fmt.Errorf("the File param should start with a slash")
+	}
+	if obj.Lens != "" && !strings.HasSuffix(obj.Lens, ".lns") {
+		return fmt.Errorf("the Lens param should have a .lns suffix")
+	}
+	if (obj.Lens == "") != (obj.File == "") {
+		return fmt.Errorf("the File and Lens params must be specified together")
+	}
+	return obj.BaseRes.Validate()
+}
+
+// Init initiates the resource.
+func (obj *AugeasRes) Init() error {
+	obj.BaseRes.Kind = "augeas"
+	return obj.BaseRes.Init() // call base init, b/c we're overriding
+}
+
+// Watch is the primary listener for this resource and it outputs events.
+// Taken from the File resource.
+// FIXME: DRY - This is taken from the file resource
+func (obj *AugeasRes) Watch() error {
+	var err error
+	obj.recWatcher, err = recwatch.NewRecWatcher(obj.File, false)
+	if err != nil {
+		return err
+	}
+	defer obj.recWatcher.Close()
+
+	// notify engine that we're running
+	if err := obj.Running(); err != nil {
+		return err // bubble up a NACK...
+	}
+
+	var send = false // send event?
+	var exit *error
+
+	for {
+		if obj.debug {
+			log.Printf("%s: Watching: %s", obj, obj.File) // attempting to watch...
+		}
+
+		select {
+		case event, ok := <-obj.recWatcher.Events():
+			if !ok { // channel shutdown
+				return nil
+			}
+			if err := event.Error; err != nil {
+				return errwrap.Wrapf(err, "Unknown %s watcher error", obj)
+			}
+			if obj.debug { // don't access event.Body if event.Error isn't nil
+				log.Printf("%s: Event(%s): %v", obj, event.Body.Name, event.Body.Op)
+			}
+			send = true
+			obj.StateOK(false) // dirty
+
+		case event := <-obj.Events():
+			if exit, send = obj.ReadEvent(event); exit != nil {
+				return *exit // exit
+			}
+			//obj.StateOK(false) // dirty // these events don't invalidate state
+		}
+
+		// do all our event sending all together to avoid duplicate msgs
+		if send {
+			send = false
+			obj.Event()
+		}
+	}
+}
+
+// checkApplySet runs CheckApply for one element of the AugeasRes.Set
+func (obj *AugeasRes) checkApplySet(apply bool, ag *augeas.Augeas, set AugeasSet) (bool, error) {
+	fullpath := fmt.Sprintf("/files/%v/%v", obj.File, set.Path)
+
+	// We do not check for errors because errors are also thrown when
+	// the path does not exist.
+	if getValue, _ := ag.Get(fullpath); set.Value == getValue {
+		// The value is what we expect, return directly
+		return true, nil
+	}
+
+	if !apply {
+		// If noop, we can return here directly. We return with
+		// nil even if err is not nil because it does not mean
+		// there is an error.
+		return false, nil
+	}
+
+	if err := ag.Set(fullpath, set.Value); err != nil {
+		return false, errwrap.Wrapf(err, "augeas: error while setting value")
+	}
+
+	return false, nil
+}
+
+// CheckApply method for Augeas resource.
+func (obj *AugeasRes) CheckApply(apply bool) (bool, error) {
+	log.Printf("%s: CheckApply: %s", obj, obj.File)
+	// By default we do not set any option to augeas, we use the defaults.
+	opts := augeas.None
+	if obj.Lens != "" {
+		// if the lens is specified, we can speed up augeas by not
+		// loading everything. Without this option, augeas will try to
+		// read all the files it knows in the complete filesystem.
+		// e.g. to change /etc/ssh/sshd_config, it would load /etc/hosts, /etc/ntpd.conf, etc...
+		opts = augeas.NoModlAutoload
+	}
+
+	// Initiate augeas
+	ag, err := augeas.New("/", "", opts)
+	if err != nil {
+		return false, errwrap.Wrapf(err, "augeas: error while initializing")
+	}
+	defer ag.Close()
+
+	if obj.Lens != "" {
+		// If the lens is set, load the lens for the file we want to edit.
+		// We pick Xmgmt, as this name will not collide with any other lens name.
+		// We do not pick Mgmt as in the future there might be an Mgmt lens.
+		// https://github.com/hercules-team/augeas/wiki/Loading-specific-files
+		if err = ag.Set(fmt.Sprintf("/augeas/load/%s/lens", NS), obj.Lens); err != nil {
+			return false, errwrap.Wrapf(err, "augeas: error while initializing lens")
+		}
+		if err = ag.Set(fmt.Sprintf("/augeas/load/%s/incl", NS), obj.File); err != nil {
+			return false, errwrap.Wrapf(err, "augeas: error while initializing incl")
+		}
+		if err = ag.Load(); err != nil {
+			return false, errwrap.Wrapf(err, "augeas: error while loading")
+		}
+	}
+
+	checkOK := true
+	for _, set := range obj.Sets {
+		if setCheckOK, err := obj.checkApplySet(apply, &ag, set); err != nil {
+			return false, errwrap.Wrapf(err, "augeas: error during CheckApply of one Set")
+		} else if !setCheckOK {
+			checkOK = false
+		}
+	}
+
+	// If the state is correct or we can't apply, return early.
+	if checkOK || !apply {
+		return checkOK, nil
+	}
+
+	log.Printf("%s: changes needed, saving", obj)
+	if err = ag.Save(); err != nil {
+		return false, errwrap.Wrapf(err, "augeas: error while saving augeas values")
+	}
+
+	// FIXME: Workaround for https://github.com/dominikh/go-augeas/issues/13
+	// To be fixed upstream.
+	if obj.File != "" {
+		if _, err := os.Stat(obj.File); os.IsNotExist(err) {
+			return false, errwrap.Wrapf(err, "augeas: error: file does not exist")
+		}
+	}
+
+	return false, nil
+}
+
+// AugeasUID is the UID struct for AugeasRes.
+type AugeasUID struct {
+	BaseUID
+	name string
+}
+
+// UIDs includes all params to make a unique identification of this object.
+func (obj *AugeasRes) UIDs() []ResUID {
+	x := &AugeasUID{
+		BaseUID: BaseUID{Name: obj.GetName(), Kind: obj.GetKind()},
+		name:    obj.Name,
+	}
+	return []ResUID{x}
+}
+
+// GroupCmp returns whether two resources can be grouped together or not.
+func (obj *AugeasRes) GroupCmp(r Res) bool {
+	return false // Augeas commands can not be grouped together.
+}
+
+// Compare two resources and return if they are equivalent.
+func (obj *AugeasRes) Compare(r Res) bool {
+	// we can only compare AugeasRes to others of the same resource kind
+	res, ok := r.(*AugeasRes)
+	if !ok {
+		return false
+	}
+	if !obj.BaseRes.Compare(res) { // call base Compare
+		return false
+	}
+	if obj.Name != res.Name {
+		return false
+	}
+
+	return true
+}
+
+// UnmarshalYAML is the custom unmarshal handler for this struct.
+// It is primarily useful for setting the defaults.
+func (obj *AugeasRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type rawRes AugeasRes // indirection to avoid infinite recursion
+
+	def := obj.Default()        // get the default
+	res, ok := def.(*AugeasRes) // put in the right format
+	if !ok {
+		return fmt.Errorf("could not convert to AugeasRes")
+	}
+	raw := rawRes(*res) // convert; the defaults go here
+
+	if err := unmarshal(&raw); err != nil {
+		return err
+	}
+
+	*obj = AugeasRes(raw) // restore from indirection with type conversion!
+	return nil
+}

resources/augeas_disabled.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -14,13 +14,11 @@
 //
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// +build noaugeas
 
-// Package global holds some global variables that are used throughout the code.
-package global
+package resources
 
-// These constants are used throughout the program.
-const (
-	DEBUG   = false // add additional log messages
-	TRACE   = false // add execution flow log messages
-	VERBOSE = false // add extra log message output
-)
+// AugeasRes represents the fields of the Augeas resource. Since this file is
+// only invoked with the tag "noaugeas", we do not need any fields here.
+type AugeasRes struct {
+}

resources/autoedge.go

@@ -0,0 +1,148 @@
+// Mgmt
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/util"
+
+	multierr "github.com/hashicorp/go-multierror"
+	errwrap "github.com/pkg/errors"
+)
+
+// The AutoEdge interface is used to implement the autoedges feature.
+type AutoEdge interface {
+	Next() []ResUID   // call to get list of edges to add
+	Test([]bool) bool // call until false
+}
+
+// UIDExistsInUIDs wraps the IFF method when used with a list of UID's.
+func UIDExistsInUIDs(uid ResUID, uids []ResUID) bool {
+	for _, u := range uids {
+		if uid.IFF(u) {
+			return true
+		}
+	}
+	return false
+}
+
+// addEdgesByMatchingUIDS adds edges to the vertex in a graph based on if it
+// matches a uid list.
+func addEdgesByMatchingUIDS(g *pgraph.Graph, v pgraph.Vertex, uids []ResUID) []bool {
+	// search for edges and see what matches!
+	var result []bool
+
+	// loop through each uid, and see if it matches any vertex
+	for _, uid := range uids {
+		var found = false
+		// uid is a ResUID object
+		for _, vv := range g.Vertices() { // search
+			if v == vv { // skip self
+				continue
+			}
+			if b, ok := g.Value("debug"); ok && util.Bool(b) {
+				log.Printf("Compile: AutoEdge: Match: %s with UID: %s", vv, uid)
+			}
+			// we must match to an effective UID for the resource,
+			// that is to say, the name value of a res is a helpful
+			// handle, but it is not necessarily a unique identity!
+			// remember, resources can return multiple UID's each!
+			if UIDExistsInUIDs(uid, VtoR(vv).UIDs()) {
+				// add edge from: vv -> v
+				if uid.IsReversed() {
+					txt := fmt.Sprintf("AutoEdge: %s -> %s", vv, v)
+					log.Printf("Compile: Adding %s", txt)
+					edge := &Edge{Name: txt}
+					g.AddEdge(vv, v, edge)
+				} else { // edges go the "normal" way, eg: pkg resource
+					txt := fmt.Sprintf("AutoEdge: %s -> %s", v, vv)
+					log.Printf("Compile: Adding %s", txt)
+					edge := &Edge{Name: txt}
+					g.AddEdge(v, vv, edge)
+				}
+				found = true
+				break
+			}
+		}
+		result = append(result, found)
+	}
+	return result
+}
+
+// AutoEdges adds the automatic edges to the graph.
+func AutoEdges(g *pgraph.Graph) error {
+	log.Println("Compile: Adding AutoEdges...")
+
+	// initially get all of the autoedges to seek out all possible errors
+	var err error
+	autoEdgeObjVertexMap := make(map[pgraph.Vertex]AutoEdge)
+	sorted := g.VerticesSorted()
+
+	for _, v := range sorted { // for each vertexes autoedges
+		if !VtoR(v).Meta().AutoEdge { // is the metaparam true?
+			continue
+		}
+		autoEdgeObj, e := VtoR(v).AutoEdges()
+		if e != nil {
+			err = multierr.Append(err, e) // collect all errors
+			continue
+		}
+		if autoEdgeObj == nil {
+			log.Printf("%s: No auto edges were found!", v)
+			continue // next vertex
+		}
+		autoEdgeObjVertexMap[v] = autoEdgeObj // save for next loop
+	}
+	if err != nil {
+		return errwrap.Wrapf(err, "the auto edges had errors")
+	}
+
+	// now that we're guaranteed error free, we can modify the graph safely
+	for _, v := range sorted { // stable sort order for determinism in logs
+		autoEdgeObj, exists := autoEdgeObjVertexMap[v]
+		if !exists {
+			continue
+		}
+
+		for { // while the autoEdgeObj has more uids to add...
+			uids := autoEdgeObj.Next() // get some!
+			if uids == nil {
+				log.Printf("%s: The auto edge list is empty!", v)
+				break // inner loop
+			}
+			if b, ok := g.Value("debug"); ok && util.Bool(b) {
+				log.Println("Compile: AutoEdge: UIDS:")
+				for i, u := range uids {
+					log.Printf("Compile: AutoEdge: UID%d: %v", i, u)
+				}
+			}
+
+			// match and add edges
+			result := addEdgesByMatchingUIDS(g, v, uids)
+
+			// report back, and find out if we should continue
+			if !autoEdgeObj.Test(result) {
+				break
+			}
+		}
+	}
+	return nil
+}

resources/autogroup.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -15,49 +15,52 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package pgraph
+package resources
 
 import (
 	"fmt"
 	"log"
 
-	"github.com/purpleidea/mgmt/global"
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/util"
+
+	errwrap "github.com/pkg/errors"
 )
 
-// AutoGrouper is the required interface to implement for an autogroup algorithm
+// AutoGrouper is the required interface to implement for an autogroup algorithm.
 type AutoGrouper interface {
 	// listed in the order these are typically called in...
-	name() string                                  // friendly identifier
-	init(*Graph) error                             // only call once
-	vertexNext() (*Vertex, *Vertex, error)         // mostly algorithmic
-	vertexCmp(*Vertex, *Vertex) error              // can we merge these ?
-	vertexMerge(*Vertex, *Vertex) (*Vertex, error) // vertex merge fn to use
-	edgeMerge(*Edge, *Edge) *Edge                  // edge merge fn to use
-	vertexTest(bool) (bool, error)                 // call until false
+	name() string                                                    // friendly identifier
+	init(*pgraph.Graph) error                                        // only call once
+	vertexNext() (pgraph.Vertex, pgraph.Vertex, error)               // mostly algorithmic
+	vertexCmp(pgraph.Vertex, pgraph.Vertex) error                    // can we merge these ?
+	vertexMerge(pgraph.Vertex, pgraph.Vertex) (pgraph.Vertex, error) // vertex merge fn to use
+	edgeMerge(pgraph.Edge, pgraph.Edge) pgraph.Edge                  // edge merge fn to use
+	vertexTest(bool) (bool, error)                                   // call until false
 }
 
-// baseGrouper is the base type for implementing the AutoGrouper interface
+// baseGrouper is the base type for implementing the AutoGrouper interface.
 type baseGrouper struct {
-	graph    *Graph    // store a pointer to the graph
-	vertices []*Vertex // cached list of vertices
+	graph    *pgraph.Graph   // store a pointer to the graph
+	vertices []pgraph.Vertex // cached list of vertices
 	i        int
 	j        int
 	done     bool
 }
 
-// name provides a friendly name for the logs to see
+// name provides a friendly name for the logs to see.
 func (ag *baseGrouper) name() string {
 	return "baseGrouper"
 }
 
 // init is called only once and before using other AutoGrouper interface methods
 // the name method is the only exception: call it any time without side effects!
-func (ag *baseGrouper) init(g *Graph) error {
+func (ag *baseGrouper) init(g *pgraph.Graph) error {
 	if ag.graph != nil {
-		return fmt.Errorf("The init method has already been called!")
+		return fmt.Errorf("the init method has already been called")
 	}
-	ag.graph = g                               // pointer
-	ag.vertices = ag.graph.GetVerticesSorted() // cache in deterministic order!
+	ag.graph = g                            // pointer
+	ag.vertices = ag.graph.VerticesSorted() // cache in deterministic order!
 	ag.i = 0
 	ag.j = 0
 	if len(ag.vertices) == 0 { // empty graph
@@ -71,7 +74,7 @@ func (ag *baseGrouper) init(g *Graph) error {
 // an intelligent algorithm would selectively offer only valid pairs of vertices
 // these should satisfy logical grouping requirements for the autogroup designs!
 // the desired algorithms can override, but keep this method as a base iterator!
-func (ag *baseGrouper) vertexNext() (v1, v2 *Vertex, err error) {
+func (ag *baseGrouper) vertexNext() (v1, v2 pgraph.Vertex, err error) {
 	// this does a for v... { for w... { return v, w }} but stepwise!
 	l := len(ag.vertices)
 	if ag.i < l {
@@ -106,48 +109,49 @@ func (ag *baseGrouper) vertexNext() (v1, v2 *Vertex, err error) {
 	return
 }
 
-func (ag *baseGrouper) vertexCmp(v1, v2 *Vertex) error {
+func (ag *baseGrouper) vertexCmp(v1, v2 pgraph.Vertex) error {
 	if v1 == nil || v2 == nil {
-		return fmt.Errorf("Vertex is nil!")
+		return fmt.Errorf("the vertex is nil")
 	}
 	if v1 == v2 { // skip yourself
-		return fmt.Errorf("Vertices are the same!")
+		return fmt.Errorf("the vertices are the same")
 	}
-	if v1.Kind() != v2.Kind() { // we must group similar kinds
+	if VtoR(v1).GetKind() != VtoR(v2).GetKind() { // we must group similar kinds
 		// TODO: maybe future resources won't need this limitation?
-		return fmt.Errorf("The two resources aren't the same kind!")
+		return fmt.Errorf("the two resources aren't the same kind")
 	}
 	// someone doesn't want to group!
-	if !v1.Meta().AutoGroup || !v2.Meta().AutoGroup {
-		return fmt.Errorf("One of the autogroup flags is false!")
+	if !VtoR(v1).Meta().AutoGroup || !VtoR(v2).Meta().AutoGroup {
+		return fmt.Errorf("one of the autogroup flags is false")
 	}
-	if v1.Res.IsGrouped() { // already grouped!
-		return fmt.Errorf("Already grouped!")
+	if VtoR(v1).IsGrouped() { // already grouped!
+		return fmt.Errorf("already grouped")
 	}
-	if len(v2.Res.GetGroup()) > 0 { // already has children grouped!
-		return fmt.Errorf("Already has groups!")
+	if len(VtoR(v2).GetGroup()) > 0 { // already has children grouped!
+		return fmt.Errorf("already has groups")
 	}
-	if !v1.Res.GroupCmp(v2.Res) { // resource groupcmp failed!
-		return fmt.Errorf("The GroupCmp failed!")
+	if !VtoR(v1).GroupCmp(VtoR(v2)) { // resource groupcmp failed!
+		return fmt.Errorf("the GroupCmp failed")
 	}
 	return nil // success
 }
 
-func (ag *baseGrouper) vertexMerge(v1, v2 *Vertex) (v *Vertex, err error) {
+func (ag *baseGrouper) vertexMerge(v1, v2 pgraph.Vertex) (v pgraph.Vertex, err error) {
 	// NOTE: it's important to use w.Res instead of w, b/c
 	// the w by itself is the *Vertex obj, not the *Res obj
 	// which is contained within it! They both satisfy the
 	// Res interface, which is why both will compile! :(
-	err = v1.Res.GroupRes(v2.Res) // GroupRes skips stupid groupings
-	return                        // success or fail, and no need to merge the actual vertices!
+	err = VtoR(v1).GroupRes(VtoR(v2)) // GroupRes skips stupid groupings
+	return                            // success or fail, and no need to merge the actual vertices!
 }
 
-func (ag *baseGrouper) edgeMerge(e1, e2 *Edge) *Edge {
+func (ag *baseGrouper) edgeMerge(e1, e2 pgraph.Edge) pgraph.Edge {
+	// FIXME: should we merge the edge.Notify or edge.refresh values?
 	return e1 // noop
 }
 
 // vertexTest processes the results of the grouping for the algorithm to know
-// return an error if something went horribly wrong, and bool false to stop
+// return an error if something went horribly wrong, and bool false to stop.
 func (ag *baseGrouper) vertexTest(b bool) (bool, error) {
 	// NOTE: this particular baseGrouper version doesn't track what happens
 	// because since we iterate over every pair, we don't care which merge!
@@ -157,23 +161,24 @@ func (ag *baseGrouper) vertexTest(b bool) (bool, error) {
 	return true, nil
 }
 
+// NonReachabilityGrouper is the most straight-forward algorithm for grouping.
 // TODO: this algorithm may not be correct in all cases. replace if needed!
-type nonReachabilityGrouper struct {
+type NonReachabilityGrouper struct {
 	baseGrouper // "inherit" what we want, and reimplement the rest
 }
 
-func (ag *nonReachabilityGrouper) name() string {
-	return "nonReachabilityGrouper"
+func (ag *NonReachabilityGrouper) name() string {
+	return "NonReachabilityGrouper"
 }
 
-// this algorithm relies on the observation that if there's a path from a to b,
+// This algorithm relies on the observation that if there's a path from a to b,
 // then they *can't* be merged (b/c of the existing dependency) so therefore we
 // merge anything that *doesn't* satisfy this condition or that of the reverse!
-func (ag *nonReachabilityGrouper) vertexNext() (v1, v2 *Vertex, err error) {
+func (ag *NonReachabilityGrouper) vertexNext() (v1, v2 pgraph.Vertex, err error) {
 	for {
 		v1, v2, err = ag.baseGrouper.vertexNext() // get all iterable pairs
 		if err != nil {
-			log.Fatalf("Error running autoGroup(vertexNext): %v", err)
+			log.Fatalf("error running autoGroup(vertexNext): %v", err)
 		}
 
 		if v1 != v2 { // ignore self cmp early (perf optimization)
@@ -187,7 +192,7 @@ func (ag *nonReachabilityGrouper) vertexNext() (v1, v2 *Vertex, err error) {
 
 		// if we got here, it means we're skipping over this candidate!
 		if ok, err := ag.baseGrouper.vertexTest(false); err != nil {
-			log.Fatalf("Error running autoGroup(vertexTest): %v", err)
+			log.Fatalf("error running autoGroup(vertexTest): %v", err)
 		} else if !ok {
 			return nil, nil, nil // done!
 		}
@@ -200,15 +205,15 @@ func (ag *nonReachabilityGrouper) vertexNext() (v1, v2 *Vertex, err error) {
 // and then by deleting v2 from the graph. Since more than one edge between two
 // vertices is not allowed, duplicate edges are merged as well. an edge merge
 // function can be provided if you'd like to control how you merge the edges!
-func (g *Graph) VertexMerge(v1, v2 *Vertex, vertexMergeFn func(*Vertex, *Vertex) (*Vertex, error), edgeMergeFn func(*Edge, *Edge) *Edge) error {
+func VertexMerge(g *pgraph.Graph, v1, v2 pgraph.Vertex, vertexMergeFn func(pgraph.Vertex, pgraph.Vertex) (pgraph.Vertex, error), edgeMergeFn func(pgraph.Edge, pgraph.Edge) pgraph.Edge) error {
 	// methodology
 	// 1) edges between v1 and v2 are removed
 	//Loop:
-	for k1 := range g.Adjacency {
-		for k2 := range g.Adjacency[k1] {
+	for k1 := range g.Adjacency() {
+		for k2 := range g.Adjacency()[k1] {
 			// v1 -> v2 || v2 -> v1
 			if (k1 == v1 && k2 == v2) || (k1 == v2 && k2 == v1) {
-				delete(g.Adjacency[k1], k2) // delete map & edge
+				delete(g.Adjacency()[k1], k2) // delete map & edge
 				// NOTE: if we assume this is a DAG, then we can
 				// assume only v1 -> v2 OR v2 -> v1 exists, and
 				// we can break out of these loops immediately!
@@ -219,11 +224,11 @@ func (g *Graph) VertexMerge(v1, v2 *Vertex, vertexMergeFn func(*Vertex, *Vertex)
 	}
 
 	// 2) edges that point towards v2 from X now point to v1 from X (no dupes)
-	for _, x := range g.IncomingGraphEdges(v2) { // all to vertex v (??? -> v)
-		e := g.Adjacency[x][v2] // previous edge
+	for _, x := range g.IncomingGraphVertices(v2) { // all to vertex v (??? -> v)
+		e := g.Adjacency()[x][v2] // previous edge
 		r := g.Reachability(x, v1)
-		// merge e with ex := g.Adjacency[x][v1] if it exists!
-		if ex, exists := g.Adjacency[x][v1]; exists && edgeMergeFn != nil && len(r) == 0 {
+		// merge e with ex := g.Adjacency()[x][v1] if it exists!
+		if ex, exists := g.Adjacency()[x][v1]; exists && edgeMergeFn != nil && len(r) == 0 {
 			e = edgeMergeFn(e, ex)
 		}
 		if len(r) == 0 { // if not reachable, add it
@@ -236,21 +241,21 @@ func (g *Graph) VertexMerge(v1, v2 *Vertex, vertexMergeFn func(*Vertex, *Vertex)
 					continue
 				}
 				// this edge is from: prev, to: next
-				ex, _ := g.Adjacency[prev][next] // get
+				ex, _ := g.Adjacency()[prev][next] // get
 				ex = edgeMergeFn(ex, e)
-				g.Adjacency[prev][next] = ex // set
+				g.Adjacency()[prev][next] = ex // set
 				prev = next
 			}
 		}
-		delete(g.Adjacency[x], v2) // delete old edge
+		delete(g.Adjacency()[x], v2) // delete old edge
 	}
 
 	// 3) edges that point from v2 to X now point from v1 to X (no dupes)
-	for _, x := range g.OutgoingGraphEdges(v2) { // all from vertex v (v -> ???)
-		e := g.Adjacency[v2][x] // previous edge
+	for _, x := range g.OutgoingGraphVertices(v2) { // all from vertex v (v -> ???)
+		e := g.Adjacency()[v2][x] // previous edge
 		r := g.Reachability(v1, x)
-		// merge e with ex := g.Adjacency[v1][x] if it exists!
-		if ex, exists := g.Adjacency[v1][x]; exists && edgeMergeFn != nil && len(r) == 0 {
+		// merge e with ex := g.Adjacency()[v1][x] if it exists!
+		if ex, exists := g.Adjacency()[v1][x]; exists && edgeMergeFn != nil && len(r) == 0 {
 			e = edgeMergeFn(e, ex)
 		}
 		if len(r) == 0 {
@@ -263,13 +268,13 @@ func (g *Graph) VertexMerge(v1, v2 *Vertex, vertexMergeFn func(*Vertex, *Vertex)
 					continue
 				}
 				// this edge is from: prev, to: next
-				ex, _ := g.Adjacency[prev][next]
+				ex, _ := g.Adjacency()[prev][next]
 				ex = edgeMergeFn(ex, e)
-				g.Adjacency[prev][next] = ex
+				g.Adjacency()[prev][next] = ex
 				prev = next
 			}
 		}
-		delete(g.Adjacency[v2], x)
+		delete(g.Adjacency()[v2], x)
 	}
 
 	// 4) merge and then remove the (now merged/grouped) vertex
@@ -277,32 +282,33 @@ func (g *Graph) VertexMerge(v1, v2 *Vertex, vertexMergeFn func(*Vertex, *Vertex)
 		if v, err := vertexMergeFn(v1, v2); err != nil {
 			return err
 		} else if v != nil { // replace v1 with the "merged" version...
-			v1 = v // XXX: will this replace v1 the way we want?
+			//*v1 = *v // TODO: is this safe? (replacing mutexes is undefined!)
+			v1 = v
 		}
 	}
 	g.DeleteVertex(v2) // remove grouped vertex
 
 	// 5) creation of a cyclic graph should throw an error
-	if _, dag := g.TopologicalSort(); !dag { // am i a dag or not?
-		return fmt.Errorf("Graph is not a dag!")
+	if _, err := g.TopologicalSort(); err != nil { // am i a dag or not?
+		return errwrap.Wrapf(err, "the TopologicalSort failed") // not a dag
 	}
 	return nil // success
 }
 
-// autoGroup is the mechanical auto group "runner" that runs the interface spec
-func (g *Graph) autoGroup(ag AutoGrouper) chan string {
+// autoGroup is the mechanical auto group "runner" that runs the interface spec.
+func autoGroup(g *pgraph.Graph, ag AutoGrouper) chan string {
 	strch := make(chan string) // output log messages here
 	go func(strch chan string) {
 		strch <- fmt.Sprintf("Compile: Grouping: Algorithm: %v...", ag.name())
 		if err := ag.init(g); err != nil {
-			log.Fatalf("Error running autoGroup(init): %v", err)
+			log.Fatalf("error running autoGroup(init): %v", err)
 		}
 
 		for {
-			var v, w *Vertex
+			var v, w pgraph.Vertex
 			v, w, err := ag.vertexNext() // get pair to compare
 			if err != nil {
-				log.Fatalf("Error running autoGroup(vertexNext): %v", err)
+				log.Fatalf("error running autoGroup(vertexNext): %v", err)
 			}
 			merged := false
 			// save names since they change during the runs
@@ -310,12 +316,12 @@ func (g *Graph) autoGroup(ag AutoGrouper) chan string {
 			wStr := fmt.Sprintf("%s", w)
 
 			if err := ag.vertexCmp(v, w); err != nil { // cmp ?
-				if global.DEBUG {
+				if b, ok := g.Value("debug"); ok && util.Bool(b) {
 					strch <- fmt.Sprintf("Compile: Grouping: !GroupCmp for: %s into %s", wStr, vStr)
 				}
 
 				// remove grouped vertex and merge edges (res is safe)
-			} else if err := g.VertexMerge(v, w, ag.vertexMerge, ag.edgeMerge); err != nil { // merge...
+			} else if err := VertexMerge(g, v, w, ag.vertexMerge, ag.edgeMerge); err != nil { // merge...
 				strch <- fmt.Sprintf("Compile: Grouping: !VertexMerge for: %s into %s", wStr, vStr)
 
 			} else { // success!
@@ -325,7 +331,7 @@ func (g *Graph) autoGroup(ag AutoGrouper) chan string {
 
 			// did these get used?
 			if ok, err := ag.vertexTest(merged); err != nil {
-				log.Fatalf("Error running autoGroup(vertexTest): %v", err)
+				log.Fatalf("error running autoGroup(vertexTest): %v", err)
 			} else if !ok {
 				break // done!
 			}
@@ -337,12 +343,12 @@ func (g *Graph) autoGroup(ag AutoGrouper) chan string {
 	return strch
 }
 
-// AutoGroup runs the auto grouping on the graph and prints out log messages
-func (g *Graph) AutoGroup() {
+// AutoGroup runs the auto grouping on the graph and prints out log messages.
+func AutoGroup(g *pgraph.Graph, ag AutoGrouper) {
 	// receive log messages from channel...
 	// this allows test cases to avoid printing them when they're unwanted!
 	// TODO: this algorithm may not be correct in all cases. replace if needed!
-	for str := range g.autoGroup(&nonReachabilityGrouper{}) {
+	for str := range autoGroup(g, ag) {
 		log.Println(str)
 	}
 }

resources/autogroup_test.go

@@ -0,0 +1,732 @@
+// Mgmt
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"fmt"
+	"reflect"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/util"
+)
+
+// NE is a helper function to make testing easier. It creates a new noop edge.
+func NE(s string) pgraph.Edge {
+	obj := &Edge{Name: s}
+	return obj
+}
+
+type testGrouper struct {
+	// TODO: this algorithm may not be correct in all cases. replace if needed!
+	NonReachabilityGrouper // "inherit" what we want, and reimplement the rest
+}
+
+func (ag *testGrouper) name() string {
+	return "testGrouper"
+}
+
+func (ag *testGrouper) vertexMerge(v1, v2 pgraph.Vertex) (v pgraph.Vertex, err error) {
+	if err := VtoR(v1).GroupRes(VtoR(v2)); err != nil { // group them first
+		return nil, err
+	}
+	// HACK: update the name so it matches full list of self+grouped
+	obj := VtoR(v1)
+	names := strings.Split(obj.GetName(), ",") // load in stored names
+	for _, n := range obj.GetGroup() {
+		names = append(names, n.GetName()) // add my contents
+	}
+	names = util.StrRemoveDuplicatesInList(names) // remove duplicates
+	sort.Strings(names)
+	obj.SetName(strings.Join(names, ","))
+	return // success or fail, and no need to merge the actual vertices!
+}
+
+func (ag *testGrouper) edgeMerge(e1, e2 pgraph.Edge) pgraph.Edge {
+	edge1 := e1.(*Edge) // panic if wrong
+	edge2 := e2.(*Edge) // panic if wrong
+	// HACK: update the name so it makes a union of both names
+	n1 := strings.Split(edge1.Name, ",") // load
+	n2 := strings.Split(edge2.Name, ",") // load
+	names := append(n1, n2...)
+	names = util.StrRemoveDuplicatesInList(names) // remove duplicates
+	sort.Strings(names)
+	return &Edge{Name: strings.Join(names, ",")}
+}
+
+// helper function
+func runGraphCmp(t *testing.T, g1, g2 *pgraph.Graph) {
+	AutoGroup(g1, &testGrouper{}) // edits the graph
+	err := GraphCmp(g1, g2)
+	if err != nil {
+		t.Logf("  actual (g1): %v%v", g1, fullPrint(g1))
+		t.Logf("expected (g2): %v%v", g2, fullPrint(g2))
+		t.Logf("Cmp error:")
+		t.Errorf("%v", err)
+	}
+}
+
+type NoopResTest struct {
+	NoopRes
+}
+
+func (obj *NoopResTest) GroupCmp(r Res) bool {
+	res, ok := r.(*NoopResTest)
+	if !ok {
+		return false
+	}
+
+	// TODO: implement this in vertexCmp for *testGrouper instead?
+	if strings.Contains(res.Name, ",") { // HACK
+		return false // element to be grouped is already grouped!
+	}
+
+	// group if they start with the same letter! (helpful hack for testing)
+	return obj.Name[0] == res.Name[0]
+}
+
+func NewNoopResTest(name string) *NoopResTest {
+	obj := &NoopResTest{
+		NoopRes: NoopRes{
+			BaseRes: BaseRes{
+				Name: name,
+				MetaParams: MetaParams{
+					AutoGroup: true, // always autogroup
+				},
+			},
+		},
+	}
+	return obj
+}
+
+// GraphCmp compares the topology of two graphs and returns nil if they're
+// equal. It also compares if grouped element groups are identical.
+// TODO: port this to use the pgraph.GraphCmp function instead.
+func GraphCmp(g1, g2 *pgraph.Graph) error {
+	if n1, n2 := g1.NumVertices(), g2.NumVertices(); n1 != n2 {
+		return fmt.Errorf("graph g1 has %d vertices, while g2 has %d", n1, n2)
+	}
+	if e1, e2 := g1.NumEdges(), g2.NumEdges(); e1 != e2 {
+		return fmt.Errorf("graph g1 has %d edges, while g2 has %d", e1, e2)
+	}
+
+	var m = make(map[pgraph.Vertex]pgraph.Vertex) // g1 to g2 vertex correspondence
+Loop:
+	// check vertices
+	for v1 := range g1.Adjacency() { // for each vertex in g1
+
+		l1 := strings.Split(VtoR(v1).GetName(), ",") // make list of everyone's names...
+		for _, x1 := range VtoR(v1).GetGroup() {
+			l1 = append(l1, x1.GetName()) // add my contents
+		}
+		l1 = util.StrRemoveDuplicatesInList(l1) // remove duplicates
+		sort.Strings(l1)
+
+		// inner loop
+		for v2 := range g2.Adjacency() { // does it match in g2 ?
+
+			l2 := strings.Split(VtoR(v2).GetName(), ",")
+			for _, x2 := range VtoR(v2).GetGroup() {
+				l2 = append(l2, x2.GetName())
+			}
+			l2 = util.StrRemoveDuplicatesInList(l2) // remove duplicates
+			sort.Strings(l2)
+
+			// does l1 match l2 ?
+			if ListStrCmp(l1, l2) { // cmp!
+				m[v1] = v2
+				continue Loop
+			}
+		}
+		return fmt.Errorf("graph g1, has no match in g2 for: %v", VtoR(v1).GetName())
+	}
+	// vertices (and groups) match :)
+
+	// check edges
+	for v1 := range g1.Adjacency() { // for each vertex in g1
+		v2 := m[v1] // lookup in map to get correspondance
+		// g1.Adjacency()[v1] corresponds to g2.Adjacency()[v2]
+		if e1, e2 := len(g1.Adjacency()[v1]), len(g2.Adjacency()[v2]); e1 != e2 {
+			return fmt.Errorf("graph g1, vertex(%v) has %d edges, while g2, vertex(%v) has %d", VtoR(v1).GetName(), e1, VtoR(v2).GetName(), e2)
+		}
+
+		for vv1, ee1 := range g1.Adjacency()[v1] {
+			vv2 := m[vv1]
+			ee1 := ee1.(*Edge)
+			ee2 := g2.Adjacency()[v2][vv2].(*Edge)
+
+			// these are edges from v1 -> vv1 via ee1 (graph 1)
+			// to cmp to edges from v2 -> vv2 via ee2 (graph 2)
+
+			// check: (1) vv1 == vv2 ? (we've already checked this!)
+			l1 := strings.Split(VtoR(vv1).GetName(), ",") // make list of everyone's names...
+			for _, x1 := range VtoR(vv1).GetGroup() {
+				l1 = append(l1, x1.GetName()) // add my contents
+			}
+			l1 = util.StrRemoveDuplicatesInList(l1) // remove duplicates
+			sort.Strings(l1)
+
+			l2 := strings.Split(VtoR(vv2).GetName(), ",")
+			for _, x2 := range VtoR(vv2).GetGroup() {
+				l2 = append(l2, x2.GetName())
+			}
+			l2 = util.StrRemoveDuplicatesInList(l2) // remove duplicates
+			sort.Strings(l2)
+
+			// does l1 match l2 ?
+			if !ListStrCmp(l1, l2) { // cmp!
+				return fmt.Errorf("graph g1 and g2 don't agree on: %v and %v", VtoR(vv1).GetName(), VtoR(vv2).GetName())
+			}
+
+			// check: (2) ee1 == ee2
+			if ee1.Name != ee2.Name {
+				return fmt.Errorf("graph g1 edge(%v) doesn't match g2 edge(%v)", ee1.Name, ee2.Name)
+			}
+		}
+	}
+
+	// check meta parameters
+	for v1 := range g1.Adjacency() { // for each vertex in g1
+		for v2 := range g2.Adjacency() { // does it match in g2 ?
+			s1, s2 := VtoR(v1).Meta().Sema, VtoR(v2).Meta().Sema
+			sort.Strings(s1)
+			sort.Strings(s2)
+			if !reflect.DeepEqual(s1, s2) {
+				return fmt.Errorf("vertex %s and vertex %s have different semaphores", VtoR(v1).GetName(), VtoR(v2).GetName())
+			}
+		}
+	}
+
+	return nil // success!
+}
+
+// ListStrCmp compares two lists of strings
+func ListStrCmp(a, b []string) bool {
+	//fmt.Printf("CMP: %v with %v\n", a, b) // debugging
+	if a == nil && b == nil {
+		return true
+	}
+	if a == nil || b == nil {
+		return false
+	}
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if a[i] != b[i] {
+			return false
+		}
+	}
+	return true
+}
+
+func fullPrint(g *pgraph.Graph) (str string) {
+	str += "\n"
+	for v := range g.Adjacency() {
+		if semas := VtoR(v).Meta().Sema; len(semas) > 0 {
+			str += fmt.Sprintf("* v: %v; sema: %v\n", VtoR(v).GetName(), semas)
+		} else {
+			str += fmt.Sprintf("* v: %v\n", VtoR(v).GetName())
+		}
+		// TODO: add explicit grouping data?
+	}
+	for v1 := range g.Adjacency() {
+		for v2, e := range g.Adjacency()[v1] {
+			edge := e.(*Edge)
+			str += fmt.Sprintf("* e: %v -> %v # %v\n", VtoR(v1).GetName(), VtoR(v2).GetName(), edge.Name)
+		}
+	}
+	return
+}
+
+func TestDurationAssumptions(t *testing.T) {
+	var d time.Duration
+	if (d == 0) != true {
+		t.Errorf("empty time.Duration is no longer equal to zero")
+	}
+	if (d > 0) != false {
+		t.Errorf("empty time.Duration is now greater than zero")
+	}
+}
+
+// all of the following test cases are laid out with the following semantics:
+// * vertices which start with the same single letter are considered "like"
+// * "like" elements should be merged
+// * vertices can have any integer after their single letter "family" type
+// * grouped vertices should have a name with a comma separated list of names
+// * edges follow the same conventions about grouping
+
+// empty graph
+func TestPgraphGrouping1(t *testing.T) {
+	g1, _ := pgraph.NewGraph("g1") // original graph
+	g2, _ := pgraph.NewGraph("g2") // expected result
+	runGraphCmp(t, g1, g2)
+}
+
+// single vertex
+func TestPgraphGrouping2(t *testing.T) {
+	g1, _ := pgraph.NewGraph("g1") // original graph
+	{                              // grouping to limit variable scope
+		a1 := NewNoopResTest("a1")
+		g1.AddVertex(a1)
+	}
+	g2, _ := pgraph.NewGraph("g2") // expected result
+	{
+		a1 := NewNoopResTest("a1")
+		g2.AddVertex(a1)
+	}
+	runGraphCmp(t, g1, g2)
+}
+
+// two vertices
+func TestPgraphGrouping3(t *testing.T) {
+	g1, _ := pgraph.NewGraph("g1") // original graph
+	{
+		a1 := NewNoopResTest("a1")
+		b1 := NewNoopResTest("b1")
+		g1.AddVertex(a1, b1)
+	}
+	g2, _ := pgraph.NewGraph("g2") // expected result
+	{
+		a1 := NewNoopResTest("a1")
+		b1 := NewNoopResTest("b1")
+		g2.AddVertex(a1, b1)
+	}
+	runGraphCmp(t, g1, g2)
+}
+
+// two vertices merge
+func TestPgraphGrouping4(t *testing.T) {
+	g1, _ := pgraph.NewGraph("g1") // original graph
+	{
+		a1 := NewNoopResTest("a1")
+		a2 := NewNoopResTest("a2")
+		g1.AddVertex(a1, a2)
+	}
+	g2, _ := pgraph.NewGraph("g2") // expected result
+	{
+		a := NewNoopResTest("a1,a2")
+		g2.AddVertex(a)
+	}
+	runGraphCmp(t, g1, g2)
+}
+
+// three vertices merge
+func TestPgraphGrouping5(t *testing.T) {
+	g1, _ := pgraph.NewGraph("g1") // original graph
+	{
+		a1 := NewNoopResTest("a1")
+		a2 := NewNoopResTest("a2")
+		a3 := NewNoopResTest("a3")
+		g1.AddVertex(a1, a2, a3)
+	}
+	g2, _ := pgraph.NewGraph("g2") // expected result
+	{
+		a := NewNoopResTest("a1,a2,a3")
+		g2.AddVertex(a)
+	}
+	runGraphCmp(t, g1, g2)
+}
+
+// three vertices, two merge
+func TestPgraphGrouping6(t *testing.T) {
+	g1, _ := pgraph.NewGraph("g1") // original graph
+	{
+		a1 := NewNoopResTest("a1")
+		a2 := NewNoopResTest("a2")
+		b1 := NewNoopResTest("b1")
+		g1.AddVertex(a1, a2, b1)
+	}
+	g2, _ := pgraph.NewGraph("g2") // expected result
+	{
+		a := NewNoopResTest("a1,a2")
+		b1 := NewNoopResTest("b1")
+		g2.AddVertex(a, b1)
+	}
+	runGraphCmp(t, g1, g2)
+}
+
+// four vertices, three merge
+func TestPgraphGrouping7(t *testing.T) {
+	g1, _ := pgraph.NewGraph("g1") // original graph
+	{
+		a1 := NewNoopResTest("a1")
+		a2 := NewNoopResTest("a2")
+		a3 := NewNoopResTest("a3")
+		b1 := NewNoopResTest("b1")
+		g1.AddVertex(a1, a2, a3, b1)
+	}
+	g2, _ := pgraph.NewGraph("g2") // expected result
+	{
+		a := NewNoopResTest("a1,a2,a3")
+		b1 := NewNoopResTest("b1")
+		g2.AddVertex(a, b1)
+	}
+	runGraphCmp(t, g1, g2)
+}
+
+// four vertices, two&two merge
+func TestPgraphGrouping8(t *testing.T) {
+	g1, _ := pgraph.NewGraph("g1") // original graph
+	{
+		a1 := NewNoopResTest("a1")
+		a2 := NewNoopResTest("a2")
+		b1 := NewNoopResTest("b1")
+		b2 := NewNoopResTest("b2")
+		g1.AddVertex(a1, a2, b1, b2)
+	}
+	g2, _ := pgraph.NewGraph("g2") // expected result
+	{
+		a := NewNoopResTest("a1,a2")
+		b := NewNoopResTest("b1,b2")
+		g2.AddVertex(a, b)
+	}
+	runGraphCmp(t, g1, g2)
+}
+
+// five vertices, two&three merge
+func TestPgraphGrouping9(t *testing.T) {
+	g1, _ := pgraph.NewGraph("g1") // original graph
+	{
+		a1 := NewNoopResTest("a1")
+		a2 := NewNoopResTest("a2")
+		b1 := NewNoopResTest("b1")
+		b2 := NewNoopResTest("b2")
+		b3 := NewNoopResTest("b3")
+		g1.AddVertex(a1, a2, b1, b2, b3)
+	}
+	g2, _ := pgraph.NewGraph("g2") // expected result
+	{
+		a := NewNoopResTest("a1,a2")
+		b := NewNoopResTest("b1,b2,b3")
+		g2.AddVertex(a, b)
+	}
+	runGraphCmp(t, g1, g2)
+}
+
+// three unique vertices
+func TestPgraphGrouping10(t *testing.T) {
+	g1, _ := pgraph.NewGraph("g1") // original graph
+	{
+		a1 := NewNoopResTest("a1")
+		b1 := NewNoopResTest("b1")
+		c1 := NewNoopResTest("c1")
+		g1.AddVertex(a1, b1, c1)
+	}
+	g2, _ := pgraph.NewGraph("g2") // expected result
+	{
+		a1 := NewNoopResTest("a1")
+		b1 := NewNoopResTest("b1")
+		c1 := NewNoopResTest("c1")
+		g2.AddVertex(a1, b1, c1)
+	}
+	runGraphCmp(t, g1, g2)
+}
+
+// three unique vertices, two merge
+func TestPgraphGrouping11(t *testing.T) {
+	g1, _ := pgraph.NewGraph("g1") // original graph
+	{
+		a1 := NewNoopResTest("a1")
+		b1 := NewNoopResTest("b1")
+		b2 := NewNoopResTest("b2")
+		c1 := NewNoopResTest("c1")
+		g1.AddVertex(a1, b1, b2, c1)
+	}
+	g2, _ := pgraph.NewGraph("g2") // expected result
+	{
+		a1 := NewNoopResTest("a1")
+		b := NewNoopResTest("b1,b2")
+		c1 := NewNoopResTest("c1")
+		g2.AddVertex(a1, b, c1)
+	}
+	runGraphCmp(t, g1, g2)
+}
+
+// simple merge 1
+// a1   a2         a1,a2
+//   \ /     >>>     |     (arrows point downwards)
+//    b              b
+func TestPgraphGrouping12(t *testing.T) {
+	g1, _ := pgraph.NewGraph("g1") // original graph
+	{
+		a1 := NewNoopResTest("a1")
+		a2 := NewNoopResTest("a2")
+		b1 := NewNoopResTest("b1")
+		e1 := NE("e1")
+		e2 := NE("e2")
+		g1.AddEdge(a1, b1, e1)
+		g1.AddEdge(a2, b1, e2)
+	}
+	g2, _ := pgraph.NewGraph("g2") // expected result
+	{
+		a := NewNoopResTest("a1,a2")
+		b1 := NewNoopResTest("b1")
+		e := NE("e1,e2")
+		g2.AddEdge(a, b1, e)
+	}
+	runGraphCmp(t, g1, g2)
+}
+
+// simple merge 2
+//    b              b
+//   / \     >>>     |     (arrows point downwards)
+// a1   a2         a1,a2
+func TestPgraphGrouping13(t *testing.T) {
+	g1, _ := pgraph.NewGraph("g1") // original graph
+	{
+		a1 := NewNoopResTest("a1")
+		a2 := NewNoopResTest("a2")
+		b1 := NewNoopResTest("b1")
+		e1 := NE("e1")
+		e2 := NE("e2")
+		g1.AddEdge(b1, a1, e1)
+		g1.AddEdge(b1, a2, e2)
+	}
+	g2, _ := pgraph.NewGraph("g2") // expected result
+	{
+		a := NewNoopResTest("a1,a2")
+		b1 := NewNoopResTest("b1")
+		e := NE("e1,e2")
+		g2.AddEdge(b1, a, e)
+	}
+	runGraphCmp(t, g1, g2)
+}
+
+// triple merge
+// a1 a2  a3         a1,a2,a3
+//   \ | /     >>>       |      (arrows point downwards)
+//     b                 b
+func TestPgraphGrouping14(t *testing.T) {
+	g1, _ := pgraph.NewGraph("g1") // original graph
+	{
+		a1 := NewNoopResTest("a1")
+		a2 := NewNoopResTest("a2")
+		a3 := NewNoopResTest("a3")
+		b1 := NewNoopResTest("b1")
+		e1 := NE("e1")
+		e2 := NE("e2")
+		e3 := NE("e3")
+		g1.AddEdge(a1, b1, e1)
+		g1.AddEdge(a2, b1, e2)
+		g1.AddEdge(a3, b1, e3)
+	}
+	g2, _ := pgraph.NewGraph("g2") // expected result
+	{
+		a := NewNoopResTest("a1,a2,a3")
+		b1 := NewNoopResTest("b1")
+		e := NE("e1,e2,e3")
+		g2.AddEdge(a, b1, e)
+	}
+	runGraphCmp(t, g1, g2)
+}
+
+// chain merge
+//    a1             a1
+//   /  \             |
+// b1    b2   >>>   b1,b2   (arrows point downwards)
+//   \  /             |
+//    c1             c1
+func TestPgraphGrouping15(t *testing.T) {
+	g1, _ := pgraph.NewGraph("g1") // original graph
+	{
+		a1 := NewNoopResTest("a1")
+		b1 := NewNoopResTest("b1")
+		b2 := NewNoopResTest("b2")
+		c1 := NewNoopResTest("c1")
+		e1 := NE("e1")
+		e2 := NE("e2")
+		e3 := NE("e3")
+		e4 := NE("e4")
+		g1.AddEdge(a1, b1, e1)
+		g1.AddEdge(a1, b2, e2)
+		g1.AddEdge(b1, c1, e3)
+		g1.AddEdge(b2, c1, e4)
+	}
+	g2, _ := pgraph.NewGraph("g2") // expected result
+	{
+		a1 := NewNoopResTest("a1")
+		b := NewNoopResTest("b1,b2")
+		c1 := NewNoopResTest("c1")
+		e1 := NE("e1,e2")
+		e2 := NE("e3,e4")
+		g2.AddEdge(a1, b, e1)
+		g2.AddEdge(b, c1, e2)
+	}
+	runGraphCmp(t, g1, g2)
+}
+
+// re-attach 1 (outer)
+// technically the second possibility is valid too, depending on which order we
+// merge edges in, and if we don't filter out any unnecessary edges afterwards!
+// a1    a2         a1,a2        a1,a2
+//  |   /             |            |  \
+// b1  /      >>>    b1     OR    b1  /   (arrows point downwards)
+//  | /               |            | /
+// c1                c1           c1
+func TestPgraphGrouping16(t *testing.T) {
+	g1, _ := pgraph.NewGraph("g1") // original graph
+	{
+		a1 := NewNoopResTest("a1")
+		a2 := NewNoopResTest("a2")
+		b1 := NewNoopResTest("b1")
+		c1 := NewNoopResTest("c1")
+		e1 := NE("e1")
+		e2 := NE("e2")
+		e3 := NE("e3")
+		g1.AddEdge(a1, b1, e1)
+		g1.AddEdge(b1, c1, e2)
+		g1.AddEdge(a2, c1, e3)
+	}
+	g2, _ := pgraph.NewGraph("g2") // expected result
+	{
+		a := NewNoopResTest("a1,a2")
+		b1 := NewNoopResTest("b1")
+		c1 := NewNoopResTest("c1")
+		e1 := NE("e1,e3")
+		e2 := NE("e2,e3") // e3 gets "merged through" to BOTH edges!
+		g2.AddEdge(a, b1, e1)
+		g2.AddEdge(b1, c1, e2)
+	}
+	runGraphCmp(t, g1, g2)
+}
+
+// re-attach 2 (inner)
+// a1    b2          a1
+//  |   /             |
+// b1  /      >>>   b1,b2   (arrows point downwards)
+//  | /               |
+// c1                c1
+func TestPgraphGrouping17(t *testing.T) {
+	g1, _ := pgraph.NewGraph("g1") // original graph
+	{
+		a1 := NewNoopResTest("a1")
+		b1 := NewNoopResTest("b1")
+		b2 := NewNoopResTest("b2")
+		c1 := NewNoopResTest("c1")
+		e1 := NE("e1")
+		e2 := NE("e2")
+		e3 := NE("e3")
+		g1.AddEdge(a1, b1, e1)
+		g1.AddEdge(b1, c1, e2)
+		g1.AddEdge(b2, c1, e3)
+	}
+	g2, _ := pgraph.NewGraph("g2") // expected result
+	{
+		a1 := NewNoopResTest("a1")
+		b := NewNoopResTest("b1,b2")
+		c1 := NewNoopResTest("c1")
+		e1 := NE("e1")
+		e2 := NE("e2,e3")
+		g2.AddEdge(a1, b, e1)
+		g2.AddEdge(b, c1, e2)
+	}
+	runGraphCmp(t, g1, g2)
+}
+
+// re-attach 3 (double)
+// similar to "re-attach 1", technically there is a second possibility for this
+// a2   a1    b2         a1,a2
+//   \   |   /             |
+//    \ b1  /      >>>   b1,b2   (arrows point downwards)
+//     \ | /               |
+//      c1                c1
+func TestPgraphGrouping18(t *testing.T) {
+	g1, _ := pgraph.NewGraph("g1") // original graph
+	{
+		a1 := NewNoopResTest("a1")
+		a2 := NewNoopResTest("a2")
+		b1 := NewNoopResTest("b1")
+		b2 := NewNoopResTest("b2")
+		c1 := NewNoopResTest("c1")
+		e1 := NE("e1")
+		e2 := NE("e2")
+		e3 := NE("e3")
+		e4 := NE("e4")
+		g1.AddEdge(a1, b1, e1)
+		g1.AddEdge(b1, c1, e2)
+		g1.AddEdge(a2, c1, e3)
+		g1.AddEdge(b2, c1, e4)
+	}
+	g2, _ := pgraph.NewGraph("g2") // expected result
+	{
+		a := NewNoopResTest("a1,a2")
+		b := NewNoopResTest("b1,b2")
+		c1 := NewNoopResTest("c1")
+		e1 := NE("e1,e3")
+		e2 := NE("e2,e3,e4") // e3 gets "merged through" to BOTH edges!
+		g2.AddEdge(a, b, e1)
+		g2.AddEdge(b, c1, e2)
+	}
+	runGraphCmp(t, g1, g2)
+}
+
+// connected merge 0, (no change!)
+// a1            a1
+//   \     >>>     \     (arrows point downwards)
+//    a2            a2
+func TestPgraphGroupingConnected0(t *testing.T) {
+	g1, _ := pgraph.NewGraph("g1") // original graph
+	{
+		a1 := NewNoopResTest("a1")
+		a2 := NewNoopResTest("a2")
+		e1 := NE("e1")
+		g1.AddEdge(a1, a2, e1)
+	}
+	g2, _ := pgraph.NewGraph("g2") // expected result ?
+	{
+		a1 := NewNoopResTest("a1")
+		a2 := NewNoopResTest("a2")
+		e1 := NE("e1")
+		g2.AddEdge(a1, a2, e1)
+	}
+	runGraphCmp(t, g1, g2)
+}
+
+// connected merge 1, (no change!)
+// a1              a1
+//   \               \
+//    b      >>>      b      (arrows point downwards)
+//     \               \
+//      a2              a2
+func TestPgraphGroupingConnected1(t *testing.T) {
+	g1, _ := pgraph.NewGraph("g1") // original graph
+	{
+		a1 := NewNoopResTest("a1")
+		b := NewNoopResTest("b")
+		a2 := NewNoopResTest("a2")
+		e1 := NE("e1")
+		e2 := NE("e2")
+		g1.AddEdge(a1, b, e1)
+		g1.AddEdge(b, a2, e2)
+	}
+	g2, _ := pgraph.NewGraph("g2") // expected result ?
+	{
+		a1 := NewNoopResTest("a1")
+		b := NewNoopResTest("b")
+		a2 := NewNoopResTest("a2")
+		e1 := NE("e1")
+		e2 := NE("e2")
+		g2.AddEdge(a1, b, e1)
+		g2.AddEdge(b, a2, e2)
+	}
+	runGraphCmp(t, g1, g2)
+}

resources/edge.go

@@ -0,0 +1,56 @@
+// Mgmt
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+// Edge is a struct that represents a graph's edge.
+type Edge struct {
+	Name   string
+	Notify bool // should we send a refresh notification along this edge?
+
+	refresh bool // is there a notify pending for the dest vertex ?
+}
+
+// String is a required method of the Edge interface that we must fulfill.
+func (obj *Edge) String() string {
+	return obj.Name
+}
+
+// Compare returns true if two edges are equivalent. Otherwise it returns false.
+func (obj *Edge) Compare(edge *Edge) bool {
+	if obj.Name != edge.Name {
+		return false
+	}
+	if obj.Notify != edge.Notify {
+		return false
+	}
+	// FIXME: should we compare this as well?
+	//if obj.refresh != edge.refresh {
+	//	return false
+	//}
+	return true
+}
+
+// Refresh returns the pending refresh status of this edge.
+func (obj *Edge) Refresh() bool {
+	return obj.refresh
+}
+
+// SetRefresh sets the pending refresh status of this edge.
+func (obj *Edge) SetRefresh(b bool) {
+	obj.refresh = b
+}

resources/exec.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -21,76 +21,62 @@ import (
 	"bufio"
 	"bytes"
 	"encoding/gob"
-	"errors"
 	"fmt"
 	"log"
 	"os/exec"
 	"strings"
-	"time"
+	"sync"
+	"syscall"
 
-	"github.com/purpleidea/mgmt/event"
 	"github.com/purpleidea/mgmt/util"
+
+	errwrap "github.com/pkg/errors"
 )
 
 func init() {
 	gob.Register(&ExecRes{})
+	RegisterResource("exec", func() Res { return &ExecRes{} })
 }
 
 // ExecRes is an exec resource for running commands.
 type ExecRes struct {
 	BaseRes    `yaml:",inline"`
-	State      string `yaml:"state"`      // state: exists/present?, absent, (undefined?)
-	Cmd        string `yaml:"cmd"`        // the command to run
-	Shell      string `yaml:"shell"`      // the (optional) shell to use to run the cmd
-	Timeout    int    `yaml:"timeout"`    // the cmd timeout in seconds
-	WatchCmd   string `yaml:"watchcmd"`   // the watch command to run
-	WatchShell string `yaml:"watchshell"` // the (optional) shell to use to run the watch cmd
-	IfCmd      string `yaml:"ifcmd"`      // the if command to run
-	IfShell    string `yaml:"ifshell"`    // the (optional) shell to use to run the if cmd
-	PollInt    int    `yaml:"pollint"`    // the poll interval for the ifcmd
+	Cmd        string  `yaml:"cmd"`        // the command to run
+	Shell      string  `yaml:"shell"`      // the (optional) shell to use to run the cmd
+	Timeout    int     `yaml:"timeout"`    // the cmd timeout in seconds
+	WatchCmd   string  `yaml:"watchcmd"`   // the watch command to run
+	WatchShell string  `yaml:"watchshell"` // the (optional) shell to use to run the watch cmd
+	IfCmd      string  `yaml:"ifcmd"`      // the if command to run
+	IfShell    string  `yaml:"ifshell"`    // the (optional) shell to use to run the if cmd
+	Output     *string // all cmd output, read only, do not set!
+	Stdout     *string // the cmd stdout, read only, do not set!
+	Stderr     *string // the cmd stderr, read only, do not set!
 }
 
-// NewExecRes is a constructor for this resource. It also calls Init() for you.
-func NewExecRes(name, cmd, shell string, timeout int, watchcmd, watchshell, ifcmd, ifshell string, pollint int, state string) *ExecRes {
-	obj := &ExecRes{
+// Default returns some sensible defaults for this resource.
+func (obj *ExecRes) Default() Res {
+	return &ExecRes{
 		BaseRes: BaseRes{
-			Name: name,
+			MetaParams: DefaultMetaParams, // force a default
 		},
-		Cmd:        cmd,
-		Shell:      shell,
-		Timeout:    timeout,
-		WatchCmd:   watchcmd,
-		WatchShell: watchshell,
-		IfCmd:      ifcmd,
-		IfShell:    ifshell,
-		PollInt:    pollint,
-		State:      state,
 	}
-	obj.Init()
-	return obj
+}
+
+// Validate if the params passed in are valid data.
+func (obj *ExecRes) Validate() error {
+	if obj.Cmd == "" { // this is the only thing that is really required
+		return fmt.Errorf("command can't be empty")
+	}
+
+	return obj.BaseRes.Validate()
 }
 
 // Init runs some startup code for this resource.
 func (obj *ExecRes) Init() error {
-	obj.BaseRes.kind = "Exec"
+	obj.BaseRes.Kind = "exec"
 	return obj.BaseRes.Init() // call base init, b/c we're overriding
 }
 
-// Validate if the params passed in are valid data.
-// FIXME: where should this get called ?
-func (obj *ExecRes) Validate() error {
-	if obj.Cmd == "" { // this is the only thing that is really required
-		return fmt.Errorf("Command can't be empty!")
-	}
-
-	// if we have a watch command, then we don't poll with the if command!
-	if obj.WatchCmd != "" && obj.PollInt > 0 {
-		return fmt.Errorf("Don't poll when we have a watch command.")
-	}
-
-	return nil
-}
-
 // BufioChanScanner wraps the scanner output in a channel.
 func (obj *ExecRes) BufioChanScanner(scanner *bufio.Scanner) (chan string, chan error) {
 	ch, errch := make(chan string), make(chan error)
@@ -99,7 +85,7 @@ func (obj *ExecRes) BufioChanScanner(scanner *bufio.Scanner) (chan string, chan
 			ch <- scanner.Text() // blocks here ?
 			if e := scanner.Err(); e != nil {
 				errch <- e // send any misc errors we encounter
-				//break // TODO ?
+				//break // TODO: ?
 			}
 		}
 		close(ch)
@@ -110,26 +96,9 @@ func (obj *ExecRes) BufioChanScanner(scanner *bufio.Scanner) (chan string, chan
 }
 
 // Watch is the primary listener for this resource and it outputs events.
-func (obj *ExecRes) Watch(processChan chan event.Event) error {
-	if obj.IsWatching() {
-		return nil
-	}
-	obj.SetWatching(true)
-	defer obj.SetWatching(false)
-	cuuid := obj.converger.Register()
-	defer cuuid.Unregister()
-
-	var startup bool
-	Startup := func(block bool) <-chan time.Time {
-		if block {
-			return nil // blocks forever
-			//return make(chan time.Time) // blocks forever
-		}
-		return time.After(time.Duration(500) * time.Millisecond) // 1/2 the resolution of converged timeout
-	}
-
+func (obj *ExecRes) Watch() error {
 	var send = false // send event?
-	var exit = false
+	var exit *error
 	bufioch, errch := make(chan string), make(chan error)
 
 	if obj.WatchCmd != "" {
@@ -149,71 +118,65 @@ func (obj *ExecRes) Watch(processChan chan event.Event) error {
 		}
 		cmd := exec.Command(cmdName, cmdArgs...)
 		//cmd.Dir = "" // look for program in pwd ?
+		// ignore signals sent to parent process (we're in our own group)
+		cmd.SysProcAttr = &syscall.SysProcAttr{
+			Setpgid: true,
+			Pgid:    0,
+		}
 
 		cmdReader, err := cmd.StdoutPipe()
 		if err != nil {
-			return fmt.Errorf("%s[%s]: Error creating StdoutPipe for Cmd: %v", obj.Kind(), obj.GetName(), err)
+			return errwrap.Wrapf(err, "error creating StdoutPipe for Cmd")
 		}
 		scanner := bufio.NewScanner(cmdReader)
 
-		defer cmd.Wait() // XXX: is this necessary?
+		defer cmd.Wait() // wait for the command to exit before return!
 		defer func() {
 			// FIXME: without wrapping this in this func it panic's
-			// when running examples/graph8d.yaml
-			cmd.Process.Kill() // TODO: is this necessary?
+			// when running certain graphs... why?
+			cmd.Process.Kill() // shutdown the Watch command on exit
 		}()
 		if err := cmd.Start(); err != nil {
-			return fmt.Errorf("%s[%s]: Error starting Cmd: %v", obj.Kind(), obj.GetName(), err)
+			return errwrap.Wrapf(err, "error starting Cmd")
 		}
 
 		bufioch, errch = obj.BufioChanScanner(scanner)
 	}
 
+	// notify engine that we're running
+	if err := obj.Running(); err != nil {
+		return err // bubble up a NACK...
+	}
+
 	for {
-		obj.SetState(ResStateWatching) // reset
 		select {
 		case text := <-bufioch:
-			cuuid.SetConverged(false)
 			// each time we get a line of output, we loop!
-			log.Printf("%v[%v]: Watch output: %s", obj.Kind(), obj.GetName(), text)
+			log.Printf("%s: Watch output: %s", obj, text)
 			if text != "" {
 				send = true
+				obj.StateOK(false) // something made state dirty
 			}
 
 		case err := <-errch:
-			cuuid.SetConverged(false)
 			if err == nil { // EOF
 				// FIXME: add an "if watch command ends/crashes"
 				// restart or generate error option
-				return fmt.Errorf("%s[%s]: Reached EOF", obj.Kind(), obj.GetName())
+				return fmt.Errorf("reached EOF")
 			}
 			// error reading input?
-			return fmt.Errorf("Unknown %s[%s] error: %v", obj.Kind(), obj.GetName(), err)
+			return errwrap.Wrapf(err, "unknown error")
 
-		case event := <-obj.events:
-			cuuid.SetConverged(false)
-			if exit, send = obj.ReadEvent(&event); exit {
-				return nil // exit
+		case event := <-obj.Events():
+			if exit, send = obj.ReadEvent(event); exit != nil {
+				return *exit // exit
 			}
-
-		case <-cuuid.ConvergedTimer():
-			cuuid.SetConverged(true) // converged!
-			continue
-
-		case <-Startup(startup):
-			cuuid.SetConverged(false)
-			send = true
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
-			startup = true // startup finished
 			send = false
-			// it is okay to invalidate the clean state on poke too
-			obj.isStateOK = false // something made state dirty
-			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
-				return err // we exit or bubble up a NACK...
-			}
+			obj.Event()
 		}
 	}
 }
@@ -221,25 +184,12 @@ func (obj *ExecRes) Watch(processChan chan event.Event) error {
 // CheckApply checks the resource state and applies the resource if the bool
 // input is true. It returns error info and if the state check passed or not.
 // TODO: expand the IfCmd to be a list of commands
-func (obj *ExecRes) CheckApply(apply bool) (checkok bool, err error) {
-	log.Printf("%v[%v]: CheckApply(%t)", obj.Kind(), obj.GetName(), apply)
+func (obj *ExecRes) CheckApply(apply bool) (bool, error) {
+	// If we receive a refresh signal, then the engine skips the IsStateOK()
+	// check and this will run. It is still guarded by the IfCmd, but it can
+	// have a chance to execute, and all without the check of obj.Refresh()!
 
-	// if there is a watch command, but no if command, run based on state
-	if obj.WatchCmd != "" && obj.IfCmd == "" {
-		if obj.isStateOK {
-			return true, nil
-		}
-
-		// if there is no watcher, but there is an onlyif check, run it to see
-	} else if obj.IfCmd != "" { // && obj.WatchCmd == ""
-		// there is a watcher, but there is also an if command
-		//} else if obj.IfCmd != "" && obj.WatchCmd != "" {
-
-		if obj.PollInt > 0 { // && obj.WatchCmd == ""
-			// XXX have the Watch() command output onlyif poll events...
-			// XXX we can optimize by saving those results for returning here
-			// return XXX
-		}
+	if obj.IfCmd != "" { // if there is no onlyif check, we should just run
 
 		var cmdName string
 		var cmdArgs []string
@@ -255,18 +205,17 @@ func (obj *ExecRes) CheckApply(apply bool) (checkok bool, err error) {
 			cmdName = obj.IfShell // usually bash, or sh
 			cmdArgs = []string{"-c", obj.IfCmd}
 		}
-		err = exec.Command(cmdName, cmdArgs...).Run()
-		if err != nil {
+		cmd := exec.Command(cmdName, cmdArgs...)
+		// ignore signals sent to parent process (we're in our own group)
+		cmd.SysProcAttr = &syscall.SysProcAttr{
+			Setpgid: true,
+			Pgid:    0,
+		}
+		if err := cmd.Run(); err != nil {
 			// TODO: check exit value
 			return true, nil // don't run
 		}
 
-		// if there is no watcher and no onlyif check, assume we should run
-	} else { // if obj.WatchCmd == "" && obj.IfCmd == "" {
-		// just run if state is dirty
-		if obj.isStateOK {
-			return true, nil
-		}
 	}
 
 	// state is not okay, no work done, exit, but without error
@@ -275,7 +224,7 @@ func (obj *ExecRes) CheckApply(apply bool) (checkok bool, err error) {
 	}
 
 	// apply portion
-	log.Printf("%v[%v]: Apply", obj.Kind(), obj.GetName())
+	log.Printf("%s: Apply", obj)
 	var cmdName string
 	var cmdArgs []string
 	if obj.Shell == "" {
@@ -293,12 +242,21 @@ func (obj *ExecRes) CheckApply(apply bool) (checkok bool, err error) {
 	}
 	cmd := exec.Command(cmdName, cmdArgs...)
 	//cmd.Dir = "" // look for program in pwd ?
-	var out bytes.Buffer
-	cmd.Stdout = &out
+	// ignore signals sent to parent process (we're in our own group)
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setpgid: true,
+		Pgid:    0,
+	}
 
-	if err = cmd.Start(); err != nil {
-		log.Printf("%v[%v]: Error starting Cmd: %v", obj.Kind(), obj.GetName(), err)
-		return false, err
+	var out splitWriter
+	out.Init()
+	// from the docs: "If Stdout and Stderr are the same writer, at most one
+	// goroutine at a time will call Write." so we trick it here!
+	cmd.Stdout = out.Stdout
+	cmd.Stderr = out.Stderr
+
+	if err := cmd.Start(); err != nil {
+		return false, errwrap.Wrapf(err, "error starting cmd")
 	}
 
 	timeout := obj.Timeout
@@ -308,97 +266,91 @@ func (obj *ExecRes) CheckApply(apply bool) (checkok bool, err error) {
 	done := make(chan error)
 	go func() { done <- cmd.Wait() }()
 
+	var err error // error returned by cmd
 	select {
-	case err = <-done:
-		if err != nil {
-			log.Printf("%v[%v]: Error waiting for Cmd: %v", obj.Kind(), obj.GetName(), err)
-			return false, err
-		}
+	case e := <-done:
+		err = e // store
 
 	case <-util.TimeAfterOrBlock(timeout):
-		log.Printf("%v[%v]: Timeout waiting for Cmd", obj.Kind(), obj.GetName())
-		//cmd.Process.Kill() // TODO: is this necessary?
-		return false, errors.New("Timeout waiting for Cmd!")
+		cmd.Process.Kill() // TODO: check error?
+		return false, fmt.Errorf("timeout for cmd")
+	}
+
+	// save in memory for send/recv
+	// we use pointers to strings to indicate if used or not
+	if out.Stdout.Activity || out.Stderr.Activity {
+		str := out.String()
+		obj.Output = &str
+	}
+	if out.Stdout.Activity {
+		str := out.Stdout.String()
+		obj.Stdout = &str
+	}
+	if out.Stderr.Activity {
+		str := out.Stderr.String()
+		obj.Stderr = &str
+	}
+
+	// process the err result from cmd, we process non-zero exits here too!
+	exitErr, ok := err.(*exec.ExitError) // embeds an os.ProcessState
+	if err != nil && ok {
+		pStateSys := exitErr.Sys() // (*os.ProcessState) Sys
+		wStatus, ok := pStateSys.(syscall.WaitStatus)
+		if !ok {
+			e := errwrap.Wrapf(err, "error running cmd")
+			return false, e
+		}
+		return false, fmt.Errorf("cmd error, exit status: %d", wStatus.ExitStatus())
+
+	} else if err != nil {
+		e := errwrap.Wrapf(err, "general cmd error")
+		return false, e
 	}
 
 	// TODO: if we printed the stdout while the command is running, this
 	// would be nice, but it would require terminal log output that doesn't
 	// interleave all the parallel parts which would mix it all up...
 	if s := out.String(); s == "" {
-		log.Printf("Exec[%v]: Command output is empty!", obj.Name)
+		log.Printf("%s: Command output is empty!", obj)
+
 	} else {
-		log.Printf("Exec[%v]: Command output is:", obj.Name)
+		log.Printf("%s: Command output is:", obj)
 		log.Printf(out.String())
 	}
-	// XXX: return based on exit value!!
 
-	// the state tracking is for exec resources that can't "detect" their
+	// The state tracking is for exec resources that can't "detect" their
 	// state, and assume it's invalid when the Watch() function triggers.
-	// if we apply state successfully, we should reset it here so that we
+	// If we apply state successfully, we should reset it here so that we
 	// know that we have applied since the state was set not ok by event!
-	obj.isStateOK = true // reset
-	return false, nil    // success
+	// This now happens automatically after the engine runs CheckApply().
+	return false, nil // success
 }
 
-// ExecUUID is the UUID struct for ExecRes.
-type ExecUUID struct {
-	BaseUUID
+// ExecUID is the UID struct for ExecRes.
+type ExecUID struct {
+	BaseUID
 	Cmd   string
 	IfCmd string
 	// TODO: add more elements here
 }
 
-// IFF aka if and only if they are equivalent, return true. If not, false.
-func (obj *ExecUUID) IFF(uuid ResUUID) bool {
-	res, ok := uuid.(*ExecUUID)
-	if !ok {
-		return false
-	}
-	if obj.Cmd != res.Cmd {
-		return false
-	}
-	// TODO: add more checks here
-	//if obj.Shell != res.Shell {
-	//	return false
-	//}
-	//if obj.Timeout != res.Timeout {
-	//	return false
-	//}
-	//if obj.WatchCmd != res.WatchCmd {
-	//	return false
-	//}
-	//if obj.WatchShell != res.WatchShell {
-	//	return false
-	//}
-	if obj.IfCmd != res.IfCmd {
-		return false
-	}
-	//if obj.PollInt != res.PollInt {
-	//	return false
-	//}
-	//if obj.State != res.State {
-	//	return false
-	//}
-	return true
-}
-
 // AutoEdges returns the AutoEdge interface. In this case no autoedges are used.
-func (obj *ExecRes) AutoEdges() AutoEdge {
+func (obj *ExecRes) AutoEdges() (AutoEdge, error) {
 	// TODO: parse as many exec params to look for auto edges, for example
 	// the path of the binary in the Cmd variable might be from in a pkg
-	return nil
+	return nil, nil
 }
 
-// GetUUIDs includes all params to make a unique identification of this object.
+// UIDs includes all params to make a unique identification of this object.
 // Most resources only return one, although some resources can return multiple.
-func (obj *ExecRes) GetUUIDs() []ResUUID {
-	x := &ExecUUID{
-		BaseUUID: BaseUUID{name: obj.GetName(), kind: obj.Kind()},
-		Cmd:      obj.Cmd,
-		IfCmd:    obj.IfCmd,
+func (obj *ExecRes) UIDs() []ResUID {
+	x := &ExecUID{
+		BaseUID: BaseUID{Name: obj.GetName(), Kind: obj.GetKind()},
+		Cmd:     obj.Cmd,
+		IfCmd:   obj.IfCmd,
 		// TODO: add more params here
 	}
-	return []ResUUID{x}
+	return []ResUID{x}
 }
 
 // GroupCmp returns whether two resources can be grouped together or not.
@@ -411,43 +363,128 @@ func (obj *ExecRes) GroupCmp(r Res) bool {
 }
 
 // Compare two resources and return if they are equivalent.
-func (obj *ExecRes) Compare(res Res) bool {
-	switch res.(type) {
-	case *ExecRes:
-		res := res.(*ExecRes)
-		if !obj.BaseRes.Compare(res) { // call base Compare
-			return false
-		}
-
-		if obj.Name != res.Name {
-			return false
-		}
-		if obj.Cmd != res.Cmd {
-			return false
-		}
-		if obj.Shell != res.Shell {
-			return false
-		}
-		if obj.Timeout != res.Timeout {
-			return false
-		}
-		if obj.WatchCmd != res.WatchCmd {
-			return false
-		}
-		if obj.WatchShell != res.WatchShell {
-			return false
-		}
-		if obj.IfCmd != res.IfCmd {
-			return false
-		}
-		if obj.PollInt != res.PollInt {
-			return false
-		}
-		if obj.State != res.State {
-			return false
-		}
-	default:
+func (obj *ExecRes) Compare(r Res) bool {
+	// we can only compare ExecRes to others of the same resource kind
+	res, ok := r.(*ExecRes)
+	if !ok {
 		return false
 	}
+	if !obj.BaseRes.Compare(res) { // call base Compare
+		return false
+	}
+	if obj.Name != res.Name {
+		return false
+	}
+
+	if obj.Cmd != res.Cmd {
+		return false
+	}
+	if obj.Shell != res.Shell {
+		return false
+	}
+	if obj.Timeout != res.Timeout {
+		return false
+	}
+	if obj.WatchCmd != res.WatchCmd {
+		return false
+	}
+	if obj.WatchShell != res.WatchShell {
+		return false
+	}
+	if obj.IfCmd != res.IfCmd {
+		return false
+	}
+	if obj.IfShell != res.IfShell {
+		return false
+	}
+
 	return true
 }
+
+// UnmarshalYAML is the custom unmarshal handler for this struct.
+// It is primarily useful for setting the defaults.
+func (obj *ExecRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type rawRes ExecRes // indirection to avoid infinite recursion
+
+	def := obj.Default()      // get the default
+	res, ok := def.(*ExecRes) // put in the right format
+	if !ok {
+		return fmt.Errorf("could not convert to ExecRes")
+	}
+	raw := rawRes(*res) // convert; the defaults go here
+
+	if err := unmarshal(&raw); err != nil {
+		return err
+	}
+
+	*obj = ExecRes(raw) // restore from indirection with type conversion!
+	return nil
+}
+
+// splitWriter mimics what the ssh.CombinedOutput command does, but stores the
+// the stdout and stderr separately. This is slightly tricky because we don't
+// want the combined output to be interleaved incorrectly. It creates sub writer
+// structs which share the same lock and a shared output buffer.
+type splitWriter struct {
+	Stdout *wrapWriter
+	Stderr *wrapWriter
+
+	stdout      bytes.Buffer // just the stdout
+	stderr      bytes.Buffer // just the stderr
+	output      bytes.Buffer // combined output
+	mutex       *sync.Mutex
+	initialized bool // is this initialized?
+}
+
+// Init initializes the splitWriter.
+func (sw *splitWriter) Init() {
+	if sw.initialized {
+		panic("splitWriter is already initialized")
+	}
+	sw.mutex = &sync.Mutex{}
+	sw.Stdout = &wrapWriter{
+		Mutex:  sw.mutex,
+		Buffer: &sw.stdout,
+		Output: &sw.output,
+	}
+	sw.Stderr = &wrapWriter{
+		Mutex:  sw.mutex,
+		Buffer: &sw.stderr,
+		Output: &sw.output,
+	}
+	sw.initialized = true
+}
+
+// String returns the contents of the combined output buffer.
+func (sw *splitWriter) String() string {
+	if !sw.initialized {
+		panic("splitWriter is not initialized")
+	}
+	return sw.output.String()
+}
+
+// wrapWriter is a simple writer which is used internally by splitWriter.
+type wrapWriter struct {
+	Mutex    *sync.Mutex
+	Buffer   *bytes.Buffer // stdout or stderr
+	Output   *bytes.Buffer // combined output
+	Activity bool          // did we get any writes?
+}
+
+// Write writes to both bytes buffers with a parent lock to mix output safely.
+func (w *wrapWriter) Write(p []byte) (int, error) {
+	// TODO: can we move the lock to only guard around the Output.Write ?
+	w.Mutex.Lock()
+	defer w.Mutex.Unlock()
+	w.Activity = true
+	i, err := w.Buffer.Write(p) // first write
+	if err != nil {
+		return i, err
+	}
+	return w.Output.Write(p) // shared write
+}
+
+// String returns the contents of the unshared buffer.
+func (w *wrapWriter) String() string {
+	return w.Buffer.String()
+}

resources/exec_test.go

@@ -0,0 +1,178 @@
+// Mgmt
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"testing"
+)
+
+func TestExecSendRecv1(t *testing.T) {
+	r1 := &ExecRes{
+		BaseRes: BaseRes{
+			Name: "exec1",
+			//MetaParams: MetaParams,
+		},
+		Cmd:   "echo hello world",
+		Shell: "/bin/bash",
+	}
+
+	r1.Setup(nil, r1, r1)
+	defer func() {
+		if err := r1.Close(); err != nil {
+			t.Errorf("close failed with: %v", err)
+		}
+	}()
+	if err := r1.Init(); err != nil {
+		t.Errorf("init failed with: %v", err)
+	}
+	// run artificially without the entire engine
+	if _, err := r1.CheckApply(true); err != nil {
+		t.Errorf("checkapply failed with: %v", err)
+	}
+
+	t.Logf("output is: %v", r1.Output)
+	if r1.Output != nil {
+		t.Logf("output is: %v", *r1.Output)
+	}
+	t.Logf("stdout is: %v", r1.Stdout)
+	if r1.Stdout != nil {
+		t.Logf("stdout is: %v", *r1.Stdout)
+	}
+	t.Logf("stderr is: %v", r1.Stderr)
+	if r1.Stderr != nil {
+		t.Logf("stderr is: %v", *r1.Stderr)
+	}
+
+	if r1.Stdout == nil {
+		t.Errorf("stdout is nil")
+	} else {
+		if out := *r1.Stdout; out != "hello world\n" {
+			t.Errorf("got wrong stdout(%d): %s", len(out), out)
+		}
+	}
+}
+
+func TestExecSendRecv2(t *testing.T) {
+	r1 := &ExecRes{
+		BaseRes: BaseRes{
+			Name: "exec1",
+			//MetaParams: MetaParams,
+		},
+		Cmd:   "echo hello world 1>&2", // to stderr
+		Shell: "/bin/bash",
+	}
+
+	r1.Setup(nil, r1, r1)
+	defer func() {
+		if err := r1.Close(); err != nil {
+			t.Errorf("close failed with: %v", err)
+		}
+	}()
+	if err := r1.Init(); err != nil {
+		t.Errorf("init failed with: %v", err)
+	}
+	// run artificially without the entire engine
+	if _, err := r1.CheckApply(true); err != nil {
+		t.Errorf("checkapply failed with: %v", err)
+	}
+
+	t.Logf("output is: %v", r1.Output)
+	if r1.Output != nil {
+		t.Logf("output is: %v", *r1.Output)
+	}
+	t.Logf("stdout is: %v", r1.Stdout)
+	if r1.Stdout != nil {
+		t.Logf("stdout is: %v", *r1.Stdout)
+	}
+	t.Logf("stderr is: %v", r1.Stderr)
+	if r1.Stderr != nil {
+		t.Logf("stderr is: %v", *r1.Stderr)
+	}
+
+	if r1.Stderr == nil {
+		t.Errorf("stderr is nil")
+	} else {
+		if out := *r1.Stderr; out != "hello world\n" {
+			t.Errorf("got wrong stderr(%d): %s", len(out), out)
+		}
+	}
+}
+
+func TestExecSendRecv3(t *testing.T) {
+	r1 := &ExecRes{
+		BaseRes: BaseRes{
+			Name: "exec1",
+			//MetaParams: MetaParams,
+		},
+		Cmd:   "echo hello world && echo goodbye world 1>&2", // to stdout && stderr
+		Shell: "/bin/bash",
+	}
+
+	r1.Setup(nil, r1, r1)
+	defer func() {
+		if err := r1.Close(); err != nil {
+			t.Errorf("close failed with: %v", err)
+		}
+	}()
+	if err := r1.Init(); err != nil {
+		t.Errorf("init failed with: %v", err)
+	}
+	// run artificially without the entire engine
+	if _, err := r1.CheckApply(true); err != nil {
+		t.Errorf("checkapply failed with: %v", err)
+	}
+
+	t.Logf("output is: %v", r1.Output)
+	if r1.Output != nil {
+		t.Logf("output is: %v", *r1.Output)
+	}
+	t.Logf("stdout is: %v", r1.Stdout)
+	if r1.Stdout != nil {
+		t.Logf("stdout is: %v", *r1.Stdout)
+	}
+	t.Logf("stderr is: %v", r1.Stderr)
+	if r1.Stderr != nil {
+		t.Logf("stderr is: %v", *r1.Stderr)
+	}
+
+	if r1.Output == nil {
+		t.Errorf("output is nil")
+	} else {
+		// it looks like bash or golang race to the write, so whichever
+		// order they come out in is ok, as long as they come out whole
+		if out := *r1.Output; out != "hello world\ngoodbye world\n" && out != "goodbye world\nhello world\n" {
+			t.Errorf("got wrong output(%d): %s", len(out), out)
+		}
+	}
+
+	if r1.Stdout == nil {
+		t.Errorf("stdout is nil")
+	} else {
+		if out := *r1.Stdout; out != "hello world\n" {
+			t.Errorf("got wrong stdout(%d): %s", len(out), out)
+		}
+	}
+
+	if r1.Stderr == nil {
+		t.Errorf("stderr is nil")
+	} else {
+		if out := *r1.Stderr; out != "goodbye world\n" {
+			t.Errorf("got wrong stderr(%d): %s", len(out), out)
+		}
+	}
+}

resources/file.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -27,77 +27,143 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
+	"os/user"
 	"path"
 	"path/filepath"
+	"strconv"
 	"strings"
-	"time"
+	"syscall"
 
-	"github.com/purpleidea/mgmt/event"
-	"github.com/purpleidea/mgmt/global" // XXX: package mgmtmain instead?
 	"github.com/purpleidea/mgmt/recwatch"
 	"github.com/purpleidea/mgmt/util"
+
+	errwrap "github.com/pkg/errors"
 )
 
 func init() {
 	gob.Register(&FileRes{})
+	RegisterResource("file", func() Res { return &FileRes{} })
 }
 
 // FileRes is a file and directory resource.
 type FileRes struct {
 	BaseRes    `yaml:",inline"`
-	Path       string `yaml:"path"` // path variable (should default to name)
-	Dirname    string `yaml:"dirname"`
-	Basename   string `yaml:"basename"`
-	Content    string `yaml:"content"` // FIXME: how do you describe: "leave content alone" - state = "create" ?
-	Source     string `yaml:"source"`  // file path for source content
-	State      string `yaml:"state"`   // state: exists/present?, absent, (undefined?)
-	Recurse    bool   `yaml:"recurse"`
-	Force      bool   `yaml:"force"`
-	path       string // computed path
-	isDir      bool   // computed isDir
+	Path       string  `yaml:"path"` // path variable (should default to name)
+	Dirname    string  `yaml:"dirname"`
+	Basename   string  `yaml:"basename"`
+	Content    *string `yaml:"content"` // nil to mark as undefined
+	Source     string  `yaml:"source"`  // file path for source content
+	State      string  `yaml:"state"`   // state: exists/present?, absent, (undefined?)
+	Owner      string  `yaml:"owner"`
+	Group      string  `yaml:"group"`
+	Mode       string  `yaml:"mode"`
+	Recurse    bool    `yaml:"recurse"`
+	Force      bool    `yaml:"force"`
+	path       string  // computed path
+	isDir      bool    // computed isDir
 	sha256sum  string
 	recWatcher *recwatch.RecWatcher
 }
 
-// NewFileRes is a constructor for this resource. It also calls Init() for you.
-func NewFileRes(name, path, dirname, basename, content, source, state string, recurse, force bool) *FileRes {
-	obj := &FileRes{
+// Default returns some sensible defaults for this resource.
+func (obj *FileRes) Default() Res {
+	return &FileRes{
 		BaseRes: BaseRes{
-			Name: name,
+			MetaParams: DefaultMetaParams, // force a default
 		},
-		Path:     path,
-		Dirname:  dirname,
-		Basename: basename,
-		Content:  content,
-		Source:   source,
-		State:    state,
-		Recurse:  recurse,
-		Force:    force,
+		State: "exists",
 	}
-	obj.Init()
-	return obj
+}
+
+// Validate reports any problems with the struct definition.
+func (obj *FileRes) Validate() error {
+	if obj.Dirname != "" && !strings.HasSuffix(obj.Dirname, "/") {
+		return fmt.Errorf("dirname must end with a slash")
+	}
+
+	if strings.HasPrefix(obj.Basename, "/") {
+		return fmt.Errorf("basename must not start with a slash")
+	}
+
+	if obj.Content != nil && obj.Source != "" {
+		return fmt.Errorf("can't specify both Content and Source")
+	}
+
+	if obj.isDir && obj.Content != nil { // makes no sense
+		return fmt.Errorf("can't specify Content when creating a Dir")
+	}
+
+	if obj.Mode != "" {
+		if _, err := obj.mode(); err != nil {
+			return err
+		}
+	}
+
+	if _, err := obj.uid(); obj.Owner != "" && err != nil {
+		return err
+	}
+
+	if _, err := obj.gid(); obj.Group != "" && err != nil {
+		return err
+	}
+
+	// XXX: should this specify that we create an empty directory instead?
+	//if obj.Source == "" && obj.isDir {
+	//	return fmt.Errorf("Can't specify an empty source when creating a Dir.")
+	//}
+
+	return obj.BaseRes.Validate()
+}
+
+// mode returns the file permission specified on the graph. It doesn't handle
+// the case where the mode is not specified. The caller should check obj.Mode is
+// not empty.
+func (obj *FileRes) mode() (os.FileMode, error) {
+	m, err := strconv.ParseInt(obj.Mode, 8, 32)
+	if err != nil {
+		return os.FileMode(0), errwrap.Wrapf(err, "Mode should be an octal number (%s)", obj.Mode)
+	}
+	return os.FileMode(m), nil
+}
+
+// uid returns the user id for the owner specified in the yaml file graph.
+// Caller should first check obj.Owner is not empty
+func (obj *FileRes) uid() (int, error) {
+	u2, err2 := user.LookupId(obj.Owner)
+	if err2 == nil {
+		return strconv.Atoi(u2.Uid)
+	}
+
+	u, err := user.Lookup(obj.Owner)
+	if err == nil {
+		return strconv.Atoi(u.Uid)
+	}
+
+	return -1, errwrap.Wrapf(err, "owner lookup error (%s)", obj.Owner)
 }
 
 // Init runs some startup code for this resource.
 func (obj *FileRes) Init() error {
 	obj.sha256sum = ""
-	if obj.Path == "" { // use the name as the path default if missing
-		obj.Path = obj.BaseRes.Name
-	}
 	obj.path = obj.GetPath()                     // compute once
 	obj.isDir = strings.HasSuffix(obj.path, "/") // dirs have trailing slashes
 
-	obj.BaseRes.kind = "File"
+	obj.BaseRes.Kind = "file"
 	return obj.BaseRes.Init() // call base init, b/c we're overriding
 }
 
 // GetPath returns the actual path to use for this resource. It computes this
 // after analysis of the Path, Dirname and Basename values. Dirs end with slash.
 func (obj *FileRes) GetPath() string {
-	d := util.Dirname(obj.Path)
-	b := util.Basename(obj.Path)
+	p := obj.Path
+	if obj.Path == "" { // use the name as the path default if missing
+		p = obj.BaseRes.Name
+	}
+
+	d := util.Dirname(p)
+	b := util.Basename(p)
 	if obj.Dirname == "" && obj.Basename == "" {
-		return obj.Path
+		return p
 	}
 	if obj.Dirname == "" {
 		return d + obj.Basename
@@ -109,117 +175,58 @@ func (obj *FileRes) GetPath() string {
 	return obj.Dirname + obj.Basename
 }
 
-// Validate reports any problems with the struct definition.
-func (obj *FileRes) Validate() error {
-	if obj.Dirname != "" && !strings.HasSuffix(obj.Dirname, "/") {
-		return fmt.Errorf("Dirname must end with a slash.")
-	}
-
-	if strings.HasPrefix(obj.Basename, "/") {
-		return fmt.Errorf("Basename must not start with a slash.")
-	}
-
-	if obj.Content != "" && obj.Source != "" {
-		return fmt.Errorf("Can't specify both Content and Source.")
-	}
-
-	if obj.isDir && obj.Content != "" { // makes no sense
-		return fmt.Errorf("Can't specify Content when creating a Dir.")
-	}
-
-	// XXX: should this specify that we create an empty directory instead?
-	//if obj.Source == "" && obj.isDir {
-	//	return fmt.Errorf("Can't specify an empty source when creating a Dir.")
-	//}
-
-	return nil
-}
-
 // Watch is the primary listener for this resource and it outputs events.
 // This one is a file watcher for files and directories.
 // Modify with caution, it is probably important to write some test cases first!
 // If the Watch returns an error, it means that something has gone wrong, and it
 // must be restarted. On a clean exit it returns nil.
 // FIXME: Also watch the source directory when using obj.Source !!!
-func (obj *FileRes) Watch(processChan chan event.Event) error {
-	if obj.IsWatching() {
-		return nil // TODO: should this be an error?
-	}
-	obj.SetWatching(true)
-	defer obj.SetWatching(false)
-	cuuid := obj.converger.Register()
-	defer cuuid.Unregister()
-
-	var startup bool
-	Startup := func(block bool) <-chan time.Time {
-		if block {
-			return nil // blocks forever
-			//return make(chan time.Time) // blocks forever
-		}
-		return time.After(time.Duration(500) * time.Millisecond) // 1/2 the resolution of converged timeout
-	}
-
+func (obj *FileRes) Watch() error {
 	var err error
-	obj.recWatcher, err = recwatch.NewRecWatcher(obj.Path, obj.Recurse)
+	obj.recWatcher, err = recwatch.NewRecWatcher(obj.path, obj.Recurse)
 	if err != nil {
 		return err
 	}
 	defer obj.recWatcher.Close()
 
+	// notify engine that we're running
+	if err := obj.Running(); err != nil {
+		return err // bubble up a NACK...
+	}
+
 	var send = false // send event?
-	var exit = false
-	var dirty = false
+	var exit *error
 
 	for {
-		if global.DEBUG {
-			log.Printf("%s[%s]: Watching: %s", obj.Kind(), obj.GetName(), obj.Path) // attempting to watch...
+		if obj.debug {
+			log.Printf("%s: Watching: %s", obj, obj.path) // attempting to watch...
 		}
 
-		obj.SetState(ResStateWatching) // reset
 		select {
 		case event, ok := <-obj.recWatcher.Events():
 			if !ok { // channel shutdown
 				return nil
 			}
-			cuuid.SetConverged(false)
 			if err := event.Error; err != nil {
-				return fmt.Errorf("Unknown %s[%s] watcher error: %v", obj.Kind(), obj.GetName(), err)
+				return errwrap.Wrapf(err, "unknown %s watcher error", obj)
 			}
-			if global.DEBUG { // don't access event.Body if event.Error isn't nil
-				log.Printf("%s[%s]: Event(%s): %v", obj.Kind(), obj.GetName(), event.Body.Name, event.Body.Op)
+			if obj.debug { // don't access event.Body if event.Error isn't nil
+				log.Printf("%s: Event(%s): %v", obj, event.Body.Name, event.Body.Op)
 			}
 			send = true
-			dirty = true
+			obj.StateOK(false) // dirty
 
-		case event := <-obj.events:
-			cuuid.SetConverged(false)
-			if exit, send = obj.ReadEvent(&event); exit {
-				return nil // exit
+		case event := <-obj.Events():
+			if exit, send = obj.ReadEvent(event); exit != nil {
+				return *exit // exit
 			}
-			//dirty = false // these events don't invalidate state
-
-		case <-cuuid.ConvergedTimer():
-			cuuid.SetConverged(true) // converged!
-			continue
-
-		case <-Startup(startup):
-			cuuid.SetConverged(false)
-			send = true
-			dirty = true
+			//obj.StateOK(false) // dirty // these events don't invalidate state
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
-			startup = true // startup finished
 			send = false
-			// only invalid state on certain types of events
-			if dirty {
-				dirty = false
-				obj.isStateOK = false // something made state dirty
-			}
-			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
-				return err // we exit or bubble up a NACK...
-			}
+			obj.Event()
 		}
 	}
 }
@@ -245,7 +252,7 @@ type FileInfo struct {
 // ReadDir reads a directory path, and returns a list of enhanced FileInfo's.
 func ReadDir(path string) ([]FileInfo, error) {
 	if !strings.HasSuffix(path, "/") { // dirs have trailing slashes
-		return nil, fmt.Errorf("Path must be a directory.")
+		return nil, fmt.Errorf("path must be a directory")
 	}
 	output := []FileInfo{} // my file info
 	fileInfos, err := ioutil.ReadDir(path)
@@ -259,7 +266,7 @@ func ReadDir(path string) ([]FileInfo, error) {
 		abs := path + smartPath(fi)
 		rel, err := filepath.Rel(path, abs) // NOTE: calls Clean()
 		if err != nil {                     // shouldn't happen
-			return nil, fmt.Errorf("ReadDir: Unhandled error: %v", err)
+			return nil, errwrap.Wrapf(err, "unhandled error in ReadDir")
 		}
 		if fi.IsDir() {
 			rel += "/" // add a trailing slash for dirs
@@ -294,14 +301,14 @@ func mapPaths(fileInfos []FileInfo) map[string]FileInfo {
 func (obj *FileRes) fileCheckApply(apply bool, src io.ReadSeeker, dst string, sha256sum string) (string, bool, error) {
 	// TODO: does it make sense to switch dst to an io.Writer ?
 	// TODO: use obj.Force when dealing with symlinks and other file types!
-	if global.DEBUG {
+	if obj.debug {
 		log.Printf("fileCheckApply: %s -> %s", src, dst)
 	}
 
 	srcFile, isFile := src.(*os.File)
 	_, isBytes := src.(*bytes.Reader) // supports seeking!
 	if !isFile && !isBytes {
-		return "", false, fmt.Errorf("Can't open src as either file or buffer!")
+		return "", false, fmt.Errorf("can't open src as either file or buffer")
 	}
 
 	var srcStat os.FileInfo
@@ -313,7 +320,7 @@ func (obj *FileRes) fileCheckApply(apply bool, src io.ReadSeeker, dst string, sh
 		}
 		// TODO: deal with symlinks
 		if !srcStat.Mode().IsRegular() { // can't copy non-regular files or dirs
-			return "", false, fmt.Errorf("Non-regular src file: %s (%q)", srcStat.Name(), srcStat.Mode())
+			return "", false, fmt.Errorf("non-regular src file: %s (%q)", srcStat.Name(), srcStat.Mode())
 		}
 	}
 
@@ -337,12 +344,12 @@ func (obj *FileRes) fileCheckApply(apply bool, src io.ReadSeeker, dst string, sh
 			return "", false, nil
 		}
 		if !obj.Force {
-			return "", false, fmt.Errorf("Can't force dir into file: %s", dst)
+			return "", false, fmt.Errorf("can't force dir into file: %s", dst)
 		}
 
 		cleanDst := path.Clean(dst)
 		if cleanDst == "" || cleanDst == "/" {
-			return "", false, fmt.Errorf("Don't want to remove root!") // safety
+			return "", false, fmt.Errorf("don't want to remove root") // safety
 		}
 		// FIXME: respect obj.Recurse here...
 		// there is a dir here, where we want a file...
@@ -354,7 +361,7 @@ func (obj *FileRes) fileCheckApply(apply bool, src io.ReadSeeker, dst string, sh
 
 	} else if err == nil {
 		if !dstStat.Mode().IsRegular() {
-			return "", false, fmt.Errorf("Non-regular dst file: %s (%q)", dstStat.Name(), dstStat.Mode())
+			return "", false, fmt.Errorf("non-regular dst file: %s (%q)", dstStat.Name(), dstStat.Mode())
 		}
 		if isFile && os.SameFile(srcStat, dstStat) { // same inode, we're done!
 			return "", true, nil
@@ -365,7 +372,7 @@ func (obj *FileRes) fileCheckApply(apply bool, src io.ReadSeeker, dst string, sh
 		// hash comparison (efficient because we can cache hash of content str)
 		if sha256sum == "" { // cache is invalid
 			hash := sha256.New()
-			// TODO file existence test?
+			// TODO: file existence test?
 			if _, err := io.Copy(hash, src); err != nil {
 				return "", false, err
 			}
@@ -391,7 +398,7 @@ func (obj *FileRes) fileCheckApply(apply bool, src io.ReadSeeker, dst string, sh
 	if !apply {
 		return sha256sum, false, nil
 	}
-	if global.DEBUG {
+	if obj.debug {
 		log.Printf("fileCheckApply: Apply: %s -> %s", src, dst)
 	}
 
@@ -412,26 +419,74 @@ func (obj *FileRes) fileCheckApply(apply bool, src io.ReadSeeker, dst string, sh
 	// syscall.Splice(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int64, err error)
 
 	// TODO: should we offer a way to cancel the copy on ^C ?
-	if global.DEBUG {
+	if obj.debug {
 		log.Printf("fileCheckApply: Copy: %s -> %s", src, dst)
 	}
 	if n, err := io.Copy(dstFile, src); err != nil {
 		return sha256sum, false, err
-	} else if global.DEBUG {
+	} else if obj.debug {
 		log.Printf("fileCheckApply: Copied: %v", n)
 	}
 	return sha256sum, false, dstFile.Sync()
 }
 
+// dirCheckApply is the CheckApply operation for an empty directory.
+func (obj *FileRes) dirCheckApply(apply bool) (bool, error) {
+	// check if the path exists and is a directory
+	st, err := os.Stat(obj.path)
+	if err != nil && !os.IsNotExist(err) {
+		return false, errwrap.Wrapf(err, "error checking file resource existence")
+	}
+
+	if err == nil && st.IsDir() {
+		return true, nil // already a directory, nothing to do
+	}
+	if err == nil && !st.IsDir() && !obj.Force {
+		return false, fmt.Errorf("can't force file into dir: %s", obj.path)
+	}
+
+	if !apply {
+		return false, nil
+	}
+
+	// the path exists and is not a directory
+	// delete the file if force is given
+	if err == nil && !st.IsDir() {
+		log.Printf("dirCheckApply: Removing (force): %s", obj.path)
+		if err := os.Remove(obj.path); err != nil {
+			return false, err
+		}
+	}
+
+	// create the empty directory
+	var mode os.FileMode
+	if obj.Mode != "" {
+		mode, err = obj.mode()
+		if err != nil {
+			return false, err
+		}
+	} else {
+		mode = os.ModePerm
+	}
+
+	if obj.Force {
+		// FIXME: respect obj.Recurse here...
+		// TODO: add recurse limit here
+		return false, os.MkdirAll(obj.path, mode)
+	}
+
+	return false, os.Mkdir(obj.path, mode)
+}
+
 // syncCheckApply is the CheckApply operation for a source and destination dir.
 // It is recursive and can create directories directly, and files via the usual
 // fileCheckApply method. It returns checkOK and error as is normally expected.
 func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
-	if global.DEBUG {
+	if obj.debug {
 		log.Printf("syncCheckApply: %s -> %s", src, dst)
 	}
 	if src == "" || dst == "" {
-		return false, fmt.Errorf("The src and dst must not be empty!")
+		return false, fmt.Errorf("the src and dst must not be empty")
 	}
 
 	var checkOK = true
@@ -441,16 +496,16 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 	dstIsDir := strings.HasSuffix(dst, "/")
 
 	if srcIsDir != dstIsDir {
-		return false, fmt.Errorf("The src and dst must be both either files or directories.")
+		return false, fmt.Errorf("the src and dst must be both either files or directories")
 	}
 
 	if !srcIsDir && !dstIsDir {
-		if global.DEBUG {
+		if obj.debug {
 			log.Printf("syncCheckApply: %s -> %s", src, dst)
 		}
 		fin, err := os.Open(src)
 		if err != nil {
-			if global.DEBUG && os.IsNotExist(err) { // if we get passed an empty src
+			if obj.debug && os.IsNotExist(err) { // if we get passed an empty src
 				log.Printf("syncCheckApply: Missing src: %s", src)
 			}
 			return false, err
@@ -494,10 +549,10 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 				if _, ok := smartDst[relPathFile]; ok {
 					absCleanDst := path.Clean(absDst)
 					if !obj.Force {
-						return false, fmt.Errorf("Can't force file into dir: %s", absCleanDst)
+						return false, fmt.Errorf("can't force file into dir: %s", absCleanDst)
 					}
 					if absCleanDst == "" || absCleanDst == "/" {
-						return false, fmt.Errorf("Don't want to remove root!") // safety
+						return false, fmt.Errorf("don't want to remove root") // safety
 					}
 					log.Printf("syncCheckApply: Removing (force): %s", absCleanDst)
 					if err := os.Remove(absCleanDst); err != nil {
@@ -506,7 +561,7 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 					delete(smartDst, relPathFile) // rm from purge list
 				}
 
-				if global.DEBUG {
+				if obj.debug {
 					log.Printf("syncCheckApply: mkdir -m %s '%s'", fileInfo.Mode(), absDst)
 				}
 				if err := os.Mkdir(absDst, fileInfo.Mode()); err != nil {
@@ -517,12 +572,12 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 			// if we're a regular file, the recurse will create it
 		}
 
-		if global.DEBUG {
+		if obj.debug {
 			log.Printf("syncCheckApply: Recurse: %s -> %s", absSrc, absDst)
 		}
 		if obj.Recurse {
 			if c, err := obj.syncCheckApply(apply, absSrc, absDst); err != nil { // recurse
-				return false, fmt.Errorf("syncCheckApply: Recurse failed: %v", err)
+				return false, errwrap.Wrapf(err, "syncCheckApply: Recurse failed")
 			} else if !c { // don't let subsequent passes make this true
 				checkOK = false
 			}
@@ -542,7 +597,7 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 		absDst := fileInfo.AbsPath // absolute path (should get removed)
 		absCleanDst := path.Clean(absDst)
 		if absCleanDst == "" || absCleanDst == "/" {
-			return false, fmt.Errorf("Don't want to remove root!") // safety
+			return false, fmt.Errorf("don't want to remove root") // safety
 		}
 
 		// FIXME: respect obj.Recurse here...
@@ -563,7 +618,7 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 		_ = absSrc
 		//log.Printf("syncCheckApply: Recurse rm: %s -> %s", absSrc, absDst)
 		//if c, err := obj.syncCheckApply(apply, absSrc, absDst); err != nil {
-		//	return false, fmt.Errorf("syncCheckApply: Recurse rm failed: %v", err)
+		//	return false, errwrap.Wrapf(err, "syncCheckApply: Recurse rm failed")
 		//} else if !c { // don't let subsequent passes make this true
 		//	checkOK = false
 		//}
@@ -581,7 +636,7 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 
 // contentCheckApply performs a CheckApply for the file existence and content.
 func (obj *FileRes) contentCheckApply(apply bool) (checkOK bool, _ error) {
-	log.Printf("%v[%v]: contentCheckApply(%t)", obj.Kind(), obj.GetName(), apply)
+	log.Printf("%s: contentCheckApply(%t)", obj, apply)
 
 	if obj.State == "absent" {
 		if _, err := os.Stat(obj.path); os.IsNotExist(err) {
@@ -600,7 +655,7 @@ func (obj *FileRes) contentCheckApply(apply bool) (checkOK bool, _ error) {
 
 		// apply portion
 		if obj.path == "" || obj.path == "/" {
-			return false, fmt.Errorf("Don't want to remove root!") // safety
+			return false, fmt.Errorf("don't want to remove root") // safety
 		}
 		log.Printf("contentCheckApply: Removing: %s", obj.path)
 		// FIXME: respect obj.Recurse here...
@@ -609,12 +664,17 @@ func (obj *FileRes) contentCheckApply(apply bool) (checkOK bool, _ error) {
 		return false, err             // either nil or not
 	}
 
-	if obj.Source == "" { // do the obj.Content checks first...
-		if obj.isDir { // TODO: should we create an empty dir this way?
-			log.Fatal("XXX: Not implemented!") // XXX
-		}
+	if obj.isDir && obj.Source == "" {
+		return obj.dirCheckApply(apply)
+	}
 
-		bufferSrc := bytes.NewReader([]byte(obj.Content))
+	// content is not defined, leave it alone...
+	if obj.Content == nil {
+		return true, nil
+	}
+
+	if obj.Source == "" { // do the obj.Content checks first...
+		bufferSrc := bytes.NewReader([]byte(*obj.Content))
 		sha256sum, checkOK, err := obj.fileCheckApply(apply, bufferSrc, obj.path, obj.sha256sum)
 		if sha256sum != "" { // empty values mean errored or didn't hash
 			// this can be valid even when the whole function errors
@@ -636,13 +696,127 @@ func (obj *FileRes) contentCheckApply(apply bool) (checkOK bool, _ error) {
 	return checkOK, nil
 }
 
+// chmodCheckApply performs a CheckApply for the file permissions.
+func (obj *FileRes) chmodCheckApply(apply bool) (checkOK bool, _ error) {
+	log.Printf("%s: chmodCheckApply(%t)", obj, apply)
+
+	if obj.State == "absent" {
+		// File is absent
+		return true, nil
+	}
+
+	if obj.Mode == "" {
+		// No mode specified, everything is ok
+		return true, nil
+	}
+
+	mode, err := obj.mode()
+
+	// If the file does not exist and we are in
+	// noop mode, do not throw an error.
+	if os.IsNotExist(err) && !apply {
+		return false, nil
+	}
+
+	if err != nil {
+		return false, err
+	}
+
+	st, err := os.Stat(obj.path)
+	if err != nil {
+		return false, err
+	}
+
+	// Nothing to do
+	if st.Mode() == mode {
+		return true, nil
+	}
+
+	// Not clean but don't apply
+	if !apply {
+		return false, nil
+	}
+
+	err = os.Chmod(obj.path, mode)
+	return false, err
+}
+
+// chownCheckApply performs a CheckApply for the file ownership.
+func (obj *FileRes) chownCheckApply(apply bool) (checkOK bool, _ error) {
+	var expectedUID, expectedGID int
+	log.Printf("%s: chownCheckApply(%t)", obj, apply)
+
+	if obj.State == "absent" {
+		// File is absent or no owner specified
+		return true, nil
+	}
+
+	st, err := os.Stat(obj.path)
+
+	// If the file does not exist and we are in
+	// noop mode, do not throw an error.
+	if os.IsNotExist(err) && !apply {
+		return false, nil
+	}
+
+	if err != nil {
+		return false, err
+	}
+
+	stUnix, ok := st.Sys().(*syscall.Stat_t)
+	if !ok {
+		// Not unix
+		panic("No support for your platform")
+	}
+
+	if obj.Owner != "" {
+		expectedUID, err = obj.uid()
+		if err != nil {
+			return false, err
+		}
+	} else {
+		// Nothing specified, no changes to be made, expect same as actual
+		expectedUID = int(stUnix.Uid)
+	}
+
+	if obj.Group != "" {
+		expectedGID, err = obj.gid()
+		if err != nil {
+			return false, err
+		}
+	} else {
+		// Nothing specified, no changes to be made, expect same as actual
+		expectedGID = int(stUnix.Gid)
+	}
+
+	// Nothing to do
+	if int(stUnix.Uid) == expectedUID && int(stUnix.Gid) == expectedGID {
+		return true, nil
+	}
+
+	// Not clean, but don't apply
+	if !apply {
+		return false, nil
+	}
+
+	err = os.Chown(obj.path, expectedUID, expectedGID)
+	return false, err
+}
+
 // CheckApply checks the resource state and applies the resource if the bool
 // input is true. It returns error info and if the state check passed or not.
 func (obj *FileRes) CheckApply(apply bool) (checkOK bool, _ error) {
-	log.Printf("%v[%v]: CheckApply(%t)", obj.Kind(), obj.GetName(), apply)
 
-	if obj.isStateOK { // cache the state
-		return true, nil
+	// NOTE: all send/recv change notifications *must* be processed before
+	// there is a possibility of failure in CheckApply. This is because if
+	// we fail (and possibly run again) the subsequent send->recv transfer
+	// might not have a new value to copy, and therefore we won't see this
+	// notification of change. Therefore, it is important to process these
+	// promptly, if they must not be lost, such as for cache invalidation.
+	if val, exists := obj.Recv["Content"]; exists && val.Changed {
+		// if we received on Content, and it changed, invalidate the cache!
+		log.Printf("contentCheckApply: Invalidating sha256sum of `Content`")
+		obj.sha256sum = "" // invalidate!!
 	}
 
 	checkOK = true
@@ -653,36 +827,30 @@ func (obj *FileRes) CheckApply(apply bool) (checkOK bool, _ error) {
 		checkOK = false
 	}
 
-	// TODO
-	//if c, err := obj.chmodCheckApply(apply); err != nil {
-	//	return false, err
-	//} else if !c {
-	//	checkOK = false
-	//}
-
-	// TODO
-	//if c, err := obj.chownCheckApply(apply); err != nil {
-	//	return false, err
-	//} else if !c {
-	//	checkOK = false
-	//}
-
-	// if we did work successfully, or are in a good state, then state is ok
-	if apply || checkOK {
-		obj.isStateOK = true
+	if c, err := obj.chmodCheckApply(apply); err != nil {
+		return false, err
+	} else if !c {
+		checkOK = false
 	}
+
+	if c, err := obj.chownCheckApply(apply); err != nil {
+		return false, err
+	} else if !c {
+		checkOK = false
+	}
+
 	return checkOK, nil // w00t
 }
 
-// FileUUID is the UUID struct for FileRes.
-type FileUUID struct {
-	BaseUUID
+// FileUID is the UID struct for FileRes.
+type FileUID struct {
+	BaseUID
 	path string
 }
 
 // IFF aka if and only if they are equivalent, return true. If not, false.
-func (obj *FileUUID) IFF(uuid ResUUID) bool {
-	res, ok := uuid.(*FileUUID)
+func (obj *FileUID) IFF(uid ResUID) bool {
+	res, ok := uid.(*FileUID)
 	if !ok {
 		return false
 	}
@@ -691,13 +859,13 @@ func (obj *FileUUID) IFF(uuid ResUUID) bool {
 
 // FileResAutoEdges holds the state of the auto edge generator.
 type FileResAutoEdges struct {
-	data    []ResUUID
+	data    []ResUID
 	pointer int
 	found   bool
 }
 
 // Next returns the next automatic edge.
-func (obj *FileResAutoEdges) Next() []ResUUID {
+func (obj *FileResAutoEdges) Next() []ResUID {
 	if obj.found {
 		log.Fatal("Shouldn't be called anymore!")
 	}
@@ -706,7 +874,7 @@ func (obj *FileResAutoEdges) Next() []ResUUID {
 	}
 	value := obj.data[obj.pointer]
 	obj.pointer++
-	return []ResUUID{value} // we return one, even though api supports N
+	return []ResUID{value} // we return one, even though api supports N
 }
 
 // Test gets results of the earlier Next() call, & returns if we should continue!
@@ -730,17 +898,18 @@ func (obj *FileResAutoEdges) Test(input []bool) bool {
 
 // AutoEdges generates a simple linear sequence of each parent directory from
 // the bottom up!
-func (obj *FileRes) AutoEdges() AutoEdge {
-	var data []ResUUID                             // store linear result chain here...
-	values := util.PathSplitFullReversed(obj.path) // build it
-	_, values = values[0], values[1:]              // get rid of first value which is me!
+func (obj *FileRes) AutoEdges() (AutoEdge, error) {
+	var data []ResUID // store linear result chain here...
+	// build it, but don't use obj.path because this gets called before Init
+	values := util.PathSplitFullReversed(obj.GetPath())
+	_, values = values[0], values[1:] // get rid of first value which is me!
 	for _, x := range values {
 		var reversed = true // cheat by passing a pointer
-		data = append(data, &FileUUID{
-			BaseUUID: BaseUUID{
-				name:     obj.GetName(),
-				kind:     obj.Kind(),
-				reversed: &reversed,
+		data = append(data, &FileUID{
+			BaseUID: BaseUID{
+				Name:     obj.GetName(),
+				Kind:     obj.GetKind(),
+				Reversed: &reversed,
 			},
 			path: x, // what matters
 		}) // build list
@@ -749,17 +918,17 @@ func (obj *FileRes) AutoEdges() AutoEdge {
 		data:    data,
 		pointer: 0,
 		found:   false,
-	}
+	}, nil
 }
 
-// GetUUIDs includes all params to make a unique identification of this object.
+// UIDs includes all params to make a unique identification of this object.
 // Most resources only return one, although some resources can return multiple.
-func (obj *FileRes) GetUUIDs() []ResUUID {
-	x := &FileUUID{
-		BaseUUID: BaseUUID{name: obj.GetName(), kind: obj.Kind()},
-		path:     obj.path,
+func (obj *FileRes) UIDs() []ResUID {
+	x := &FileUID{
+		BaseUID: BaseUID{Name: obj.GetName(), Kind: obj.GetKind()},
+		path:    obj.GetPath(), // not obj.path b/c we didn't init yet!
 	}
-	return []ResUUID{x}
+	return []ResUID{x}
 }
 
 // GroupCmp returns whether two resources can be grouped together or not.
@@ -774,38 +943,43 @@ func (obj *FileRes) GroupCmp(r Res) bool {
 }
 
 // Compare two resources and return if they are equivalent.
-func (obj *FileRes) Compare(res Res) bool {
-	switch res.(type) {
-	case *FileRes:
-		res := res.(*FileRes)
-		if !obj.BaseRes.Compare(res) { // call base Compare
-			return false
-		}
-
-		if obj.Name != res.Name {
-			return false
-		}
-		if obj.path != res.Path {
-			return false
-		}
-		if obj.Content != res.Content {
-			return false
-		}
-		if obj.Source != res.Source {
-			return false
-		}
-		if obj.State != res.State {
-			return false
-		}
-		if obj.Recurse != res.Recurse {
-			return false
-		}
-		if obj.Force != res.Force {
-			return false
-		}
-	default:
+func (obj *FileRes) Compare(r Res) bool {
+	// we can only compare FileRes to others of the same resource kind
+	res, ok := r.(*FileRes)
+	if !ok {
 		return false
 	}
+	if !obj.BaseRes.Compare(res) { // call base Compare
+		return false
+	}
+	if obj.Name != res.Name {
+		return false
+	}
+
+	if obj.path != res.path {
+		return false
+	}
+	if (obj.Content == nil) != (res.Content == nil) { // xor
+		return false
+	}
+	if obj.Content != nil && res.Content != nil {
+		if *obj.Content != *res.Content { // compare the strings
+			return false
+		}
+	}
+	if obj.Source != res.Source {
+		return false
+	}
+	if obj.State != res.State {
+		return false
+	}
+	if obj.Recurse != res.Recurse {
+		return false
+	}
+	if obj.Force != res.Force {
+		return false
+	}
+
 	return true
 }
 
@@ -814,3 +988,23 @@ func (obj *FileRes) CollectPattern(pattern string) {
 	// XXX: currently the pattern for files can only override the Dirname variable :P
 	obj.Dirname = pattern // XXX: simplistic for now
 }
+
+// UnmarshalYAML is the custom unmarshal handler for this struct.
+// It is primarily useful for setting the defaults.
+func (obj *FileRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type rawRes FileRes // indirection to avoid infinite recursion
+
+	def := obj.Default()      // get the default
+	res, ok := def.(*FileRes) // put in the right format
+	if !ok {
+		return fmt.Errorf("could not convert to FileRes")
+	}
+	raw := rawRes(*res) // convert; the defaults go here
+
+	if err := unmarshal(&raw); err != nil {
+		return err
+	}
+
+	*obj = FileRes(raw) // restore from indirection with type conversion!
+	return nil
+}

resources/file_attrs.go

@@ -0,0 +1,43 @@
+// Mgmt
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// +build go1.7
+
+package resources
+
+import (
+	"os/user"
+	"strconv"
+
+	errwrap "github.com/pkg/errors"
+)
+
+// gid returns the group id for the group specified in the yaml file graph.
+// Caller should first check obj.Group is not empty
+func (obj *FileRes) gid() (int, error) {
+	g2, err2 := user.LookupGroupId(obj.Group)
+	if err2 == nil {
+		return strconv.Atoi(g2.Gid)
+	}
+
+	g, err := user.LookupGroup(obj.Group)
+	if err == nil {
+		return strconv.Atoi(g.Gid)
+	}
+
+	return -1, errwrap.Wrapf(err, "Group lookup error (%s)", obj.Group)
+}