golint: Fix some golint issues

Let tip builds pass in travis too!

.ackrc

@@ -1,2 +1,3 @@
 --ignore-dir=old/
 --ignore-dir=tmp/
+--ignore-dir=vendor/

.dockerignore

@@ -0,0 +1 @@
+docker

.editorconfig

@@ -0,0 +1,19 @@
+; This file is for unifying the coding style for different editors and IDEs.
+; Plugins are available for notepad++, emacs, vim, gedit,
+; textmate, visual studio, and more.
+;
+; See http://editorconfig.org for details.
+
+# Top-most EditorConfig file.
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.go]
+indent_style = tab
+
+[Makefile]
+indent_style = tab

.gitignore

@@ -1,9 +1,11 @@
+.idea/
 .omv/
 .ssh/
 .vagrant/
-mgmt-documentation.pdf
 old/
 tmp/
 *_stringer.go
 mgmt
+mgmt.static
+mgmt.iml
 rpmbuild/

.gitmodules

@@ -0,0 +1,15 @@
+[submodule "vendor/github.com/coreos/etcd"]
+	path = vendor/github.com/coreos/etcd
+	url = https://github.com/coreos/etcd/
+[submodule "vendor/google.golang.org/grpc"]
+	path = vendor/google.golang.org/grpc
+	url = https://github.com/grpc/grpc-go
+[submodule "vendor/github.com/grpc-ecosystem/grpc-gateway"]
+	path = vendor/github.com/grpc-ecosystem/grpc-gateway
+	url = https://github.com/grpc-ecosystem/grpc-gateway
+[submodule "vendor/gopkg.in/fsnotify.v1"]
+	path = vendor/gopkg.in/fsnotify.v1
+	url = https://gopkg.in/fsnotify.v1
+[submodule "vendor/github.com/purpleidea/go-systemd"]
+	path = vendor/github.com/purpleidea/go-systemd
+	url = https://github.com/purpleidea/go-systemd

.travis.yml

@@ -1,17 +1,18 @@
 language: go
 go:
-  - 1.4.3
-  - 1.5.3
   - 1.6
+  - 1.7
   - tip
-sudo: false
+sudo: true
+dist: trusty
 before_install: 'git fetch --unshallow'
 install: 'make deps'
 script: 'make test'
 matrix:
+  fast_finish: true
   allow_failures:
     - go: tip
-    - go: 1.4.3
+    - go: 1.7
 notifications:
   irc:
     channels:

AUTHORS

@@ -5,3 +5,4 @@ For a more exhaustive list please run: git log --format='%aN' | sort -u
 This list is sorted alphabetically by first name.
 
 James Shubin
+Paul Morgan

DOCUMENTATION.md

@@ -1,198 +0,0 @@
-#mgmt
-
-<!--
-Mgmt
-Copyright (C) 2013-2016+ James Shubin and the project contributors
-Written by James Shubin <james@shubin.ca> and the project contributors
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
--->
-
-##mgmt by [James](https://ttboj.wordpress.com/)
-####Available from:
-####[https://github.com/purpleidea/mgmt/](https://github.com/purpleidea/mgmt/)
-
-####This documentation is available in: [Markdown](https://github.com/purpleidea/mgmt/blob/master/DOCUMENTATION.md) or [PDF](https://pdfdoc-purpleidea.rhcloud.com/pdf/https://github.com/purpleidea/mgmt/blob/master/DOCUMENTATION.md) format.
-
-####Table of Contents
-
-1. [Overview](#overview)
-2. [Project description - What the project does](#project-description)
-3. [Setup - Getting started with mgmt](#setup)
-4. [Features - All things mgmt can do](#features)
-	* [Autoedges - Automatic resource relationships](#autoedges)
-	* [Autogrouping - Automatic resource grouping](#autogrouping)
-5. [Usage/FAQ - Notes on usage and frequently asked questions](#usage-and-frequently-asked-questions)
-6. [Reference - Detailed reference](#reference)
-	* [Graph definition file](#graph-definition-file)
-	* [Command line](#command-line)
-7. [Examples - Example configurations](#examples)
-8. [Development - Background on module development and reporting bugs](#development)
-9. [Authors - Authors and contact information](#authors)
-
-##Overview
-
-The `mgmt` tool is a research prototype to demonstrate next generation config
-management techniques. Hopefully it will evolve into a useful, robust tool.
-
-##Project Description
-
-The mgmt tool is a distributed, event driven, config management tool, that
-supports parallel execution, and librarification to be used as the management
-foundation in and for, new and existing software.
-
-For more information, you may like to read some blog posts from the author:
-
-* [Next generation config mgmt](https://ttboj.wordpress.com/2016/01/18/next-generation-configuration-mgmt/)
-* [Automatic edges in mgmt](https://ttboj.wordpress.com/2016/03/14/automatic-edges-in-mgmt/)
-
-There is also an [introductory video](https://www.youtube.com/watch?v=GVhpPF0j-iE&html5=1) available.
-
-##Setup
-
-During this prototype phase, the tool can be run out of the source directory.
-You'll probably want to use ```./run.sh run --file examples/graph1.yaml``` to
-get started. Beware that this _can_ cause data loss. Understand what you're
-doing first, or perform these actions in a virtual environment such as the one
-provided by [Oh-My-Vagrant](https://github.com/purpleidea/oh-my-vagrant).
-
-##Features
-
-This section details the numerous features of mgmt and some caveats you might
-need to be aware of.
-
-###Autoedges
-
-Automatic edges, or AutoEdges, is the mechanism in mgmt by which it will
-automatically create dependencies for you between resources. For example,
-since mgmt can discover which files are installed by a package it will
-automatically ensure that any file resource you declare that matches a
-file installed by your package resource will only be processed after the
-package is installed.
-
-####Controlling autodeges
-
-Though autoedges is likely to be very helpful and avoid you having to declare
-all dependencies explicitly, there are cases where this behaviour is
-undesirable.
-
-Some distributions allow package installations to automatically start the
-service they ship. This can be problematic in the case of packages like MySQL
-as there are configuration options that need to be set before MySQL is ever
-started for the first time (or you'll need to wipe the data directory). In
-order to handle this situation you can disable autoedges per resource and
-explicitly declare that you want `my.cnf` to be written to disk before the
-installation of the `mysql-server` package.
-
-You can disable autoedges for a resource by setting the `autoedge` key on
-the meta attributes of that resource to `false`.
-
-###Autogrouping
-
-Automatic grouping or AutoGroup is the mechanism in mgmt by which it will
-automatically group multiple resource vertices into a single one. This is
-particularly useful for grouping multiple package resources into a single
-resource, since the multiple installations can happen together in a single
-transaction, which saves a lot of time because package resources typically have
-a large fixed cost to running (downloading and verifying the package repo) and
-if they are grouped they share this fixed cost. This grouping feature can be
-used for other use cases too.
-
-You can disable autogrouping for a resource by setting the `autogroup` key on
-the meta attributes of that resource to `false`.
-
-##Usage and frequently asked questions
-(Send your questions as a patch to this FAQ! I'll review it, merge it, and
-respond by commit with the answer.)
-
-###Why did you start this project?
-
-I wanted a next generation config management solution that didn't have all of
-the design flaws or limitations that the current generation of tools do, and no
-tool existed!
-
-###Why did you use etcd? What about consul?
-
-Etcd and consul are both written in golang, which made them the top two
-contenders for my prototype. Ultimately a choice had to be made, and etcd was
-chosen, but it was also somewhat arbitrary. If there is available interest,
-good reasoning, *and* patches, then we would consider either switching or
-supporting both, but this is not a high priority at this time.
-
-###You didn't answer my question, or I have a question!
-
-It's best to ask on [IRC](https://webchat.freenode.net/?channels=#mgmtconfig)
-to see if someone can help you. Once we get a big enough community going, we'll
-add a mailing list. If you don't get any response from the above, you can
-contact me through my [technical blog](https://ttboj.wordpress.com/contact/)
-and I'll do my best to help. If you have a good question, please add it as a
-patch to this documentation. I'll merge your question, and add a patch with the
-answer!
-
-##Reference
-Please note that there are a number of undocumented options. For more
-information on these options, please view the source at:
-[https://github.com/purpleidea/mgmt/](https://github.com/purpleidea/mgmt/).
-If you feel that a well used option needs documenting here, please patch it!
-
-###Overview of reference
-* [Graph definition file](#graph-definition-file): Main graph definition file.
-* [Command line](#command-line): Command line parameters.
-
-###Graph definition file
-graph.yaml is the compiled graph definition file. The format is currently
-undocumented, but by looking through the [examples/](https://github.com/purpleidea/mgmt/tree/master/examples)
-you can probably figure out most of it, as it's fairly intuitive.
-
-###Command line
-The main interface to the `mgmt` tool is the command line. For the most recent
-documentation, please run `mgmt --help`.
-
-####`--file <graph.yaml>`
-Point to a graph file to run.
-
-####`--converged-timeout <seconds>`
-Exit if the machine has converged for approximately this many seconds.
-
-####`--max-runtime <seconds>`
-Exit when the agent has run for approximately this many seconds. This is not
-generally recommended, but may be useful for users who know what they're doing.
-
-##Examples
-For example configurations, please consult the [examples/](https://github.com/purpleidea/mgmt/tree/master/examples) directory in the git
-source repository. It is available from:
-
-[https://github.com/purpleidea/mgmt/tree/master/examples](https://github.com/purpleidea/mgmt/tree/master/examples)
-
-##Development
-
-This is a project that I started in my free time in 2013. Development is driven
-by all of our collective patches! Dive right in, and start hacking!
-Please contact me if you'd like to invite me to speak about this at your event.
-
-You can follow along [on my technical blog](https://ttboj.wordpress.com/).
-
-To report any bugs, please file a ticket at: [https://github.com/purpleidea/mgmt/issues](https://github.com/purpleidea/mgmt/issues).
-
-##Authors
-
-Copyright (C) 2013-2016+ James Shubin and the project contributors
-
-Please see the
-[AUTHORS](https://github.com/purpleidea/mgmt/tree/master/AUTHORS) file
-for more information.
-
-* [github](https://github.com/purpleidea/)
-* [&#64;purpleidea](https://twitter.com/#!/purpleidea)
-* [https://ttboj.wordpress.com/](https://ttboj.wordpress.com/)

Makefile

@@ -15,12 +15,12 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-SHELL = /bin/bash
-.PHONY: all version program path deps run race build clean test format docs rpmbuild mkdirs rpm srpm spec tar upload upload-sources upload-srpms upload-rpms copr
+SHELL = /usr/bin/env bash
+.PHONY: all art cleanart version program path deps run race generate build clean test gofmt yamlfmt format docs rpmbuild mkdirs rpm srpm spec tar upload upload-sources upload-srpms upload-rpms copr
 .SILENT: clean
 
-SVERSION := $(shell git describe --match '[0-9]*\.[0-9]*\.[0-9]*' --tags --dirty --always)
-VERSION := $(shell git describe --match '[0-9]*\.[0-9]*\.[0-9]*' --tags --abbrev=0)
+SVERSION := $(or $(SVERSION),$(shell git describe --match '[0-9]*\.[0-9]*\.[0-9]*' --tags --dirty --always))
+VERSION := $(or $(VERSION),$(shell git describe --match '[0-9]*\.[0-9]*\.[0-9]*' --tags --abbrev=0))
 PROGRAM := $(shell echo $(notdir $(CURDIR)) | cut -f1 -d"-")
 OLDGOLANG := $(shell go version | grep -E 'go1.3|go1.4')
 ifeq ($(VERSION),$(SVERSION))
@@ -28,7 +28,7 @@ ifeq ($(VERSION),$(SVERSION))
 else
 	RELEASE = untagged
 endif
-ARCH = $(shell arch)
+ARCH = $(uname -m)
 SPEC = rpmbuild/SPECS/$(PROGRAM).spec
 SOURCE = rpmbuild/SOURCES/$(PROGRAM)-$(VERSION).tar.bz2
 SRPM = rpmbuild/SRPMS/$(PROGRAM)-$(VERSION)-$(RELEASE).src.rpm
@@ -38,7 +38,43 @@ USERNAME := $(shell cat ~/.config/copr 2>/dev/null | grep username | awk -F '='
 SERVER = 'dl.fedoraproject.org'
 REMOTE_PATH = 'pub/alt/$(USERNAME)/$(PROGRAM)'
 
-all: docs
+#
+#	art
+#
+art: art/mgmt_logo_default_symbol.png art/mgmt_logo_default_tall.png art/mgmt_logo_default_wide.png art/mgmt_logo_reversed_symbol.png art/mgmt_logo_reversed_tall.png art/mgmt_logo_reversed_wide.png art/mgmt_logo_white_symbol.png art/mgmt_logo_white_tall.png art/mgmt_logo_white_wide.png
+
+cleanart:
+	rm -f art/mgmt_logo_default_symbol.png art/mgmt_logo_default_tall.png art/mgmt_logo_default_wide.png art/mgmt_logo_reversed_symbol.png art/mgmt_logo_reversed_tall.png art/mgmt_logo_reversed_wide.png art/mgmt_logo_white_symbol.png art/mgmt_logo_white_tall.png art/mgmt_logo_white_wide.png
+
+# NOTE: the widths are arbitrary
+art/mgmt_logo_default_symbol.png: art/mgmt_logo_default_symbol.svg
+	inkscape --export-background='#ffffff' --without-gui --export-png "$@" --export-width 300 $(@:png=svg)
+
+art/mgmt_logo_default_tall.png: art/mgmt_logo_default_tall.svg
+	inkscape --export-background='#ffffff' --without-gui --export-png "$@" --export-width 400 $(@:png=svg)
+
+art/mgmt_logo_default_wide.png: art/mgmt_logo_default_wide.svg
+	inkscape --export-background='#ffffff' --without-gui --export-png "$@" --export-width 800 $(@:png=svg)
+
+art/mgmt_logo_reversed_symbol.png: art/mgmt_logo_reversed_symbol.svg
+	inkscape --export-background='#231f20' --without-gui --export-png "$@" --export-width 300 $(@:png=svg)
+
+art/mgmt_logo_reversed_tall.png: art/mgmt_logo_reversed_tall.svg
+	inkscape --export-background='#231f20' --without-gui --export-png "$@" --export-width 400 $(@:png=svg)
+
+art/mgmt_logo_reversed_wide.png: art/mgmt_logo_reversed_wide.svg
+	inkscape --export-background='#231f20' --without-gui --export-png "$@" --export-width 800 $(@:png=svg)
+
+art/mgmt_logo_white_symbol.png: art/mgmt_logo_white_symbol.svg
+	inkscape --export-background='#231f20' --without-gui --export-png "$@" --export-width 300 $(@:png=svg)
+
+art/mgmt_logo_white_tall.png: art/mgmt_logo_white_tall.svg
+	inkscape --export-background='#231f20' --without-gui --export-png "$@" --export-width 400 $(@:png=svg)
+
+art/mgmt_logo_white_wide.png: art/mgmt_logo_white_wide.svg
+	inkscape --export-background='#231f20' --without-gui --export-png "$@" --export-width 800 $(@:png=svg)
+
+all: docs $(PROGRAM).static
 
 # show the current version
 version:
@@ -54,39 +90,56 @@ deps:
 	./misc/make-deps.sh
 
 run:
-	find -maxdepth 1 -type f -name '*.go' -not -name '*_test.go' | xargs go run -ldflags "-X main.program=$(PROGRAM) -X main.version=$(SVERSION)"
+	find . -maxdepth 1 -type f -name '*.go' -not -name '*_test.go' | xargs go run -ldflags "-X main.program=$(PROGRAM) -X main.version=$(SVERSION)"
 
 # include race flag
 race:
-	find -maxdepth 1 -type f -name '*.go' -not -name '*_test.go' | xargs go run -race -ldflags "-X main.program=$(PROGRAM) -X main.version=$(SVERSION)"
+	find . -maxdepth 1 -type f -name '*.go' -not -name '*_test.go' | xargs go run -race -ldflags "-X main.program=$(PROGRAM) -X main.version=$(SVERSION)"
+
+generate:
+	go generate
 
 build: $(PROGRAM)
 
 $(PROGRAM): main.go
 	@echo "Building: $(PROGRAM), version: $(SVERSION)..."
+ifneq ($(OLDGOLANG),)
+	@# avoid equals sign in old golang versions eg in: -X foo=bar
+	time go build -ldflags "-X main.program $(PROGRAM) -X main.version $(SVERSION)" -o $(PROGRAM);
+else
+	time go build -ldflags "-X main.program=$(PROGRAM) -X main.version=$(SVERSION)" -o $(PROGRAM);
+endif
+
+$(PROGRAM).static: main.go
+	@echo "Building: $(PROGRAM).static, version: $(SVERSION)..."
 	go generate
 ifneq ($(OLDGOLANG),)
 	@# avoid equals sign in old golang versions eg in: -X foo=bar
-	go build -ldflags "-X main.program $(PROGRAM) -X main.version $(SVERSION)" -o $(PROGRAM);
+	go build -a -installsuffix cgo -tags netgo -ldflags '-extldflags "-static" -X main.program $(PROGRAM) -X main.version $(SVERSION)' -o $(PROGRAM).static;
 else
-	go build -ldflags "-X main.program=$(PROGRAM) -X main.version=$(SVERSION)" -o $(PROGRAM);
+	go build -a -installsuffix cgo -tags netgo -ldflags '-extldflags "-static" -X main.program=$(PROGRAM) -X main.version=$(SVERSION)' -o $(PROGRAM).static;
 endif
 
 clean:
 	[ ! -e $(PROGRAM) ] || rm $(PROGRAM)
 	rm -f *_stringer.go	# generated by `go generate`
+	rm -f *_mock.go		# generated by `go generate`
 
 test:
 	./test.sh
 
-format:
-	find -type f -name '*.go' -not -path './old/*' -not -path './tmp/*' -exec gofmt -w {} \;
-	find -type f -name '*.yaml' -not -path './old/*' -not -path './tmp/*' -not -path './omv.yaml' -exec ruby -e "require 'yaml'; x=YAML.load_file('{}').to_yaml.each_line.map(&:rstrip).join(10.chr)+10.chr; File.open('{}', 'w').write x" \;
+gofmt:
+	find . -maxdepth 3 -type f -name '*.go' -not -path './old/*' -not -path './tmp/*' -exec gofmt -w {} \;
+
+yamlfmt:
+	find . -maxdepth 3 -type f -name '*.yaml' -not -path './old/*' -not -path './tmp/*' -not -path './omv.yaml' -exec ruby -e "require 'yaml'; x=YAML.load_file('{}').to_yaml.each_line.map(&:rstrip).join(10.chr)+10.chr; File.open('{}', 'w').write x" \;
+
+format: gofmt yamlfmt
 
 docs: $(PROGRAM)-documentation.pdf
 
-$(PROGRAM)-documentation.pdf: DOCUMENTATION.md
-	pandoc DOCUMENTATION.md -o '$(PROGRAM)-documentation.pdf'
+$(PROGRAM)-documentation.pdf: docs/documentation.md
+	pandoc docs/documentation.md -o docs/'$(PROGRAM)-documentation.pdf'
 
 #
 #	build aliases
@@ -131,7 +184,7 @@ $(SRPM): $(SPEC) $(SOURCE)
 $(SPEC): rpmbuild/ spec.in
 	@echo Running templater...
 	#cat spec.in > $(SPEC)
-	sed -e s/__PROGRAM__/$(PROGRAM)/ -e s/__VERSION__/$(VERSION)/ -e s/__RELEASE__/$(RELEASE)/ < spec.in > $(SPEC)
+	sed -e s/__PROGRAM__/$(PROGRAM)/g -e s/__VERSION__/$(VERSION)/g -e s/__RELEASE__/$(RELEASE)/g < spec.in > $(SPEC)
 	# append a changelog to the .spec file
 	git log --format="* %cd %aN <%aE>%n- (%h) %s%d%n" --date=local | sed -r 's/[0-9]+:[0-9]+:[0-9]+ //' >> $(SPEC)
 

README.md

@@ -1,24 +1,47 @@
-# *mgmt*: This is: mgmt!
+# *mgmt*: next generation config management!
 
+[![mgmt!](art/mgmt.png)](art/)
+
+[![Go Report Card](https://goreportcard.com/badge/github.com/purpleidea/mgmt)](https://goreportcard.com/report/github.com/purpleidea/mgmt)
 [![Build Status](https://secure.travis-ci.org/purpleidea/mgmt.png?branch=master)](http://travis-ci.org/purpleidea/mgmt)
-[![Documentation](https://img.shields.io/docs/markdown.png)](DOCUMENTATION.md)
+[![Documentation](https://img.shields.io/docs/markdown.png)](docs/documentation.md)
+[![GoDoc](https://godoc.org/github.com/purpleidea/mgmt?status.svg)](https://godoc.org/github.com/purpleidea/mgmt)
 [![IRC](https://img.shields.io/irc/%23mgmtconfig.png)](https://webchat.freenode.net/?channels=#mgmtconfig)
 [![Jenkins](https://img.shields.io/jenkins/status.png)](https://ci.centos.org/job/purpleidea-mgmt/)
 [![COPR](https://img.shields.io/copr/builds.png)](https://copr.fedoraproject.org/coprs/purpleidea/mgmt/)
+[![arch](https://img.shields.io/arch/aur.png)](https://aur.archlinux.org/packages/mgmt/)
 
 ## Community:
 Come join us on IRC in [#mgmtconfig](https://webchat.freenode.net/?channels=#mgmtconfig) on Freenode!
 You may like the [#mgmtconfig](https://twitter.com/hashtag/mgmtconfig) hashtag if you're on [Twitter](https://twitter.com/#!/purpleidea).
 
+## Status:
+Mgmt is a fairly new project.
+We're working towards being minimally useful for production environments.
+We aren't feature complete for what we'd consider a 1.x release yet.
+With your help you'll be able to influence our design and get us there sooner!
+
 ## Questions:
 Please join the [#mgmtconfig](https://webchat.freenode.net/?channels=#mgmtconfig) IRC community!
-If you have a well phrased question that might benefit others, consider asking it by sending a patch to the documentation [FAQ](https://github.com/purpleidea/mgmt/blob/master/DOCUMENTATION.md#usage-and-frequently-asked-questions) section. I'll merge your question, and a patch with the answer!
+If you have a well phrased question that might benefit others, consider asking it by sending a patch to the documentation [FAQ](https://github.com/purpleidea/mgmt/blob/master/docs/documentation.md#usage-and-frequently-asked-questions) section. I'll merge your question, and a patch with the answer!
 
 ## Quick start:
-* Either get the golang dependencies on your own, or run `make deps` if you're comfortable with how we install them.
+* Make sure you have golang version 1.6 or greater installed.
+* If you do not have a GOPATH yet, create one and export it:
+```
+mkdir $HOME/gopath
+export GOPATH=$HOME/gopath
+```
+* You might also want to add the GOPATH to your `~/.bashrc` or `~/.profile`.
+* For more information you can read the [GOPATH documentation](https://golang.org/cmd/go/#hdr-GOPATH_environment_variable).
+* Next download the mgmt code base, and switch to that directory:
+```
+go get -u github.com/purpleidea/mgmt
+cd $GOPATH/src/github.com/purpleidea/mgmt
+```
+* Get the remaining golang deps with `go get ./...`, or run `make deps` if you're comfortable with how we install them.
 * Run `make build` to get a freshly built `mgmt` binary.
-* Run `cd $(mktemp --tmpdir -d tmp.XXX) && etcd` to get etcd running. The `mgmt` software will do this automatically for you in the future.
-* Run `time ./mgmt run --file examples/graph0.yaml --converged-timeout=1` to try out a very simple example!
+* Run `time ./mgmt run --yaml examples/graph0.yaml --converged-timeout=5 --tmp-prefix` to try out a very simple example!
 * To run continuously in the default mode of operation, omit the `--converged-timeout` option.
 * Have fun hacking on our future technology!
 
@@ -26,7 +49,7 @@ If you have a well phrased question that might benefit others, consider asking i
 Please look in the [examples/](examples/) folder for more examples!
 
 ## Documentation:
-Please see: [DOCUMENTATION.md](DOCUMENTATION.md) or [PDF](https://pdfdoc-purpleidea.rhcloud.com/pdf/https://github.com/purpleidea/mgmt/blob/master/DOCUMENTATION.md).
+Please see: the manually created [documentation.md](docs/documentation.md) (also available as [PDF](https://pdfdoc-purpleidea.rhcloud.com/pdf/https://github.com/purpleidea/mgmt/blob/master/docs/documentation.md)) and the automatically generated [GoDoc documentation](https://godoc.org/github.com/purpleidea/mgmt).
 
 ## Roadmap:
 Please see: [TODO.md](TODO.md) for a list of upcoming work and TODO items.
@@ -39,20 +62,22 @@ Bonus points if you provide a [shell](https://github.com/purpleidea/mgmt/tree/ma
 Feel free to read my article on [debugging golang programs](https://ttboj.wordpress.com/2016/02/15/debugging-golang-programs/).
 
 ## Dependencies:
-* golang 1.4 or higher (required, available in most distros)
+* golang 1.6 or higher (required, available in most distros)
 * golang libraries (required, available with `go get`)
-
-        go get github.com/coreos/etcd/client
-        go get gopkg.in/yaml.v2
-        go get gopkg.in/fsnotify.v1
-        go get github.com/codegangsta/cli
-        go get github.com/coreos/go-systemd/dbus
-        go get github.com/coreos/go-systemd/util
-
-* stringer (required for building), available as a package on some platforms, otherwise via `go get`
-
-        go get golang.org/x/tools/cmd/stringer
-
+```
+go get github.com/coreos/etcd/client
+go get gopkg.in/yaml.v2
+go get gopkg.in/fsnotify.v1
+go get github.com/urfave/cli
+go get github.com/coreos/go-systemd/dbus
+go get github.com/coreos/go-systemd/util
+go get github.com/coreos/pkg/capnslog
+go get github.com/rgbkrk/libvirt-go
+```
+* stringer (optional for building), available as a package on some platforms, otherwise via `go get`
+```
+go get golang.org/x/tools/cmd/stringer
+```
 * pandoc (optional, for building a pdf of the documentation)
 * graphviz (optional, for building a visual representation of the graph)
 
@@ -60,14 +85,28 @@ Feel free to read my article on [debugging golang programs](https://ttboj.wordpr
 We'd love to have your patches! Please send them by email, or as a pull request.
 
 ## On the web:
-* Introductory blog post: [https://ttboj.wordpress.com/2016/01/18/next-generation-configuration-mgmt/](https://ttboj.wordpress.com/2016/01/18/next-generation-configuration-mgmt/)
-* Introductory recording from DevConf.cz 2016: [https://www.youtube.com/watch?v=GVhpPF0j-iE&html5=1](https://www.youtube.com/watch?v=GVhpPF0j-iE&html5=1)
-* Introductory recording from CfgMgmtCamp.eu 2016: [https://www.youtube.com/watch?v=fNeooSiIRnA&html5=1](https://www.youtube.com/watch?v=fNeooSiIRnA&html5=1)
-* Julian Dunn at CfgMgmtCamp.eu 2016: [https://www.youtube.com/watch?v=kfF9IATUask&t=1949&html5=1](https://www.youtube.com/watch?v=kfF9IATUask&t=1949&html5=1)
-* Walter Heck at CfgMgmtCamp.eu 2016: [http://www.slideshare.net/olindata/configuration-management-time-for-a-4th-generation/3](http://www.slideshare.net/olindata/configuration-management-time-for-a-4th-generation/3)
-* Marco Marongiu on mgmt: [http://syslog.me/2016/02/15/leap-or-die/](http://syslog.me/2016/02/15/leap-or-die/)
-* Felix Frank on puppet to mgmt "transpiling" [https://ffrank.github.io/features/2016/02/18/from-catalog-to-mgmt/](https://ffrank.github.io/features/2016/02/18/from-catalog-to-mgmt/)
-* Blog post on automatic edges and the pkg resource: [https://ttboj.wordpress.com/2016/03/14/automatic-edges-in-mgmt/](https://ttboj.wordpress.com/2016/03/14/automatic-edges-in-mgmt/)
+* James Shubin; blog: [Next generation configuration mgmt](https://ttboj.wordpress.com/2016/01/18/next-generation-configuration-mgmt/)
+* James Shubin; video: [Introductory recording from DevConf.cz 2016](https://www.youtube.com/watch?v=GVhpPF0j-iE&html5=1)
+* James Shubin; video: [Introductory recording from CfgMgmtCamp.eu 2016](https://www.youtube.com/watch?v=fNeooSiIRnA&html5=1)
+* Julian Dunn; video: [On mgmt at CfgMgmtCamp.eu 2016](https://www.youtube.com/watch?v=kfF9IATUask&t=1949&html5=1)
+* Walter Heck; slides: [On mgmt at CfgMgmtCamp.eu 2016](http://www.slideshare.net/olindata/configuration-management-time-for-a-4th-generation/3)
+* Marco Marongiu; blog: [On mgmt](http://syslog.me/2016/02/15/leap-or-die/)
+* Felix Frank; blog: [From Catalog To Mgmt (on puppet to mgmt "transpiling")](https://ffrank.github.io/features/2016/02/18/from-catalog-to-mgmt/)
+* James Shubin; blog: [Automatic edges in mgmt (...and the pkg resource)](https://ttboj.wordpress.com/2016/03/14/automatic-edges-in-mgmt/)
+* James Shubin; blog: [Automatic grouping in mgmt](https://ttboj.wordpress.com/2016/03/30/automatic-grouping-in-mgmt/)
+* John Arundel; tweet: [“Puppet’s days are numbered.”](https://twitter.com/bitfield/status/732157519142002688)
+* Felix Frank; blog: [Puppet, Meet Mgmt (on puppet to mgmt internals)](https://ffrank.github.io/features/2016/06/12/puppet,-meet-mgmt/)
+* Felix Frank; blog: [Puppet Powered Mgmt (puppet to mgmt tl;dr)](https://ffrank.github.io/features/2016/06/19/puppet-powered-mgmt/)
+* James Shubin; blog: [Automatic clustering in mgmt](https://ttboj.wordpress.com/2016/06/20/automatic-clustering-in-mgmt/)
+* James Shubin; video: [Recording from CoreOSFest 2016](https://www.youtube.com/watch?v=KVmDCUA42wc&html5=1)
+* James Shubin; video: [Recording from DebConf16](http://meetings-archive.debian.net/pub/debian-meetings/2016/debconf16/Next_Generation_Config_Mgmt.webm) ([Slides](https://annex.debconf.org//debconf-share/debconf16/slides/15-next-generation-config-mgmt.pdf))
+* Felix Frank; blog: [Edging It All In (puppet and mgmt edges)](https://ffrank.github.io/features/2016/07/12/edging-it-all-in/)
+* Felix Frank; blog: [Translating All The Things (puppet to mgmt translation warnings)](https://ffrank.github.io/features/2016/08/19/translating-all-the-things/)
+* James Shubin; video: [Recording from systemd.conf 2016](https://www.youtube.com/watch?v=jB992Zb3nH0&html5=1)
+* James Shubin; blog: [Remote execution in mgmt](https://ttboj.wordpress.com/2016/10/07/remote-execution-in-mgmt/)
+* James Shubin; video: [Recording from High Load Strategy 2016](https://vimeo.com/191493409)
+* James Shubin; video: [Recording from NLUUG 2016](https://www.youtube.com/watch?v=MmpwOQAb_SE&html5=1)
+* James Shubin; blog: [Send/Recv in mgmt](https://ttboj.wordpress.com/2016/12/07/sendrecv-in-mgmt/)
 
 ##
 

THANKS

@@ -9,10 +9,16 @@ Chris Wright - For encouraging me to continue work on my prototype.
 
 Daniel Riek - For supporting and sheltering this project from bureaucracy.
 
+Diego Ongaro - For good chats, particularly around distributed systems.
+
+Felix Frank - For taking a difficult problem and building an inspiring solution.
+
 Ira Cooper - For having an algorithmic design discussion with me.
 
 Jeff Darcy - For some algorithm recommendations, and NACKing my TopoSort idea!
 
 Red Hat, inc. - For paying my salary, thus financially supporting my hacking.
 
+Samuel Gélineau - For help with programming language theory and design.
+
 And many others...

TODO.md

@@ -3,27 +3,54 @@ If you're looking for something to do, look here!
 Let us know if you're working on one of the items.
 
 ## Package resource
-- [ ] base resource [bug](https://github.com/purpleidea/mgmt/issues/11)
+- [ ] getfiles support on debian [bug](https://github.com/hughsie/PackageKit/issues/118)
+- [ ] directory info on fedora [bug](https://github.com/hughsie/PackageKit/issues/117)
 - [ ] dnf blocker [bug](https://github.com/hughsie/PackageKit/issues/110)
 - [ ] install signal blocker [bug](https://github.com/hughsie/PackageKit/issues/109)
 
 ## File resource [bug](https://github.com/purpleidea/mgmt/issues/13) [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
-- [ ] ability to make/delete folders
-- [ ] recursive argument (can recursively watch/modify contents)
-- [ ] force argument (can cause switch from file <-> folder)
+- [ ] chown/chmod support [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
+- [ ] user/group support [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
+- [ ] recurse limit support [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
 - [ ] fanotify support [bug](https://github.com/go-fsnotify/fsnotify/issues/114)
 
+## Svc resource
+- [ ] base resource improvements
+
 ## Exec resource
 - [ ] base resource improvements
 
 ## Timer resource
-- [ ] base resource [bug](https://github.com/purpleidea/mgmt/issues/15) [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
 - [ ] reset on recompile
-- [ ] increment algorithm (linear, exponential, etc...)
+- [ ] increment algorithm (linear, exponential, etc...) [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
+
+## User/Group resource
+- [ ] base resource [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
+- [ ] automatic edges to file resource [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
+
+## Virt (libvirt) resource
+- [ ] base resource [bug](https://github.com/purpleidea/mgmt/issues/25)
+
+## Net (systemd-networkd) resource
+- [ ] base resource
+
+## Nspawn (systemd-nspawn) resource
+- [ ] base resource [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
+
+## Mount (systemd-mount) resource
+- [ ] base resource [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
+
+## Cron (systemd-timer) resource
+- [ ] base resource [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
+
+## Http resource
+- [ ] base resource
 
 ## Etcd improvements
-- [ ] embedded etcd master
-- [ ] capnslog fixes [bug](https://github.com/coreos/etcd/issues/4115)
+- [ ] fix embedded etcd master race
+
+## Torrent/dht file transfer
+- [ ] base plumbing
 
 ## Language improvements
 - [ ] language design
@@ -35,9 +62,6 @@ Let us know if you're working on one of the items.
 
 ## Other
 - [ ] better error/retry handling
-- [ ] resource grouping
-- [ ] automatic dependency adding (eg: packagekit file dependencies)
-- [ ] mgmt systemd service file [bug](https://github.com/purpleidea/mgmt/issues/12) [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
 - [ ] deb package target in Makefile
 - [ ] reproducible builds
 - [ ] add your suggestions!

art/.gitignore

@@ -0,0 +1,2 @@
+*.png
+misc/

art/mgmt_logo_default_symbol.svg

@@ -0,0 +1,94 @@
+<?xml version="1.0" encoding="utf-8"?>
+<svg version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 120 107.1" style="enable-background:new 0 0 120 107.1;" xml:space="preserve">
+<style type="text/css">
+	.st0{fill:#FFFFFF;}
+	.st1{fill:#E22434;}
+	.st2{display:none;}
+	.st3{fill:#1B3663;}
+	.st4{fill:#00B1D1;}
+	.st5{fill:#BFE6EF;}
+	.st6{fill:#69CBE0;}
+	.st7{fill:#0080BD;}
+	.st8{fill:#005DAB;}
+	.st9{fill:#183660;}
+	.st10{fill:none;stroke:#183660;stroke-width:1.9441;stroke-miterlimit:10;}
+	.st11{fill:none;stroke:#183660;stroke-width:1.6201;stroke-miterlimit:10;}
+	.st12{fill:none;stroke:#183660;stroke-width:1.2961;stroke-miterlimit:10;}
+	.st13{fill:none;stroke:#C0E6EF;stroke-width:1.9441;stroke-miterlimit:10;}
+	.st14{fill:#C0E6EF;}
+	.st15{fill:none;stroke:#C0E6EF;stroke-width:1.2961;stroke-miterlimit:10;}
+	.st16{fill:none;stroke:#C0E6EF;stroke-width:1.6201;stroke-miterlimit:10;}
+	.st17{fill:none;stroke:#FFFFFF;stroke-width:1.9441;stroke-miterlimit:10;}
+	.st18{fill:none;stroke:#FFFFFF;stroke-width:1.2961;stroke-miterlimit:10;}
+	.st19{fill:none;stroke:#FFFFFF;stroke-width:1.6201;stroke-miterlimit:10;}
+</style>
+<g id="Layer_2">
+</g>
+<g id="Layer_1">
+	<g>
+		<g>
+			<g>
+				<line class="st10" x1="29.2" y1="24.1" x2="52.1" y2="42.8"/>
+				<g>
+					<polygon class="st9" points="27.7,27.8 24.9,20.5 32.6,21.8 					"/>
+				</g>
+			</g>
+		</g>
+		<circle class="st5" cx="16.1" cy="12.2" r="12.1"/>
+		<g>
+			<g>
+				<line class="st10" x1="52.1" y1="42.1" x2="74.1" y2="80"/>
+				<g>
+					<polygon class="st9" points="70.1,80.9 76.9,84.8 76.8,77.1 					"/>
+				</g>
+			</g>
+		</g>
+		<g>
+			<g>
+				<line class="st10" x1="69.4" y1="46.7" x2="95.8" y2="52.4"/>
+				<g>
+					<polygon class="st9" points="69.7,50.7 63.9,45.6 71.3,43.2 					"/>
+				</g>
+			</g>
+		</g>
+		<g>
+			<g>
+				<line class="st10" x1="52.1" y1="42.8" x2="71.9" y2="30.1"/>
+				<g>
+					<polygon class="st9" points="73.1,34 76.6,27.1 68.9,27.5 					"/>
+				</g>
+			</g>
+		</g>
+		<g>
+			<g>
+				<line class="st10" x1="16.8" y1="49.6" x2="34.8" y2="46.5"/>
+				<g>
+					<polygon class="st9" points="34.3,50.5 40.3,45.6 33,42.9 					"/>
+				</g>
+			</g>
+		</g>
+		<g>
+			<g>
+				<line class="st10" x1="92.3" y1="79.5" x2="107.8" y2="54.2"/>
+				<g>
+					<polygon class="st9" points="96.1,80.6 89.3,84.3 89.5,76.5 					"/>
+				</g>
+			</g>
+		</g>
+		<g>
+			<g>
+				<line class="st10" x1="97.3" y1="36.5" x2="107.8" y2="54.2"/>
+				<g>
+					<polygon class="st9" points="94.6,39.4 94.5,31.7 101.2,35.5 					"/>
+				</g>
+			</g>
+		</g>
+		<circle class="st3" cx="52.1" cy="42.8" r="12.1"/>
+		<circle class="st4" cx="12.2" cy="50.8" r="12.1"/>
+		<circle class="st7" cx="87.5" cy="21.7" r="12.1"/>
+		<circle class="st8" cx="83.5" cy="95" r="12.1"/>
+		<circle class="st6" cx="107.8" cy="54.2" r="12.1"/>
+	</g>
+</g>
+</svg>

art/mgmt_logo_default_tall.svg

@@ -0,0 +1,132 @@
+<?xml version="1.0" encoding="utf-8"?>
+<svg version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 168.3 131.6" style="enable-background:new 0 0 168.3 131.6;" xml:space="preserve">
+<style type="text/css">
+	.st0{fill:#FFFFFF;}
+	.st1{fill:#E22434;}
+	.st2{display:none;}
+	.st3{fill:#1B3663;}
+	.st4{fill:#00B1D1;}
+	.st5{fill:#BFE6EF;}
+	.st6{fill:#69CBE0;}
+	.st7{fill:#0080BD;}
+	.st8{fill:#005DAB;}
+	.st9{fill:#183660;}
+	.st10{fill:none;stroke:#183660;stroke-width:1.9441;stroke-miterlimit:10;}
+	.st11{fill:none;stroke:#183660;stroke-width:1.6201;stroke-miterlimit:10;}
+	.st12{fill:none;stroke:#183660;stroke-width:1.2961;stroke-miterlimit:10;}
+	.st13{fill:none;stroke:#C0E6EF;stroke-width:1.9441;stroke-miterlimit:10;}
+	.st14{fill:#C0E6EF;}
+	.st15{fill:none;stroke:#C0E6EF;stroke-width:1.2961;stroke-miterlimit:10;}
+	.st16{fill:none;stroke:#C0E6EF;stroke-width:1.6201;stroke-miterlimit:10;}
+	.st17{fill:none;stroke:#FFFFFF;stroke-width:1.9441;stroke-miterlimit:10;}
+	.st18{fill:none;stroke:#FFFFFF;stroke-width:1.2961;stroke-miterlimit:10;}
+	.st19{fill:none;stroke:#FFFFFF;stroke-width:1.6201;stroke-miterlimit:10;}
+</style>
+<g id="Layer_2">
+</g>
+<g id="Layer_1">
+	<g>
+		<g>
+			<g>
+				<path class="st3" d="M4.7,105l0.1,1.8c1.1-1.4,2.6-2.1,4.4-2.1c1.9,0,3.2,0.9,4,2.6c1.1-1.7,2.6-2.6,4.7-2.6
+					c3.3,0,5,2.3,5.1,6.9V124h-5v-12.1c0-1.1-0.2-1.9-0.5-2.4c-0.3-0.5-0.8-0.7-1.5-0.7c-0.9,0-1.6,0.6-2.1,1.7l0,0.6V124H9v-12.1
+					c0-1.1-0.1-1.9-0.4-2.4c-0.3-0.5-0.8-0.7-1.6-0.7c-0.9,0-1.5,0.5-2,1.4V124H0v-19H4.7z"/>
+				<path class="st3" d="M26.4,113.9c0-3.1,0.6-5.4,1.7-7s2.7-2.3,4.7-2.3c1.7,0,3.1,0.7,4,2L37,105h4.5v19c0,2.4-0.7,4.3-2,5.6
+					c-1.4,1.3-3.3,1.9-5.9,1.9c-1,0-2.1-0.2-3.3-0.6s-2-0.9-2.6-1.6l1.7-3.4c0.5,0.5,1.1,0.9,1.8,1.2c0.7,0.3,1.5,0.5,2.1,0.5
+					c1.1,0,1.9-0.3,2.4-0.8s0.7-1.4,0.7-2.6v-1.6c-0.9,1.2-2.2,1.9-3.7,1.9c-2,0-3.6-0.8-4.7-2.4s-1.7-3.8-1.7-6.7V113.9z
+					 M31.4,115.2c0,1.8,0.2,3,0.7,3.8s1.2,1.2,2.2,1.2c1,0,1.8-0.4,2.3-1.1V110c-0.5-0.8-1.3-1.2-2.2-1.2c-1,0-1.7,0.4-2.2,1.2
+					s-0.7,2.1-0.7,3.9V115.2z"/>
+				<path class="st3" d="M50.1,105l0.1,1.8c1.1-1.4,2.6-2.1,4.4-2.1c1.9,0,3.2,0.9,4,2.6c1.1-1.7,2.6-2.6,4.7-2.6
+					c3.3,0,5,2.3,5.1,6.9V124h-5v-12.1c0-1.1-0.2-1.9-0.5-2.4s-0.8-0.7-1.5-0.7c-0.9,0-1.6,0.6-2.1,1.7l0,0.6V124h-5v-12.1
+					c0-1.1-0.1-1.9-0.4-2.4s-0.8-0.7-1.6-0.7c-0.9,0-1.5,0.5-2,1.4V124h-5v-19H50.1z"/>
+				<path class="st3" d="M78.2,100.3v4.7h2.5v3.7h-2.5v9.5c0,0.8,0.1,1.3,0.3,1.5c0.2,0.3,0.6,0.4,1.2,0.4c0.5,0,0.9,0,1.2-0.1
+					l0,3.9c-0.8,0.3-1.8,0.5-2.7,0.5c-3.2,0-4.8-1.8-4.9-5.5v-10.1h-2.2V105h2.2v-4.7H78.2z"/>
+				<path class="st4" d="M90.6,122.6c1.4,0,2.4-0.4,3.1-1.1c0.7-0.8,1.1-1.9,1.2-3.3h1.9c-0.1,1.9-0.7,3.5-1.9,4.6
+					c-1.1,1.1-2.6,1.7-4.3,1.7c-2.3,0-4-0.7-5.1-2.2s-1.7-3.6-1.8-6.4v-2.3c0-2.9,0.6-5.1,1.7-6.6s2.9-2.2,5.1-2.2
+					c1.9,0,3.4,0.6,4.5,1.8s1.7,2.9,1.7,5H95c-0.1-1.6-0.5-2.8-1.2-3.6s-1.8-1.3-3.1-1.3c-1.7,0-2.9,0.6-3.7,1.7s-1.2,2.9-1.2,5.2
+					v2.2c0,2.4,0.4,4.2,1.2,5.3S89,122.6,90.6,122.6z"/>
+				<path class="st4" d="M100.5,113.6c0-2.7,0.6-4.9,1.9-6.5s2.9-2.4,5.1-2.4c2.2,0,3.9,0.8,5.1,2.4s1.9,3.7,1.9,6.5v2
+					c0,2.8-0.6,5-1.9,6.5s-2.9,2.3-5.1,2.3c-2.1,0-3.8-0.8-5-2.3s-1.9-3.6-1.9-6.3V113.6z M102.5,115.5c0,2.2,0.4,3.9,1.3,5.2
+					c0.9,1.3,2.1,1.9,3.7,1.9c1.6,0,2.8-0.6,3.7-1.8s1.3-2.9,1.3-5.2v-2c0-2.2-0.4-3.9-1.3-5.2c-0.9-1.3-2.1-1.9-3.7-1.9
+					c-1.5,0-2.7,0.6-3.6,1.8s-1.3,2.9-1.4,5.1V115.5z"/>
+				<path class="st4" d="M121.1,105l0.1,3c0.6-1,1.3-1.9,2.2-2.5s1.9-0.9,3-0.9c3.3,0,5,2.2,5.1,6.6V124h-1.9v-12.5
+					c0-1.7-0.3-3-0.9-3.8c-0.6-0.8-1.5-1.2-2.8-1.2c-1,0-1.9,0.4-2.8,1.2s-1.4,1.8-1.9,3.2V124h-2v-19H121.1z"/>
+				<path class="st4" d="M138.2,124v-17.3h-2.6V105h2.6v-2.5c0-1.9,0.5-3.3,1.3-4.3s2-1.5,3.5-1.5c0.7,0,1.3,0.1,1.9,0.3l-0.1,1.8
+					c-0.5-0.1-1-0.2-1.6-0.2c-0.9,0-1.7,0.4-2.2,1.1s-0.8,1.7-0.8,3v2.3h3.7v1.8h-3.7V124H138.2z"/>
+				<path class="st4" d="M148,99.5c0-0.4,0.1-0.7,0.3-1s0.5-0.4,0.9-0.4s0.7,0.1,1,0.4s0.3,0.6,0.3,1s-0.1,0.7-0.3,1s-0.5,0.4-1,0.4
+					s-0.7-0.1-0.9-0.4S148,99.9,148,99.5z M150.2,124h-2v-19h2V124z"/>
+				<path class="st4" d="M155.3,113.6c0-3,0.5-5.2,1.5-6.7s2.6-2.2,4.6-2.2c2.2,0,3.8,1,4.9,3l0.1-2.7h1.8v19.6c0,2.3-0.6,4-1.6,5.2
+					c-1.1,1.2-2.7,1.8-4.7,1.8c-1,0-2.1-0.3-3.2-0.8s-1.9-1.1-2.4-1.9l0.9-1.4c1.3,1.5,2.8,2.2,4.5,2.2c1.6,0,2.7-0.4,3.5-1.3
+					c0.7-0.8,1.1-2.1,1.1-3.8v-3c-1.1,1.8-2.7,2.7-4.9,2.7c-2,0-3.5-0.7-4.5-2.2s-1.6-3.6-1.6-6.5V113.6z M157.2,115.4
+					c0,2.4,0.4,4.2,1.1,5.4s1.9,1.7,3.5,1.7c2.1,0,3.6-1,4.5-3v-9.7c-0.9-2.2-2.4-3.3-4.4-3.3c-1.6,0-2.8,0.6-3.5,1.7
+					s-1.1,2.9-1.1,5.3V115.4z"/>
+			</g>
+		</g>
+		<g>
+			<g>
+				<g>
+					<line class="st11" x1="58.9" y1="20.1" x2="77.9" y2="35.6"/>
+					<g>
+						<polygon class="st9" points="57.6,23.2 55.3,17.1 61.7,18.2 						"/>
+					</g>
+				</g>
+			</g>
+			<circle class="st5" cx="48" cy="10.1" r="10.1"/>
+			<g>
+				<g>
+					<line class="st11" x1="77.9" y1="35.1" x2="96.2" y2="66.7"/>
+					<g>
+						<polygon class="st9" points="93,67.5 98.6,70.7 98.6,64.2 						"/>
+					</g>
+				</g>
+			</g>
+			<g>
+				<g>
+					<line class="st11" x1="92.3" y1="38.9" x2="114.4" y2="43.6"/>
+					<g>
+						<polygon class="st9" points="92.6,42.3 87.8,38 93.9,36 						"/>
+					</g>
+				</g>
+			</g>
+			<g>
+				<g>
+					<line class="st11" x1="77.9" y1="35.6" x2="94.5" y2="25.1"/>
+					<g>
+						<polygon class="st9" points="95.4,28.3 98.4,22.6 91.9,22.9 						"/>
+					</g>
+				</g>
+			</g>
+			<g>
+				<g>
+					<line class="st11" x1="48.5" y1="41.3" x2="63.5" y2="38.7"/>
+					<g>
+						<polygon class="st9" points="63.1,42.1 68.1,38 62,35.7 						"/>
+					</g>
+				</g>
+			</g>
+			<g>
+				<g>
+					<line class="st11" x1="111.4" y1="66.3" x2="124.4" y2="45.2"/>
+					<g>
+						<polygon class="st9" points="114.6,67.2 109,70.2 109.1,63.8 						"/>
+					</g>
+				</g>
+			</g>
+			<g>
+				<g>
+					<line class="st11" x1="115.6" y1="30.4" x2="124.4" y2="45.2"/>
+					<g>
+						<polygon class="st9" points="113.3,32.8 113.3,26.4 118.9,29.5 						"/>
+					</g>
+				</g>
+			</g>
+			<circle class="st3" cx="77.9" cy="35.6" r="10.1"/>
+			<circle class="st4" cx="44.6" cy="42.4" r="10.1"/>
+			<circle class="st7" cx="107.4" cy="18.1" r="10.1"/>
+			<circle class="st8" cx="104.1" cy="79.1" r="10.1"/>
+			<circle class="st6" cx="124.4" cy="45.2" r="10.1"/>
+		</g>
+	</g>
+</g>
+</svg>

art/mgmt_logo_default_wide.svg

@@ -0,0 +1,132 @@
+<?xml version="1.0" encoding="utf-8"?>
+<svg version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 260.4 71.4" style="enable-background:new 0 0 260.4 71.4;" xml:space="preserve">
+<style type="text/css">
+	.st0{fill:#FFFFFF;}
+	.st1{fill:#E22434;}
+	.st2{display:none;}
+	.st3{fill:#1B3663;}
+	.st4{fill:#00B1D1;}
+	.st5{fill:#BFE6EF;}
+	.st6{fill:#69CBE0;}
+	.st7{fill:#0080BD;}
+	.st8{fill:#005DAB;}
+	.st9{fill:#183660;}
+	.st10{fill:none;stroke:#183660;stroke-width:1.9441;stroke-miterlimit:10;}
+	.st11{fill:none;stroke:#183660;stroke-width:1.6201;stroke-miterlimit:10;}
+	.st12{fill:none;stroke:#183660;stroke-width:1.2961;stroke-miterlimit:10;}
+	.st13{fill:none;stroke:#C0E6EF;stroke-width:1.9441;stroke-miterlimit:10;}
+	.st14{fill:#C0E6EF;}
+	.st15{fill:none;stroke:#C0E6EF;stroke-width:1.2961;stroke-miterlimit:10;}
+	.st16{fill:none;stroke:#C0E6EF;stroke-width:1.6201;stroke-miterlimit:10;}
+	.st17{fill:none;stroke:#FFFFFF;stroke-width:1.9441;stroke-miterlimit:10;}
+	.st18{fill:none;stroke:#FFFFFF;stroke-width:1.2961;stroke-miterlimit:10;}
+	.st19{fill:none;stroke:#FFFFFF;stroke-width:1.6201;stroke-miterlimit:10;}
+</style>
+<g id="Layer_2">
+</g>
+<g id="Layer_1">
+	<g>
+		<g>
+			<g>
+				<path class="st3" d="M96.7,25.7l0.1,1.8c1.1-1.4,2.6-2.1,4.4-2.1c1.9,0,3.2,0.9,4,2.6c1.1-1.7,2.6-2.6,4.7-2.6
+					c3.3,0,5,2.3,5.1,6.9v12.5h-5V32.6c0-1.1-0.2-1.9-0.5-2.4s-0.8-0.7-1.5-0.7c-0.9,0-1.6,0.6-2.1,1.7l0,0.6v12.9h-5V32.6
+					c0-1.1-0.1-1.9-0.4-2.4s-0.8-0.7-1.6-0.7c-0.9,0-1.5,0.5-2,1.4v13.8h-5v-19H96.7z"/>
+				<path class="st3" d="M118.5,34.6c0-3.1,0.6-5.4,1.7-7s2.7-2.3,4.7-2.3c1.7,0,3.1,0.7,4,2l0.2-1.7h4.5v19c0,2.4-0.7,4.3-2,5.6
+					c-1.4,1.3-3.3,1.9-5.9,1.9c-1,0-2.1-0.2-3.3-0.6s-2-0.9-2.6-1.6l1.7-3.4c0.5,0.5,1.1,0.9,1.8,1.2c0.7,0.3,1.5,0.5,2.1,0.5
+					c1.1,0,1.9-0.3,2.4-0.8s0.7-1.4,0.7-2.6v-1.6c-0.9,1.2-2.2,1.9-3.7,1.9c-2,0-3.6-0.8-4.7-2.4s-1.7-3.8-1.7-6.7V34.6z
+					 M123.5,35.9c0,1.8,0.2,3,0.7,3.8s1.2,1.2,2.2,1.2c1,0,1.8-0.4,2.3-1.1v-9.1c-0.5-0.8-1.3-1.2-2.2-1.2c-1,0-1.7,0.4-2.2,1.2
+					s-0.7,2.1-0.7,3.9V35.9z"/>
+				<path class="st3" d="M142.2,25.7l0.1,1.8c1.1-1.4,2.6-2.1,4.4-2.1c1.9,0,3.2,0.9,4,2.6c1.1-1.7,2.6-2.6,4.7-2.6
+					c3.3,0,5,2.3,5.1,6.9v12.5h-5V32.6c0-1.1-0.2-1.9-0.5-2.4c-0.3-0.5-0.8-0.7-1.5-0.7c-0.9,0-1.6,0.6-2.1,1.7l0,0.6v12.9h-5V32.6
+					c0-1.1-0.1-1.9-0.4-2.4c-0.3-0.5-0.8-0.7-1.6-0.7c-0.9,0-1.5,0.5-2,1.4v13.8h-5v-19H142.2z"/>
+				<path class="st3" d="M170.3,21v4.7h2.5v3.7h-2.5v9.5c0,0.8,0.1,1.3,0.3,1.5c0.2,0.3,0.6,0.4,1.2,0.4c0.5,0,0.9,0,1.2-0.1l0,3.9
+					c-0.8,0.3-1.8,0.5-2.7,0.5c-3.2,0-4.8-1.8-4.9-5.5V29.4h-2.2v-3.7h2.2V21H170.3z"/>
+				<path class="st4" d="M182.7,43.2c1.4,0,2.4-0.4,3.1-1.1s1.1-1.8,1.2-3.3h1.9c-0.1,1.9-0.7,3.5-1.9,4.6s-2.6,1.7-4.3,1.7
+					c-2.3,0-4-0.7-5.1-2.2s-1.7-3.6-1.8-6.4v-2.3c0-2.9,0.6-5.1,1.7-6.6s2.9-2.2,5.1-2.2c1.9,0,3.4,0.6,4.5,1.8s1.7,2.9,1.7,5H187
+					c-0.1-1.6-0.5-2.8-1.2-3.6s-1.8-1.3-3.1-1.3c-1.7,0-2.9,0.6-3.7,1.7c-0.8,1.1-1.2,2.9-1.2,5.2v2.2c0,2.4,0.4,4.2,1.2,5.3
+					S181,43.2,182.7,43.2z"/>
+				<path class="st4" d="M192.6,34.2c0-2.7,0.6-4.9,1.9-6.5s2.9-2.4,5.1-2.4c2.2,0,3.9,0.8,5.1,2.4c1.2,1.6,1.9,3.7,1.9,6.5v2
+					c0,2.8-0.6,5-1.9,6.5s-2.9,2.3-5.1,2.3s-3.8-0.8-5-2.3s-1.9-3.6-1.9-6.3V34.2z M194.6,36.2c0,2.2,0.4,3.9,1.3,5.2
+					c0.9,1.3,2.1,1.9,3.7,1.9c1.6,0,2.8-0.6,3.7-1.8c0.8-1.2,1.3-2.9,1.3-5.2v-2c0-2.2-0.4-3.9-1.3-5.2c-0.9-1.3-2.1-1.9-3.7-1.9
+					c-1.5,0-2.7,0.6-3.6,1.8c-0.9,1.2-1.3,2.9-1.4,5.1V36.2z"/>
+				<path class="st4" d="M213.2,25.7l0.1,3c0.6-1,1.3-1.9,2.2-2.5s1.9-0.9,3-0.9c3.3,0,5,2.2,5.1,6.6v12.7h-1.9V32.2
+					c0-1.7-0.3-3-0.9-3.8c-0.6-0.8-1.5-1.2-2.8-1.2c-1,0-1.9,0.4-2.8,1.2s-1.4,1.8-1.9,3.2v13.2h-2v-19H213.2z"/>
+				<path class="st4" d="M230.3,44.7V27.4h-2.6v-1.8h2.6v-2.5c0-1.9,0.5-3.3,1.3-4.3s2-1.5,3.5-1.5c0.7,0,1.3,0.1,1.9,0.3l-0.1,1.8
+					c-0.5-0.1-1-0.2-1.6-0.2c-0.9,0-1.7,0.4-2.2,1.1s-0.8,1.7-0.8,3v2.3h3.7v1.8h-3.7v17.3H230.3z"/>
+				<path class="st4" d="M240.1,20.2c0-0.4,0.1-0.7,0.3-1s0.5-0.4,0.9-0.4s0.7,0.1,1,0.4s0.3,0.6,0.3,1s-0.1,0.7-0.3,1
+					s-0.5,0.4-1,0.4s-0.7-0.1-0.9-0.4S240.1,20.6,240.1,20.2z M242.3,44.7h-2v-19h2V44.7z"/>
+				<path class="st4" d="M247.4,34.3c0-3,0.5-5.2,1.5-6.7s2.6-2.2,4.6-2.2c2.2,0,3.8,1,4.9,3l0.1-2.7h1.8v19.6c0,2.3-0.6,4-1.6,5.2
+					s-2.7,1.8-4.7,1.8c-1,0-2.1-0.3-3.2-0.8s-1.9-1.1-2.4-1.9l0.9-1.4c1.3,1.5,2.8,2.2,4.5,2.2c1.6,0,2.7-0.4,3.5-1.3
+					c0.7-0.8,1.1-2.1,1.1-3.8v-3c-1.1,1.8-2.7,2.7-4.9,2.7c-2,0-3.5-0.7-4.5-2.2s-1.6-3.6-1.6-6.5V34.3z M249.3,36.1
+					c0,2.4,0.4,4.2,1.1,5.4s1.9,1.7,3.5,1.7c2.1,0,3.6-1,4.5-3v-9.7c-0.9-2.2-2.4-3.3-4.4-3.3c-1.6,0-2.8,0.6-3.5,1.7
+					s-1.1,2.9-1.1,5.3V36.1z"/>
+			</g>
+		</g>
+		<g>
+			<g>
+				<g>
+					<line class="st12" x1="19.5" y1="16" x2="34.7" y2="28.5"/>
+					<g>
+						<polygon class="st9" points="18.4,18.5 16.6,13.7 21.7,14.5 						"/>
+					</g>
+				</g>
+			</g>
+			<circle class="st5" cx="10.8" cy="8.1" r="8.1"/>
+			<g>
+				<g>
+					<line class="st12" x1="34.7" y1="28" x2="49.4" y2="53.3"/>
+					<g>
+						<polygon class="st9" points="46.8,54 51.2,56.5 51.2,51.4 						"/>
+					</g>
+				</g>
+			</g>
+			<g>
+				<g>
+					<line class="st12" x1="46.2" y1="31.1" x2="63.9" y2="34.9"/>
+					<g>
+						<polygon class="st9" points="46.4,33.8 42.6,30.4 47.5,28.8 						"/>
+					</g>
+				</g>
+			</g>
+			<g>
+				<g>
+					<line class="st12" x1="34.7" y1="28.5" x2="47.9" y2="20.1"/>
+					<g>
+						<polygon class="st9" points="48.7,22.7 51.1,18.1 45.9,18.3 						"/>
+					</g>
+				</g>
+			</g>
+			<g>
+				<g>
+					<line class="st12" x1="11.2" y1="33.1" x2="23.2" y2="31"/>
+					<g>
+						<polygon class="st9" points="22.9,33.7 26.8,30.4 22,28.6 						"/>
+					</g>
+				</g>
+			</g>
+			<g>
+				<g>
+					<line class="st12" x1="61.5" y1="53" x2="71.9" y2="36.1"/>
+					<g>
+						<polygon class="st9" points="64.1,53.7 59.5,56.2 59.7,51 						"/>
+					</g>
+				</g>
+			</g>
+			<g>
+				<g>
+					<line class="st12" x1="64.2" y1="24.2" x2="70.7" y2="33.9"/>
+					<g>
+						<polygon class="st9" points="62.5,26.3 62.2,21.1 66.8,23.4 						"/>
+					</g>
+				</g>
+			</g>
+			<circle class="st3" cx="34.7" cy="28.5" r="8.1"/>
+			<circle class="st4" cx="8.1" cy="33.9" r="8.1"/>
+			<circle class="st7" cx="58.3" cy="14.5" r="8.1"/>
+			<circle class="st8" cx="55.7" cy="63.3" r="8.1"/>
+			<circle class="st6" cx="71.9" cy="36.1" r="8.1"/>
+		</g>
+	</g>
+</g>
+</svg>

art/mgmt_logo_reversed_symbol.svg

@@ -0,0 +1,94 @@
+<?xml version="1.0" encoding="utf-8"?>
+<svg version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 120 107.1" style="enable-background:new 0 0 120 107.1;" xml:space="preserve">
+<style type="text/css">
+	.st0{fill:#FFFFFF;}
+	.st1{fill:#E22434;}
+	.st2{display:none;}
+	.st3{fill:#1B3663;}
+	.st4{fill:#00B1D1;}
+	.st5{fill:#BFE6EF;}
+	.st6{fill:#69CBE0;}
+	.st7{fill:#0080BD;}
+	.st8{fill:#005DAB;}
+	.st9{fill:#183660;}
+	.st10{fill:none;stroke:#183660;stroke-width:1.9441;stroke-miterlimit:10;}
+	.st11{fill:none;stroke:#183660;stroke-width:1.6201;stroke-miterlimit:10;}
+	.st12{fill:none;stroke:#183660;stroke-width:1.2961;stroke-miterlimit:10;}
+	.st13{fill:none;stroke:#C0E6EF;stroke-width:1.9441;stroke-miterlimit:10;}
+	.st14{fill:#C0E6EF;}
+	.st15{fill:none;stroke:#C0E6EF;stroke-width:1.2961;stroke-miterlimit:10;}
+	.st16{fill:none;stroke:#C0E6EF;stroke-width:1.6201;stroke-miterlimit:10;}
+	.st17{fill:none;stroke:#FFFFFF;stroke-width:1.9441;stroke-miterlimit:10;}
+	.st18{fill:none;stroke:#FFFFFF;stroke-width:1.2961;stroke-miterlimit:10;}
+	.st19{fill:none;stroke:#FFFFFF;stroke-width:1.6201;stroke-miterlimit:10;}
+</style>
+<g id="Layer_2">
+</g>
+<g id="Layer_1">
+	<g>
+		<g>
+			<g>
+				<line class="st13" x1="29.2" y1="24.1" x2="52.1" y2="42.8"/>
+				<g>
+					<polygon class="st14" points="27.7,27.8 24.9,20.5 32.6,21.8 					"/>
+				</g>
+			</g>
+		</g>
+		<circle class="st3" cx="16.1" cy="12.2" r="12.1"/>
+		<g>
+			<g>
+				<line class="st13" x1="52.1" y1="42.1" x2="74.1" y2="80"/>
+				<g>
+					<polygon class="st14" points="70.1,80.9 76.9,84.8 76.8,77.1 					"/>
+				</g>
+			</g>
+		</g>
+		<g>
+			<g>
+				<line class="st13" x1="69.4" y1="46.7" x2="95.8" y2="52.4"/>
+				<g>
+					<polygon class="st14" points="69.7,50.7 63.9,45.6 71.3,43.2 					"/>
+				</g>
+			</g>
+		</g>
+		<g>
+			<g>
+				<line class="st13" x1="52.1" y1="42.8" x2="71.9" y2="30.1"/>
+				<g>
+					<polygon class="st14" points="73.1,34 76.6,27.1 68.9,27.5 					"/>
+				</g>
+			</g>
+		</g>
+		<g>
+			<g>
+				<line class="st13" x1="16.8" y1="49.6" x2="34.8" y2="46.5"/>
+				<g>
+					<polygon class="st14" points="34.3,50.5 40.3,45.6 33,42.9 					"/>
+				</g>
+			</g>
+		</g>
+		<g>
+			<g>
+				<line class="st13" x1="92.3" y1="79.5" x2="107.8" y2="54.2"/>
+				<g>
+					<polygon class="st14" points="96.1,80.6 89.3,84.3 89.5,76.5 					"/>
+				</g>
+			</g>
+		</g>
+		<g>
+			<g>
+				<line class="st13" x1="97.2" y1="36.3" x2="107.8" y2="54.2"/>
+				<g>
+					<polygon class="st14" points="94.4,39.3 94.3,31.5 101.1,35.3 					"/>
+				</g>
+			</g>
+		</g>
+		<circle class="st5" cx="52.1" cy="42.8" r="12.1"/>
+		<circle class="st4" cx="12.2" cy="50.8" r="12.1"/>
+		<circle class="st7" cx="87.5" cy="21.7" r="12.1"/>
+		<circle class="st8" cx="83.5" cy="95" r="12.1"/>
+		<circle class="st6" cx="107.8" cy="54.2" r="12.1"/>
+	</g>
+</g>
+</svg>

art/mgmt_logo_reversed_tall.svg

@@ -0,0 +1,132 @@
+<?xml version="1.0" encoding="utf-8"?>
+<svg version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 168.3 133" style="enable-background:new 0 0 168.3 133;" xml:space="preserve">
+<style type="text/css">
+	.st0{fill:#FFFFFF;}
+	.st1{fill:#E22434;}
+	.st2{display:none;}
+	.st3{fill:#1B3663;}
+	.st4{fill:#00B1D1;}
+	.st5{fill:#BFE6EF;}
+	.st6{fill:#69CBE0;}
+	.st7{fill:#0080BD;}
+	.st8{fill:#005DAB;}
+	.st9{fill:#183660;}
+	.st10{fill:none;stroke:#183660;stroke-width:1.9441;stroke-miterlimit:10;}
+	.st11{fill:none;stroke:#183660;stroke-width:1.6201;stroke-miterlimit:10;}
+	.st12{fill:none;stroke:#183660;stroke-width:1.2961;stroke-miterlimit:10;}
+	.st13{fill:none;stroke:#C0E6EF;stroke-width:1.9441;stroke-miterlimit:10;}
+	.st14{fill:#C0E6EF;}
+	.st15{fill:none;stroke:#C0E6EF;stroke-width:1.2961;stroke-miterlimit:10;}
+	.st16{fill:none;stroke:#C0E6EF;stroke-width:1.6201;stroke-miterlimit:10;}
+	.st17{fill:none;stroke:#FFFFFF;stroke-width:1.9441;stroke-miterlimit:10;}
+	.st18{fill:none;stroke:#FFFFFF;stroke-width:1.2961;stroke-miterlimit:10;}
+	.st19{fill:none;stroke:#FFFFFF;stroke-width:1.6201;stroke-miterlimit:10;}
+</style>
+<g id="Layer_2">
+</g>
+<g id="Layer_1">
+	<g>
+		<g>
+			<g>
+				<path class="st0" d="M4.7,106.4l0.1,1.8c1.1-1.4,2.6-2.1,4.4-2.1c1.9,0,3.2,0.9,4,2.6c1.1-1.7,2.6-2.6,4.7-2.6
+					c3.3,0,5,2.3,5.1,6.9v12.5h-5v-12.1c0-1.1-0.2-1.9-0.5-2.4c-0.3-0.5-0.8-0.7-1.5-0.7c-0.9,0-1.6,0.6-2.1,1.7l0,0.6v12.9H9v-12.1
+					c0-1.1-0.1-1.9-0.4-2.4c-0.3-0.5-0.8-0.7-1.6-0.7c-0.9,0-1.5,0.5-2,1.4v13.8H0v-19H4.7z"/>
+				<path class="st0" d="M26.4,115.3c0-3.1,0.6-5.4,1.7-7s2.7-2.3,4.7-2.3c1.7,0,3.1,0.7,4,2l0.2-1.7h4.5v19c0,2.4-0.7,4.3-2,5.6
+					c-1.4,1.3-3.3,1.9-5.9,1.9c-1,0-2.1-0.2-3.3-0.6s-2-0.9-2.6-1.6l1.7-3.4c0.5,0.5,1.1,0.9,1.8,1.2c0.7,0.3,1.5,0.5,2.1,0.5
+					c1.1,0,1.9-0.3,2.4-0.8c0.5-0.5,0.7-1.4,0.7-2.6v-1.6c-0.9,1.2-2.2,1.9-3.7,1.9c-2,0-3.6-0.8-4.7-2.4s-1.7-3.8-1.7-6.7V115.3z
+					 M31.4,116.6c0,1.8,0.2,3,0.7,3.8s1.2,1.2,2.2,1.2c1,0,1.8-0.4,2.3-1.1v-9.1c-0.5-0.8-1.3-1.2-2.2-1.2c-1,0-1.7,0.4-2.2,1.2
+					s-0.7,2.1-0.7,3.9V116.6z"/>
+				<path class="st0" d="M50.1,106.4l0.1,1.8c1.1-1.4,2.6-2.1,4.4-2.1c1.9,0,3.2,0.9,4,2.6c1.1-1.7,2.6-2.6,4.7-2.6
+					c3.3,0,5,2.3,5.1,6.9v12.5h-5v-12.1c0-1.1-0.2-1.9-0.5-2.4s-0.8-0.7-1.5-0.7c-0.9,0-1.6,0.6-2.1,1.7l0,0.6v12.9h-5v-12.1
+					c0-1.1-0.1-1.9-0.4-2.4s-0.8-0.7-1.6-0.7c-0.9,0-1.5,0.5-2,1.4v13.8h-5v-19H50.1z"/>
+				<path class="st0" d="M78.2,101.7v4.7h2.5v3.7h-2.5v9.5c0,0.8,0.1,1.3,0.3,1.5c0.2,0.3,0.6,0.4,1.2,0.4c0.5,0,0.9,0,1.2-0.1
+					l0,3.9c-0.8,0.3-1.8,0.5-2.7,0.5c-3.2,0-4.8-1.8-4.9-5.5v-10.1h-2.2v-3.7h2.2v-4.7H78.2z"/>
+				<path class="st4" d="M90.6,124c1.4,0,2.4-0.4,3.1-1.1c0.7-0.8,1.1-1.9,1.2-3.3h1.9c-0.1,1.9-0.7,3.5-1.9,4.6
+					c-1.1,1.1-2.6,1.7-4.3,1.7c-2.3,0-4-0.7-5.1-2.2s-1.7-3.6-1.8-6.4v-2.3c0-2.9,0.6-5.1,1.7-6.6s2.9-2.2,5.1-2.2
+					c1.9,0,3.4,0.6,4.5,1.8s1.7,2.9,1.7,5H95c-0.1-1.6-0.5-2.8-1.2-3.6s-1.8-1.3-3.1-1.3c-1.7,0-2.9,0.6-3.7,1.7s-1.2,2.9-1.2,5.2
+					v2.2c0,2.4,0.4,4.2,1.2,5.3S89,124,90.6,124z"/>
+				<path class="st4" d="M100.5,115c0-2.7,0.6-4.9,1.9-6.5s2.9-2.4,5.1-2.4c2.2,0,3.9,0.8,5.1,2.4s1.9,3.7,1.9,6.5v2
+					c0,2.8-0.6,5-1.9,6.5s-2.9,2.3-5.1,2.3c-2.1,0-3.8-0.8-5-2.3s-1.9-3.6-1.9-6.3V115z M102.5,116.9c0,2.2,0.4,3.9,1.3,5.2
+					c0.9,1.3,2.1,1.9,3.7,1.9c1.6,0,2.8-0.6,3.7-1.8s1.3-2.9,1.3-5.2v-2c0-2.2-0.4-3.9-1.3-5.2c-0.9-1.3-2.1-1.9-3.7-1.9
+					c-1.5,0-2.7,0.6-3.6,1.8s-1.3,2.9-1.4,5.1V116.9z"/>
+				<path class="st4" d="M121.1,106.4l0.1,3c0.6-1,1.3-1.9,2.2-2.5s1.9-0.9,3-0.9c3.3,0,5,2.2,5.1,6.6v12.7h-1.9v-12.5
+					c0-1.7-0.3-3-0.9-3.8c-0.6-0.8-1.5-1.2-2.8-1.2c-1,0-1.9,0.4-2.8,1.2s-1.4,1.8-1.9,3.2v13.2h-2v-19H121.1z"/>
+				<path class="st4" d="M138.2,125.4v-17.3h-2.6v-1.8h2.6v-2.5c0-1.9,0.5-3.3,1.3-4.3s2-1.5,3.5-1.5c0.7,0,1.3,0.1,1.9,0.3
+					l-0.1,1.8c-0.5-0.1-1-0.2-1.6-0.2c-0.9,0-1.7,0.4-2.2,1.1s-0.8,1.7-0.8,3v2.3h3.7v1.8h-3.7v17.3H138.2z"/>
+				<path class="st4" d="M148,100.9c0-0.4,0.1-0.7,0.3-1s0.5-0.4,0.9-0.4s0.7,0.1,1,0.4s0.3,0.6,0.3,1s-0.1,0.7-0.3,1
+					s-0.5,0.4-1,0.4s-0.7-0.1-0.9-0.4S148,101.3,148,100.9z M150.2,125.4h-2v-19h2V125.4z"/>
+				<path class="st4" d="M155.3,115c0-3,0.5-5.2,1.5-6.7s2.6-2.2,4.6-2.2c2.2,0,3.8,1,4.9,3l0.1-2.7h1.8V126c0,2.3-0.6,4-1.6,5.2
+					c-1.1,1.2-2.7,1.8-4.7,1.8c-1,0-2.1-0.3-3.2-0.8s-1.9-1.1-2.4-1.9l0.9-1.4c1.3,1.5,2.8,2.2,4.5,2.2c1.6,0,2.7-0.4,3.5-1.3
+					c0.7-0.8,1.1-2.1,1.1-3.8v-3c-1.1,1.8-2.7,2.7-4.9,2.7c-2,0-3.5-0.7-4.5-2.2s-1.6-3.6-1.6-6.5V115z M157.2,116.8
+					c0,2.4,0.4,4.2,1.1,5.4s1.9,1.7,3.5,1.7c2.1,0,3.6-1,4.5-3v-9.7c-0.9-2.2-2.4-3.3-4.4-3.3c-1.6,0-2.8,0.6-3.5,1.7
+					s-1.1,2.9-1.1,5.3V116.8z"/>
+			</g>
+		</g>
+		<g>
+			<g>
+				<g>
+					<line class="st16" x1="58.9" y1="20.1" x2="77.9" y2="35.6"/>
+					<g>
+						<polygon class="st14" points="57.6,23.2 55.3,17.1 61.7,18.2 						"/>
+					</g>
+				</g>
+			</g>
+			<circle class="st3" cx="48" cy="10.1" r="10.1"/>
+			<g>
+				<g>
+					<line class="st16" x1="77.9" y1="35.1" x2="96.2" y2="66.7"/>
+					<g>
+						<polygon class="st14" points="93,67.5 98.6,70.7 98.6,64.2 						"/>
+					</g>
+				</g>
+			</g>
+			<g>
+				<g>
+					<line class="st16" x1="92.3" y1="38.9" x2="114.4" y2="43.6"/>
+					<g>
+						<polygon class="st14" points="92.6,42.3 87.8,38 93.9,36 						"/>
+					</g>
+				</g>
+			</g>
+			<g>
+				<g>
+					<line class="st16" x1="77.9" y1="35.6" x2="94.5" y2="25.1"/>
+					<g>
+						<polygon class="st14" points="95.4,28.3 98.4,22.6 91.9,22.9 						"/>
+					</g>
+				</g>
+			</g>
+			<g>
+				<g>
+					<line class="st16" x1="48.5" y1="41.3" x2="63.5" y2="38.7"/>
+					<g>
+						<polygon class="st14" points="63.1,42.1 68.1,38 62,35.7 						"/>
+					</g>
+				</g>
+			</g>
+			<g>
+				<g>
+					<line class="st16" x1="111.4" y1="66.3" x2="124.4" y2="45.2"/>
+					<g>
+						<polygon class="st14" points="114.6,67.2 109,70.2 109.1,63.8 						"/>
+					</g>
+				</g>
+			</g>
+			<g>
+				<g>
+					<line class="st16" x1="115.5" y1="30.5" x2="124.4" y2="45.2"/>
+					<g>
+						<polygon class="st14" points="113.2,32.9 113.1,26.5 118.8,29.6 						"/>
+					</g>
+				</g>
+			</g>
+			<circle class="st5" cx="77.9" cy="35.6" r="10.1"/>
+			<circle class="st4" cx="44.6" cy="42.4" r="10.1"/>
+			<circle class="st7" cx="107.4" cy="18.1" r="10.1"/>
+			<circle class="st8" cx="104.1" cy="79.1" r="10.1"/>
+			<circle class="st6" cx="124.4" cy="45.2" r="10.1"/>
+		</g>
+	</g>
+</g>
+</svg>

art/mgmt_logo_reversed_wide.svg

@@ -0,0 +1,132 @@
+<?xml version="1.0" encoding="utf-8"?>
+<svg version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 260.4 71.4" style="enable-background:new 0 0 260.4 71.4;" xml:space="preserve">
+<style type="text/css">
+	.st0{fill:#FFFFFF;}
+	.st1{fill:#E22434;}
+	.st2{display:none;}
+	.st3{fill:#1B3663;}
+	.st4{fill:#00B1D1;}
+	.st5{fill:#BFE6EF;}
+	.st6{fill:#69CBE0;}
+	.st7{fill:#0080BD;}
+	.st8{fill:#005DAB;}
+	.st9{fill:#183660;}
+	.st10{fill:none;stroke:#183660;stroke-width:1.9441;stroke-miterlimit:10;}
+	.st11{fill:none;stroke:#183660;stroke-width:1.6201;stroke-miterlimit:10;}
+	.st12{fill:none;stroke:#183660;stroke-width:1.2961;stroke-miterlimit:10;}
+	.st13{fill:none;stroke:#C0E6EF;stroke-width:1.9441;stroke-miterlimit:10;}
+	.st14{fill:#C0E6EF;}
+	.st15{fill:none;stroke:#C0E6EF;stroke-width:1.2961;stroke-miterlimit:10;}
+	.st16{fill:none;stroke:#C0E6EF;stroke-width:1.6201;stroke-miterlimit:10;}
+	.st17{fill:none;stroke:#FFFFFF;stroke-width:1.9441;stroke-miterlimit:10;}
+	.st18{fill:none;stroke:#FFFFFF;stroke-width:1.2961;stroke-miterlimit:10;}
+	.st19{fill:none;stroke:#FFFFFF;stroke-width:1.6201;stroke-miterlimit:10;}
+</style>
+<g id="Layer_2">
+</g>
+<g id="Layer_1">
+	<g>
+		<g>
+			<g>
+				<path class="st0" d="M96.7,27.6l0.1,1.8c1.1-1.4,2.6-2.1,4.4-2.1c1.9,0,3.2,0.9,4,2.6c1.1-1.7,2.6-2.6,4.7-2.6
+					c3.3,0,5,2.3,5.1,6.9v12.5h-5V34.4c0-1.1-0.2-1.9-0.5-2.4c-0.3-0.5-0.8-0.7-1.5-0.7c-0.9,0-1.6,0.6-2.1,1.7l0,0.6v12.9h-5V34.5
+					c0-1.1-0.1-1.9-0.4-2.4c-0.3-0.5-0.8-0.7-1.6-0.7c-0.9,0-1.5,0.5-2,1.4v13.8h-5v-19H96.7z"/>
+				<path class="st0" d="M118.5,36.5c0-3.1,0.6-5.4,1.7-7s2.7-2.3,4.7-2.3c1.7,0,3.1,0.7,4,2l0.2-1.7h4.5v19c0,2.4-0.7,4.3-2,5.6
+					c-1.4,1.3-3.3,1.9-5.9,1.9c-1,0-2.1-0.2-3.3-0.6s-2-0.9-2.6-1.6l1.7-3.4c0.5,0.5,1.1,0.9,1.8,1.2c0.7,0.3,1.5,0.5,2.1,0.5
+					c1.1,0,1.9-0.3,2.4-0.8s0.7-1.4,0.7-2.6v-1.6c-0.9,1.2-2.2,1.9-3.7,1.9c-2,0-3.6-0.8-4.7-2.4s-1.7-3.8-1.7-6.7V36.5z
+					 M123.5,37.7c0,1.8,0.2,3,0.7,3.8s1.2,1.2,2.2,1.2c1,0,1.8-0.4,2.3-1.1v-9.1c-0.5-0.8-1.3-1.2-2.2-1.2c-1,0-1.7,0.4-2.2,1.2
+					c-0.5,0.8-0.7,2.1-0.7,3.9V37.7z"/>
+				<path class="st0" d="M142.2,27.6l0.1,1.8c1.1-1.4,2.6-2.1,4.4-2.1c1.9,0,3.2,0.9,4,2.6c1.1-1.7,2.6-2.6,4.7-2.6
+					c3.3,0,5,2.3,5.1,6.9v12.5h-5V34.4c0-1.1-0.2-1.9-0.5-2.4c-0.3-0.5-0.8-0.7-1.5-0.7c-0.9,0-1.6,0.6-2.1,1.7l0,0.6v12.9h-5V34.5
+					c0-1.1-0.1-1.9-0.4-2.4c-0.3-0.5-0.8-0.7-1.6-0.7c-0.9,0-1.5,0.5-2,1.4v13.8h-5v-19H142.2z"/>
+				<path class="st0" d="M170.3,22.9v4.7h2.5v3.7h-2.5v9.5c0,0.8,0.1,1.3,0.3,1.5c0.2,0.3,0.6,0.4,1.2,0.4c0.5,0,0.9,0,1.2-0.1
+					l0,3.9c-0.8,0.3-1.8,0.5-2.7,0.5c-3.2,0-4.8-1.8-4.9-5.5V31.3h-2.2v-3.7h2.2v-4.7H170.3z"/>
+				<path class="st4" d="M182.7,45.1c1.4,0,2.4-0.4,3.1-1.1c0.7-0.8,1.1-1.9,1.2-3.3h1.9c-0.1,1.9-0.7,3.5-1.9,4.6s-2.6,1.7-4.3,1.7
+					c-2.3,0-4-0.7-5.1-2.2s-1.7-3.6-1.8-6.4V36c0-2.9,0.6-5.1,1.7-6.6s2.9-2.2,5.1-2.2c1.9,0,3.4,0.6,4.5,1.8s1.7,2.9,1.7,5H187
+					c-0.1-1.6-0.5-2.8-1.2-3.6s-1.8-1.3-3.1-1.3c-1.7,0-2.9,0.6-3.7,1.7s-1.2,2.9-1.2,5.2v2.2c0,2.4,0.4,4.2,1.2,5.3
+					S181,45.1,182.7,45.1z"/>
+				<path class="st4" d="M192.6,36.1c0-2.7,0.6-4.9,1.9-6.5s2.9-2.4,5.1-2.4c2.2,0,3.9,0.8,5.1,2.4c1.2,1.6,1.9,3.7,1.9,6.5v2
+					c0,2.8-0.6,5-1.9,6.5s-2.9,2.3-5.1,2.3s-3.8-0.8-5-2.3s-1.9-3.6-1.9-6.3V36.1z M194.6,38.1c0,2.2,0.4,3.9,1.3,5.2
+					c0.9,1.3,2.1,1.9,3.7,1.9c1.6,0,2.8-0.6,3.7-1.8c0.8-1.2,1.3-2.9,1.3-5.2v-2c0-2.2-0.4-3.9-1.3-5.2c-0.9-1.3-2.1-1.9-3.7-1.9
+					c-1.5,0-2.7,0.6-3.6,1.8c-0.9,1.2-1.3,2.9-1.4,5.1V38.1z"/>
+				<path class="st4" d="M213.2,27.6l0.1,3c0.6-1,1.3-1.9,2.2-2.5s1.9-0.9,3-0.9c3.3,0,5,2.2,5.1,6.6v12.7h-1.9V34.1
+					c0-1.7-0.3-3-0.9-3.8c-0.6-0.8-1.5-1.2-2.8-1.2c-1,0-1.9,0.4-2.8,1.2s-1.4,1.8-1.9,3.2v13.2h-2v-19H213.2z"/>
+				<path class="st4" d="M230.3,46.6V29.3h-2.6v-1.8h2.6v-2.5c0-1.9,0.5-3.3,1.3-4.3s2-1.5,3.5-1.5c0.7,0,1.3,0.1,1.9,0.3l-0.1,1.8
+					c-0.5-0.1-1-0.2-1.6-0.2c-0.9,0-1.7,0.4-2.2,1.1s-0.8,1.7-0.8,3v2.3h3.7v1.8h-3.7v17.3H230.3z"/>
+				<path class="st4" d="M240.1,22.1c0-0.4,0.1-0.7,0.3-1s0.5-0.4,0.9-0.4s0.7,0.1,1,0.4s0.3,0.6,0.3,1s-0.1,0.7-0.3,1
+					s-0.5,0.4-1,0.4s-0.7-0.1-0.9-0.4S240.1,22.5,240.1,22.1z M242.3,46.6h-2v-19h2V46.6z"/>
+				<path class="st4" d="M247.4,36.2c0-3,0.5-5.2,1.5-6.7s2.6-2.2,4.6-2.2c2.2,0,3.8,1,4.9,3l0.1-2.7h1.8v19.6c0,2.3-0.6,4-1.6,5.2
+					s-2.7,1.8-4.7,1.8c-1,0-2.1-0.3-3.2-0.8s-1.9-1.1-2.4-1.9l0.9-1.4c1.3,1.5,2.8,2.2,4.5,2.2c1.6,0,2.7-0.4,3.5-1.3
+					c0.7-0.8,1.1-2.1,1.1-3.8v-3c-1.1,1.8-2.7,2.7-4.9,2.7c-2,0-3.5-0.7-4.5-2.2s-1.6-3.6-1.6-6.5V36.2z M249.3,38
+					c0,2.4,0.4,4.2,1.1,5.4s1.9,1.7,3.5,1.7c2.1,0,3.6-1,4.5-3v-9.7c-0.9-2.2-2.4-3.3-4.4-3.3c-1.6,0-2.8,0.6-3.5,1.7
+					s-1.1,2.9-1.1,5.3V38z"/>
+			</g>
+		</g>
+		<g>
+			<g>
+				<g>
+					<line class="st15" x1="19.5" y1="16" x2="34.7" y2="28.5"/>
+					<g>
+						<polygon class="st14" points="18.4,18.5 16.6,13.7 21.7,14.5 						"/>
+					</g>
+				</g>
+			</g>
+			<circle class="st3" cx="10.8" cy="8.1" r="8.1"/>
+			<g>
+				<g>
+					<line class="st15" x1="34.7" y1="28" x2="49.4" y2="53.3"/>
+					<g>
+						<polygon class="st14" points="46.8,54 51.2,56.5 51.2,51.4 						"/>
+					</g>
+				</g>
+			</g>
+			<g>
+				<g>
+					<line class="st15" x1="46.2" y1="31.1" x2="63.9" y2="34.9"/>
+					<g>
+						<polygon class="st14" points="46.4,33.8 42.6,30.4 47.5,28.8 						"/>
+					</g>
+				</g>
+			</g>
+			<g>
+				<g>
+					<line class="st15" x1="34.7" y1="28.5" x2="47.9" y2="20.1"/>
+					<g>
+						<polygon class="st14" points="48.7,22.7 51.1,18.1 45.9,18.3 						"/>
+					</g>
+				</g>
+			</g>
+			<g>
+				<g>
+					<line class="st15" x1="11.2" y1="33.1" x2="23.2" y2="31"/>
+					<g>
+						<polygon class="st14" points="22.9,33.7 26.8,30.4 22,28.6 						"/>
+					</g>
+				</g>
+			</g>
+			<g>
+				<g>
+					<line class="st15" x1="61.5" y1="53" x2="71.9" y2="36.1"/>
+					<g>
+						<polygon class="st14" points="64.1,53.7 59.5,56.2 59.7,51 						"/>
+					</g>
+				</g>
+			</g>
+			<g>
+				<g>
+					<line class="st15" x1="64.8" y1="24.4" x2="71.9" y2="36.1"/>
+					<g>
+						<polygon class="st14" points="63,26.4 62.9,21.2 67.4,23.7 						"/>
+					</g>
+				</g>
+			</g>
+			<circle class="st5" cx="34.7" cy="28.5" r="8.1"/>
+			<circle class="st4" cx="8.1" cy="33.9" r="8.1"/>
+			<circle class="st7" cx="58.3" cy="14.5" r="8.1"/>
+			<circle class="st8" cx="55.7" cy="63.3" r="8.1"/>
+			<circle class="st6" cx="71.9" cy="36.1" r="8.1"/>
+		</g>
+	</g>
+</g>
+</svg>

art/mgmt_logo_white_symbol.svg

@@ -0,0 +1,94 @@
+<?xml version="1.0" encoding="utf-8"?>
+<svg version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 120 107.1" style="enable-background:new 0 0 120 107.1;" xml:space="preserve">
+<style type="text/css">
+	.st0{fill:#FFFFFF;}
+	.st1{fill:#E22434;}
+	.st2{display:none;}
+	.st3{fill:#1B3663;}
+	.st4{fill:#00B1D1;}
+	.st5{fill:#BFE6EF;}
+	.st6{fill:#69CBE0;}
+	.st7{fill:#0080BD;}
+	.st8{fill:#005DAB;}
+	.st9{fill:#183660;}
+	.st10{fill:none;stroke:#183660;stroke-width:1.9441;stroke-miterlimit:10;}
+	.st11{fill:none;stroke:#183660;stroke-width:1.6201;stroke-miterlimit:10;}
+	.st12{fill:none;stroke:#183660;stroke-width:1.2961;stroke-miterlimit:10;}
+	.st13{fill:none;stroke:#C0E6EF;stroke-width:1.9441;stroke-miterlimit:10;}
+	.st14{fill:#C0E6EF;}
+	.st15{fill:none;stroke:#C0E6EF;stroke-width:1.2961;stroke-miterlimit:10;}
+	.st16{fill:none;stroke:#C0E6EF;stroke-width:1.6201;stroke-miterlimit:10;}
+	.st17{fill:none;stroke:#FFFFFF;stroke-width:1.9441;stroke-miterlimit:10;}
+	.st18{fill:none;stroke:#FFFFFF;stroke-width:1.2961;stroke-miterlimit:10;}
+	.st19{fill:none;stroke:#FFFFFF;stroke-width:1.6201;stroke-miterlimit:10;}
+</style>
+<g id="Layer_2">
+</g>
+<g id="Layer_1">
+	<g>
+		<g>
+			<g>
+				<line class="st17" x1="29.2" y1="24.1" x2="52.1" y2="42.8"/>
+				<g>
+					<polygon class="st0" points="27.7,27.8 24.9,20.5 32.6,21.8 					"/>
+				</g>
+			</g>
+		</g>
+		<circle class="st0" cx="16.1" cy="12.2" r="12.1"/>
+		<g>
+			<g>
+				<line class="st17" x1="52.1" y1="42.1" x2="74.1" y2="80"/>
+				<g>
+					<polygon class="st0" points="70.1,80.9 76.9,84.8 76.8,77.1 					"/>
+				</g>
+			</g>
+		</g>
+		<g>
+			<g>
+				<line class="st17" x1="69.4" y1="46.7" x2="95.8" y2="52.4"/>
+				<g>
+					<polygon class="st0" points="69.7,50.7 63.9,45.6 71.3,43.2 					"/>
+				</g>
+			</g>
+		</g>
+		<g>
+			<g>
+				<line class="st17" x1="52.1" y1="42.8" x2="71.9" y2="30.1"/>
+				<g>
+					<polygon class="st0" points="73.1,34 76.6,27.1 68.9,27.5 					"/>
+				</g>
+			</g>
+		</g>
+		<g>
+			<g>
+				<line class="st17" x1="16.8" y1="49.6" x2="34.8" y2="46.5"/>
+				<g>
+					<polygon class="st0" points="34.3,50.5 40.3,45.6 33,42.9 					"/>
+				</g>
+			</g>
+		</g>
+		<g>
+			<g>
+				<line class="st17" x1="92.3" y1="79.5" x2="107.8" y2="54.2"/>
+				<g>
+					<polygon class="st0" points="96.1,80.6 89.3,84.3 89.5,76.5 					"/>
+				</g>
+			</g>
+		</g>
+		<g>
+			<g>
+				<line class="st17" x1="97.2" y1="36.6" x2="107.8" y2="54.2"/>
+				<g>
+					<polygon class="st0" points="94.4,39.5 94.3,31.8 101.1,35.5 					"/>
+				</g>
+			</g>
+		</g>
+		<circle class="st0" cx="52.1" cy="42.8" r="12.1"/>
+		<circle class="st0" cx="12.2" cy="50.8" r="12.1"/>
+		<circle class="st0" cx="87.5" cy="21.7" r="12.1"/>
+		<circle class="st0" cx="83.5" cy="95" r="12.1"/>
+		<circle class="st0" cx="107.8" cy="54.2" r="12.1"/>
+	</g>
+</g>
+</svg>

art/mgmt_logo_white_tall.svg

@@ -0,0 +1,132 @@
+<?xml version="1.0" encoding="utf-8"?>
+<svg version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 168.3 131.4" style="enable-background:new 0 0 168.3 131.4;" xml:space="preserve">
+<style type="text/css">
+	.st0{fill:#FFFFFF;}
+	.st1{fill:#E22434;}
+	.st2{display:none;}
+	.st3{fill:#1B3663;}
+	.st4{fill:#00B1D1;}
+	.st5{fill:#BFE6EF;}
+	.st6{fill:#69CBE0;}
+	.st7{fill:#0080BD;}
+	.st8{fill:#005DAB;}
+	.st9{fill:#183660;}
+	.st10{fill:none;stroke:#183660;stroke-width:1.9441;stroke-miterlimit:10;}
+	.st11{fill:none;stroke:#183660;stroke-width:1.6201;stroke-miterlimit:10;}
+	.st12{fill:none;stroke:#183660;stroke-width:1.2961;stroke-miterlimit:10;}
+	.st13{fill:none;stroke:#C0E6EF;stroke-width:1.9441;stroke-miterlimit:10;}
+	.st14{fill:#C0E6EF;}
+	.st15{fill:none;stroke:#C0E6EF;stroke-width:1.2961;stroke-miterlimit:10;}
+	.st16{fill:none;stroke:#C0E6EF;stroke-width:1.6201;stroke-miterlimit:10;}
+	.st17{fill:none;stroke:#FFFFFF;stroke-width:1.9441;stroke-miterlimit:10;}
+	.st18{fill:none;stroke:#FFFFFF;stroke-width:1.2961;stroke-miterlimit:10;}
+	.st19{fill:none;stroke:#FFFFFF;stroke-width:1.6201;stroke-miterlimit:10;}
+</style>
+<g id="Layer_2">
+</g>
+<g id="Layer_1">
+	<g>
+		<g>
+			<g>
+				<path class="st0" d="M4.7,104.8l0.1,1.8c1.1-1.4,2.6-2.1,4.4-2.1c1.9,0,3.2,0.9,4,2.6c1.1-1.7,2.6-2.6,4.7-2.6
+					c3.3,0,5,2.3,5.1,6.9v12.5h-5v-12.1c0-1.1-0.2-1.9-0.5-2.4c-0.3-0.5-0.8-0.7-1.5-0.7c-0.9,0-1.6,0.6-2.1,1.7l0,0.6v12.9H9v-12.1
+					c0-1.1-0.1-1.9-0.4-2.4c-0.3-0.5-0.8-0.7-1.6-0.7c-0.9,0-1.5,0.5-2,1.4v13.8H0v-19H4.7z"/>
+				<path class="st0" d="M26.4,113.8c0-3.1,0.6-5.4,1.7-7s2.7-2.3,4.7-2.3c1.7,0,3.1,0.7,4,2l0.2-1.7h4.5v19c0,2.4-0.7,4.3-2,5.6
+					c-1.4,1.3-3.3,1.9-5.9,1.9c-1,0-2.1-0.2-3.3-0.6s-2-0.9-2.6-1.6l1.7-3.4c0.5,0.5,1.1,0.9,1.8,1.2c0.7,0.3,1.5,0.5,2.1,0.5
+					c1.1,0,1.9-0.3,2.4-0.8s0.7-1.4,0.7-2.6v-1.6c-0.9,1.2-2.2,1.9-3.7,1.9c-2,0-3.6-0.8-4.7-2.4s-1.7-3.8-1.7-6.7V113.8z M31.4,115
+					c0,1.8,0.2,3,0.7,3.8s1.2,1.2,2.2,1.2c1,0,1.8-0.4,2.3-1.1v-9.1c-0.5-0.8-1.3-1.2-2.2-1.2c-1,0-1.7,0.4-2.2,1.2
+					s-0.7,2.1-0.7,3.9V115z"/>
+				<path class="st0" d="M50.1,104.8l0.1,1.8c1.1-1.4,2.6-2.1,4.4-2.1c1.9,0,3.2,0.9,4,2.6c1.1-1.7,2.6-2.6,4.7-2.6
+					c3.3,0,5,2.3,5.1,6.9v12.5h-5v-12.1c0-1.1-0.2-1.9-0.5-2.4s-0.8-0.7-1.5-0.7c-0.9,0-1.6,0.6-2.1,1.7l0,0.6v12.9h-5v-12.1
+					c0-1.1-0.1-1.9-0.4-2.4s-0.8-0.7-1.6-0.7c-0.9,0-1.5,0.5-2,1.4v13.8h-5v-19H50.1z"/>
+				<path class="st0" d="M78.2,100.2v4.7h2.5v3.7h-2.5v9.5c0,0.8,0.1,1.3,0.3,1.5c0.2,0.3,0.6,0.4,1.2,0.4c0.5,0,0.9,0,1.2-0.1
+					l0,3.9c-0.8,0.3-1.8,0.5-2.7,0.5c-3.2,0-4.8-1.8-4.9-5.5v-10.1h-2.2v-3.7h2.2v-4.7H78.2z"/>
+				<path class="st0" d="M90.6,122.4c1.4,0,2.4-0.4,3.1-1.1c0.7-0.8,1.1-1.8,1.2-3.3h1.9c-0.1,1.9-0.7,3.5-1.9,4.6
+					c-1.1,1.1-2.6,1.7-4.3,1.7c-2.3,0-4-0.7-5.1-2.2s-1.7-3.6-1.8-6.4v-2.3c0-2.9,0.6-5.1,1.7-6.6s2.9-2.2,5.1-2.2
+					c1.9,0,3.4,0.6,4.5,1.8s1.7,2.9,1.7,5H95c-0.1-1.6-0.5-2.8-1.2-3.6s-1.8-1.3-3.1-1.3c-1.7,0-2.9,0.6-3.7,1.7
+					c-0.8,1.1-1.2,2.9-1.2,5.2v2.2c0,2.4,0.4,4.2,1.2,5.3S89,122.4,90.6,122.4z"/>
+				<path class="st0" d="M100.5,113.4c0-2.7,0.6-4.9,1.9-6.5s2.9-2.4,5.1-2.4c2.2,0,3.9,0.8,5.1,2.4s1.9,3.7,1.9,6.5v2
+					c0,2.8-0.6,5-1.9,6.5s-2.9,2.3-5.1,2.3c-2.1,0-3.8-0.8-5-2.3s-1.9-3.6-1.9-6.3V113.4z M102.5,115.3c0,2.2,0.4,3.9,1.3,5.2
+					c0.9,1.3,2.1,1.9,3.7,1.9c1.6,0,2.8-0.6,3.7-1.8c0.8-1.2,1.3-2.9,1.3-5.2v-2c0-2.2-0.4-3.9-1.3-5.2c-0.9-1.3-2.1-1.9-3.7-1.9
+					c-1.5,0-2.7,0.6-3.6,1.8c-0.9,1.2-1.3,2.9-1.4,5.1V115.3z"/>
+				<path class="st0" d="M121.1,104.8l0.1,3c0.6-1,1.3-1.9,2.2-2.5s1.9-0.9,3-0.9c3.3,0,5,2.2,5.1,6.6v12.7h-1.9v-12.5
+					c0-1.7-0.3-3-0.9-3.8c-0.6-0.8-1.5-1.2-2.8-1.2c-1,0-1.9,0.4-2.8,1.2s-1.4,1.8-1.9,3.2v13.2h-2v-19H121.1z"/>
+				<path class="st0" d="M138.2,123.8v-17.3h-2.6v-1.8h2.6v-2.5c0-1.9,0.5-3.3,1.3-4.3s2-1.5,3.5-1.5c0.7,0,1.3,0.1,1.9,0.3
+					l-0.1,1.8c-0.5-0.1-1-0.2-1.6-0.2c-0.9,0-1.7,0.4-2.2,1.1s-0.8,1.7-0.8,3v2.3h3.7v1.8h-3.7v17.3H138.2z"/>
+				<path class="st0" d="M148,99.3c0-0.4,0.1-0.7,0.3-1s0.5-0.4,0.9-0.4s0.7,0.1,1,0.4s0.3,0.6,0.3,1s-0.1,0.7-0.3,1s-0.5,0.4-1,0.4
+					s-0.7-0.1-0.9-0.4S148,99.7,148,99.3z M150.2,123.8h-2v-19h2V123.8z"/>
+				<path class="st0" d="M155.3,113.5c0-3,0.5-5.2,1.5-6.7s2.6-2.2,4.6-2.2c2.2,0,3.8,1,4.9,3l0.1-2.7h1.8v19.6c0,2.3-0.6,4-1.6,5.2
+					c-1.1,1.2-2.7,1.8-4.7,1.8c-1,0-2.1-0.3-3.2-0.8s-1.9-1.1-2.4-1.9l0.9-1.4c1.3,1.5,2.8,2.2,4.5,2.2c1.6,0,2.7-0.4,3.5-1.3
+					c0.7-0.8,1.1-2.1,1.1-3.8v-3c-1.1,1.8-2.7,2.7-4.9,2.7c-2,0-3.5-0.7-4.5-2.2s-1.6-3.6-1.6-6.5V113.5z M157.2,115.2
+					c0,2.4,0.4,4.2,1.1,5.4s1.9,1.7,3.5,1.7c2.1,0,3.6-1,4.5-3v-9.7c-0.9-2.2-2.4-3.3-4.4-3.3c-1.6,0-2.8,0.6-3.5,1.7
+					s-1.1,2.9-1.1,5.3V115.2z"/>
+			</g>
+		</g>
+		<g>
+			<g>
+				<g>
+					<line class="st19" x1="58.9" y1="20.1" x2="77.9" y2="35.6"/>
+					<g>
+						<polygon class="st0" points="57.6,23.2 55.3,17.1 61.7,18.2 						"/>
+					</g>
+				</g>
+			</g>
+			<circle class="st0" cx="48" cy="10.1" r="10.1"/>
+			<g>
+				<g>
+					<line class="st19" x1="77.9" y1="35.1" x2="96.2" y2="66.7"/>
+					<g>
+						<polygon class="st0" points="93,67.5 98.6,70.7 98.6,64.2 						"/>
+					</g>
+				</g>
+			</g>
+			<g>
+				<g>
+					<line class="st19" x1="92.3" y1="38.9" x2="114.4" y2="43.6"/>
+					<g>
+						<polygon class="st0" points="92.6,42.3 87.8,38 93.9,36 						"/>
+					</g>
+				</g>
+			</g>
+			<g>
+				<g>
+					<line class="st19" x1="77.9" y1="35.6" x2="94.5" y2="25.1"/>
+					<g>
+						<polygon class="st0" points="95.4,28.3 98.4,22.6 91.9,22.9 						"/>
+					</g>
+				</g>
+			</g>
+			<g>
+				<g>
+					<line class="st19" x1="48.5" y1="41.3" x2="63.5" y2="38.7"/>
+					<g>
+						<polygon class="st0" points="63.1,42.1 68.1,38 62,35.7 						"/>
+					</g>
+				</g>
+			</g>
+			<g>
+				<g>
+					<line class="st19" x1="111.4" y1="66.3" x2="124.4" y2="45.2"/>
+					<g>
+						<polygon class="st0" points="114.6,67.2 109,70.2 109.1,63.8 						"/>
+					</g>
+				</g>
+			</g>
+			<g>
+				<g>
+					<line class="st19" x1="115.4" y1="30.4" x2="124.4" y2="45.2"/>
+					<g>
+						<polygon class="st0" points="113.2,32.8 113,26.4 118.7,29.5 						"/>
+					</g>
+				</g>
+			</g>
+			<circle class="st0" cx="77.9" cy="35.6" r="10.1"/>
+			<circle class="st0" cx="44.6" cy="42.4" r="10.1"/>
+			<circle class="st0" cx="107.4" cy="18.1" r="10.1"/>
+			<circle class="st0" cx="104.1" cy="79.1" r="10.1"/>
+			<circle class="st0" cx="124.4" cy="45.2" r="10.1"/>
+		</g>
+	</g>
+</g>
+</svg>

art/mgmt_logo_white_wide.svg

@@ -0,0 +1,132 @@
+<?xml version="1.0" encoding="utf-8"?>
+<svg version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 260.4 71.4" style="enable-background:new 0 0 260.4 71.4;" xml:space="preserve">
+<style type="text/css">
+	.st0{fill:#FFFFFF;}
+	.st1{fill:#E22434;}
+	.st2{display:none;}
+	.st3{fill:#1B3663;}
+	.st4{fill:#00B1D1;}
+	.st5{fill:#BFE6EF;}
+	.st6{fill:#69CBE0;}
+	.st7{fill:#0080BD;}
+	.st8{fill:#005DAB;}
+	.st9{fill:#183660;}
+	.st10{fill:none;stroke:#183660;stroke-width:1.9441;stroke-miterlimit:10;}
+	.st11{fill:none;stroke:#183660;stroke-width:1.6201;stroke-miterlimit:10;}
+	.st12{fill:none;stroke:#183660;stroke-width:1.2961;stroke-miterlimit:10;}
+	.st13{fill:none;stroke:#C0E6EF;stroke-width:1.9441;stroke-miterlimit:10;}
+	.st14{fill:#C0E6EF;}
+	.st15{fill:none;stroke:#C0E6EF;stroke-width:1.2961;stroke-miterlimit:10;}
+	.st16{fill:none;stroke:#C0E6EF;stroke-width:1.6201;stroke-miterlimit:10;}
+	.st17{fill:none;stroke:#FFFFFF;stroke-width:1.9441;stroke-miterlimit:10;}
+	.st18{fill:none;stroke:#FFFFFF;stroke-width:1.2961;stroke-miterlimit:10;}
+	.st19{fill:none;stroke:#FFFFFF;stroke-width:1.6201;stroke-miterlimit:10;}
+</style>
+<g id="Layer_2">
+</g>
+<g id="Layer_1">
+	<g>
+		<g>
+			<g>
+				<path class="st0" d="M96.7,26l0.1,1.8c1.1-1.4,2.6-2.1,4.4-2.1c1.9,0,3.2,0.9,4,2.6c1.1-1.7,2.6-2.6,4.7-2.6
+					c3.3,0,5,2.3,5.1,6.9V45h-5V32.9c0-1.1-0.2-1.9-0.5-2.4s-0.8-0.7-1.5-0.7c-0.9,0-1.6,0.6-2.1,1.7l0,0.6V45h-5V32.9
+					c0-1.1-0.1-1.9-0.4-2.4s-0.8-0.7-1.6-0.7c-0.9,0-1.5,0.5-2,1.4V45h-5V26H96.7z"/>
+				<path class="st0" d="M118.5,34.9c0-3.1,0.6-5.4,1.7-7s2.7-2.3,4.7-2.3c1.7,0,3.1,0.7,4,2l0.2-1.7h4.5v19c0,2.4-0.7,4.3-2,5.6
+					c-1.4,1.3-3.3,1.9-5.9,1.9c-1,0-2.1-0.2-3.3-0.6s-2-0.9-2.6-1.6l1.7-3.4c0.5,0.5,1.1,0.9,1.8,1.2c0.7,0.3,1.5,0.5,2.1,0.5
+					c1.1,0,1.9-0.3,2.4-0.8s0.7-1.4,0.7-2.6v-1.6c-0.9,1.2-2.2,1.9-3.7,1.9c-2,0-3.6-0.8-4.7-2.4s-1.7-3.8-1.7-6.7V34.9z
+					 M123.5,36.2c0,1.8,0.2,3,0.7,3.8s1.2,1.2,2.2,1.2c1,0,1.8-0.4,2.3-1.1V31c-0.5-0.8-1.3-1.2-2.2-1.2c-1,0-1.7,0.4-2.2,1.2
+					s-0.7,2.1-0.7,3.9V36.2z"/>
+				<path class="st0" d="M142.2,26l0.1,1.8c1.1-1.4,2.6-2.1,4.4-2.1c1.9,0,3.2,0.9,4,2.6c1.1-1.7,2.6-2.6,4.7-2.6
+					c3.3,0,5,2.3,5.1,6.9V45h-5V32.9c0-1.1-0.2-1.9-0.5-2.4c-0.3-0.5-0.8-0.7-1.5-0.7c-0.9,0-1.6,0.6-2.1,1.7l0,0.6V45h-5V32.9
+					c0-1.1-0.1-1.9-0.4-2.4c-0.3-0.5-0.8-0.7-1.6-0.7c-0.9,0-1.5,0.5-2,1.4V45h-5V26H142.2z"/>
+				<path class="st0" d="M170.3,21.3V26h2.5v3.7h-2.5v9.5c0,0.8,0.1,1.3,0.3,1.5c0.2,0.3,0.6,0.4,1.2,0.4c0.5,0,0.9,0,1.2-0.1l0,3.9
+					c-0.8,0.3-1.8,0.5-2.7,0.5c-3.2,0-4.8-1.8-4.9-5.5V29.7h-2.2V26h2.2v-4.7H170.3z"/>
+				<path class="st0" d="M182.7,43.5c1.4,0,2.4-0.4,3.1-1.1c0.7-0.8,1.1-1.9,1.2-3.3h1.9c-0.1,1.9-0.7,3.5-1.9,4.6s-2.6,1.7-4.3,1.7
+					c-2.3,0-4-0.7-5.1-2.2s-1.7-3.6-1.8-6.4v-2.3c0-2.9,0.6-5.1,1.7-6.6s2.9-2.2,5.1-2.2c1.9,0,3.4,0.6,4.5,1.8s1.7,2.8,1.7,5H187
+					c-0.1-1.6-0.5-2.8-1.2-3.6s-1.8-1.3-3.1-1.3c-1.7,0-2.9,0.6-3.7,1.7c-0.8,1.1-1.2,2.9-1.2,5.2v2.2c0,2.4,0.4,4.2,1.2,5.3
+					C179.8,43,181,43.5,182.7,43.5z"/>
+				<path class="st0" d="M192.6,34.5c0-2.7,0.6-4.9,1.9-6.5s2.9-2.4,5.1-2.4c2.2,0,3.9,0.8,5.1,2.4c1.2,1.6,1.9,3.7,1.9,6.5v2
+					c0,2.8-0.6,5-1.9,6.5s-2.9,2.3-5.1,2.3s-3.8-0.8-5-2.3s-1.9-3.6-1.9-6.3V34.5z M194.6,36.5c0,2.2,0.4,3.9,1.3,5.2
+					c0.9,1.3,2.1,1.9,3.7,1.9c1.6,0,2.8-0.6,3.7-1.8c0.8-1.2,1.3-2.9,1.3-5.2v-2c0-2.2-0.4-3.9-1.3-5.2c-0.9-1.3-2.1-1.9-3.7-1.9
+					c-1.5,0-2.7,0.6-3.6,1.8c-0.9,1.2-1.3,2.9-1.4,5.1V36.5z"/>
+				<path class="st0" d="M213.2,26l0.1,3c0.6-1,1.3-1.9,2.2-2.5s1.9-0.9,3-0.9c3.3,0,5,2.2,5.1,6.6V45h-1.9V32.5
+					c0-1.7-0.3-3-0.9-3.8c-0.6-0.8-1.5-1.2-2.8-1.2c-1,0-1.9,0.4-2.8,1.2s-1.4,1.8-1.9,3.2V45h-2V26H213.2z"/>
+				<path class="st0" d="M230.3,45V27.7h-2.6V26h2.6v-2.5c0-1.9,0.5-3.3,1.3-4.3s2-1.5,3.5-1.5c0.7,0,1.3,0.1,1.9,0.3l-0.1,1.8
+					c-0.5-0.1-1-0.2-1.6-0.2c-0.9,0-1.7,0.4-2.2,1.1s-0.8,1.7-0.8,3V26h3.7v1.8h-3.7V45H230.3z"/>
+				<path class="st0" d="M240.1,20.5c0-0.4,0.1-0.7,0.3-1s0.5-0.4,0.9-0.4s0.7,0.1,1,0.4s0.3,0.6,0.3,1s-0.1,0.7-0.3,1
+					s-0.5,0.4-1,0.4s-0.7-0.1-0.9-0.4S240.1,20.9,240.1,20.5z M242.3,45h-2V26h2V45z"/>
+				<path class="st0" d="M247.4,34.6c0-3,0.5-5.2,1.5-6.7s2.6-2.2,4.6-2.2c2.2,0,3.8,1,4.9,3l0.1-2.7h1.8v19.6c0,2.3-0.6,4-1.6,5.2
+					s-2.7,1.8-4.7,1.8c-1,0-2.1-0.3-3.2-0.8s-1.9-1.1-2.4-1.9l0.9-1.4c1.3,1.5,2.8,2.2,4.5,2.2c1.6,0,2.7-0.4,3.5-1.3
+					c0.7-0.8,1.1-2.1,1.1-3.8v-3c-1.1,1.8-2.7,2.7-4.9,2.7c-2,0-3.5-0.7-4.5-2.2s-1.6-3.6-1.6-6.5V34.6z M249.3,36.4
+					c0,2.4,0.4,4.2,1.1,5.4s1.9,1.7,3.5,1.7c2.1,0,3.6-1,4.5-3v-9.7c-0.9-2.2-2.4-3.3-4.4-3.3c-1.6,0-2.8,0.6-3.5,1.7
+					s-1.1,2.9-1.1,5.3V36.4z"/>
+			</g>
+		</g>
+		<g>
+			<g>
+				<g>
+					<line class="st18" x1="19.5" y1="16" x2="34.7" y2="28.5"/>
+					<g>
+						<polygon class="st0" points="18.4,18.5 16.6,13.7 21.7,14.5 						"/>
+					</g>
+				</g>
+			</g>
+			<circle class="st0" cx="10.8" cy="8.1" r="8.1"/>
+			<g>
+				<g>
+					<line class="st18" x1="34.7" y1="28" x2="49.4" y2="53.3"/>
+					<g>
+						<polygon class="st0" points="46.8,54 51.2,56.5 51.2,51.4 						"/>
+					</g>
+				</g>
+			</g>
+			<g>
+				<g>
+					<line class="st18" x1="46.2" y1="31.1" x2="63.9" y2="34.9"/>
+					<g>
+						<polygon class="st0" points="46.4,33.8 42.6,30.4 47.5,28.8 						"/>
+					</g>
+				</g>
+			</g>
+			<g>
+				<g>
+					<line class="st18" x1="34.7" y1="28.5" x2="47.9" y2="20.1"/>
+					<g>
+						<polygon class="st0" points="48.7,22.7 51.1,18.1 45.9,18.3 						"/>
+					</g>
+				</g>
+			</g>
+			<g>
+				<g>
+					<line class="st18" x1="11.2" y1="33.1" x2="23.2" y2="31"/>
+					<g>
+						<polygon class="st0" points="22.9,33.7 26.8,30.4 22,28.6 						"/>
+					</g>
+				</g>
+			</g>
+			<g>
+				<g>
+					<line class="st18" x1="61.5" y1="53" x2="71.9" y2="36.1"/>
+					<g>
+						<polygon class="st0" points="64.1,53.7 59.5,56.2 59.7,51 						"/>
+					</g>
+				</g>
+			</g>
+			<g>
+				<g>
+					<line class="st18" x1="64.7" y1="24.4" x2="71.9" y2="36.6"/>
+					<g>
+						<polygon class="st0" points="62.9,26.4 62.9,21.2 67.3,23.7 						"/>
+					</g>
+				</g>
+			</g>
+			<circle class="st0" cx="34.7" cy="28.5" r="8.1"/>
+			<circle class="st0" cx="8.1" cy="33.9" r="8.1"/>
+			<circle class="st0" cx="58.3" cy="14.5" r="8.1"/>
+			<circle class="st0" cx="55.7" cy="63.3" r="8.1"/>
+			<circle class="st0" cx="71.9" cy="36.1" r="8.1"/>
+		</g>
+	</g>
+</g>
+</svg>

config.go

@@ -1,535 +0,0 @@
-// Mgmt
-// Copyright (C) 2013-2016+ James Shubin and the project contributors
-// Written by James Shubin <james@shubin.ca> and the project contributors
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package main
-
-import (
-	"errors"
-	"fmt"
-	"gopkg.in/yaml.v2"
-	"io/ioutil"
-	"log"
-	"reflect"
-	"strings"
-)
-
-type collectorResConfig struct {
-	Kind    string `yaml:"kind"`
-	Pattern string `yaml:"pattern"` // XXX: Not Implemented
-}
-
-type vertexConfig struct {
-	Kind string `yaml:"kind"`
-	Name string `yaml:"name"`
-}
-
-type edgeConfig struct {
-	Name string       `yaml:"name"`
-	From vertexConfig `yaml:"from"`
-	To   vertexConfig `yaml:"to"`
-}
-
-type GraphConfig struct {
-	Graph     string `yaml:"graph"`
-	Resources struct {
-		Noop []*NoopRes `yaml:"noop"`
-		Pkg  []*PkgRes  `yaml:"pkg"`
-		File []*FileRes `yaml:"file"`
-		Svc  []*SvcRes  `yaml:"svc"`
-		Exec []*ExecRes `yaml:"exec"`
-	} `yaml:"resources"`
-	Collector []collectorResConfig `yaml:"collect"`
-	Edges     []edgeConfig         `yaml:"edges"`
-	Comment   string               `yaml:"comment"`
-}
-
-func (c *GraphConfig) Parse(data []byte) error {
-	if err := yaml.Unmarshal(data, c); err != nil {
-		return err
-	}
-	if c.Graph == "" {
-		return errors.New("Graph config: invalid `graph`")
-	}
-	return nil
-}
-
-func ParseConfigFromFile(filename string) *GraphConfig {
-	data, err := ioutil.ReadFile(filename)
-	if err != nil {
-		log.Printf("Error: Config: ParseConfigFromFile: File: %v", err)
-		return nil
-	}
-
-	var config GraphConfig
-	if err := config.Parse(data); err != nil {
-		log.Printf("Error: Config: ParseConfigFromFile: Parse: %v", err)
-		return nil
-	}
-
-	return &config
-}
-
-// NewGraphFromConfig returns a new graph from existing input, such as from the
-// existing graph, and a GraphConfig struct.
-func (g *Graph) NewGraphFromConfig(config *GraphConfig, etcdO *EtcdWObject, hostname string) (*Graph, error) {
-
-	var graph *Graph // new graph to return
-	if g == nil {    // FIXME: how can we check for an empty graph?
-		graph = NewGraph("Graph") // give graph a default name
-	} else {
-		graph = g.Copy() // same vertices, since they're pointers!
-	}
-
-	var lookup = make(map[string]map[string]*Vertex)
-
-	//log.Printf("%+v", config) // debug
-
-	// TODO: if defined (somehow)...
-	graph.SetName(config.Graph) // set graph name
-
-	var keep []*Vertex // list of vertex which are the same in new graph
-
-	// use reflection to avoid duplicating code... better options welcome!
-	value := reflect.Indirect(reflect.ValueOf(config.Resources))
-	vtype := value.Type()
-	for i := 0; i < vtype.NumField(); i++ { // number of fields in struct
-		name := vtype.Field(i).Name // string of field name
-		field := value.FieldByName(name)
-		iface := field.Interface() // interface type of value
-		slice := reflect.ValueOf(iface)
-		// XXX: should we just drop these everywhere and have the kind strings be all lowercase?
-		kind := FirstToUpper(name)
-		if DEBUG {
-			log.Printf("Config: Processing: %v...", kind)
-		}
-		for j := 0; j < slice.Len(); j++ { // loop through resources of same kind
-			x := slice.Index(j).Interface()
-			obj, ok := x.(Res) // convert to Res type
-			if !ok {
-				return nil, fmt.Errorf("Error: Config: Can't convert: %v of type: %T to Res.", x, x)
-			}
-
-			if _, exists := lookup[kind]; !exists {
-				lookup[kind] = make(map[string]*Vertex)
-			}
-			// XXX: should we export based on a @@ prefix, or a metaparam
-			// like exported => true || exported => (host pattern)||(other pattern?)
-			if !strings.HasPrefix(obj.GetName(), "@@") { // exported resource
-				// XXX: we don't have a way of knowing if any of the
-				// metaparams are undefined, and as a result to set the
-				// defaults that we want! I hate the go yaml parser!!!
-				v := graph.GetVertexMatch(obj)
-				if v == nil { // no match found
-					obj.Init()
-					v = NewVertex(obj)
-					graph.AddVertex(v) // call standalone in case not part of an edge
-				}
-				lookup[kind][obj.GetName()] = v // used for constructing edges
-				keep = append(keep, v)          // append
-
-			} else {
-				// XXX: do this in a different function...
-				// add to etcd storage...
-				obj.SetName(obj.GetName()[2:]) //slice off @@
-
-				data, err := ResToB64(obj)
-				if err != nil {
-					return nil, fmt.Errorf("Config: Could not encode %v resource: %v, error: %v", kind, obj.GetName(), err)
-				}
-
-				if !etcdO.EtcdPut(hostname, obj.GetName(), kind, data) {
-					return nil, fmt.Errorf("Config: Could not export %v resource: %v", kind, obj.GetName())
-				}
-			}
-		}
-	}
-
-	// lookup from etcd graph
-	// do all the graph look ups in one single step, so that if the etcd
-	// database changes, we don't have a partial state of affairs...
-	nodes, ok := etcdO.EtcdGet()
-	if ok {
-		for _, t := range config.Collector {
-			// XXX: should we just drop these everywhere and have the kind strings be all lowercase?
-			kind := FirstToUpper(t.Kind)
-
-			// use t.Kind and optionally t.Pattern to collect from etcd storage
-			log.Printf("Collect: %v; Pattern: %v", kind, t.Pattern)
-			for _, str := range etcdO.EtcdGetProcess(nodes, kind) {
-				obj, err := B64ToRes(str)
-				if err != nil {
-					log.Printf("B64ToRes failed to decode: %v", err)
-					log.Printf("Collect: %v: not collected!", kind)
-					continue
-				}
-
-				if t.Pattern != "" { // XXX: simplistic for now
-					obj.CollectPattern(t.Pattern) // obj.Dirname = t.Pattern
-				}
-
-				log.Printf("Collect: %v[%v]: collected!", kind, obj.GetName())
-
-				// XXX: similar to other resource add code:
-				if _, exists := lookup[kind]; !exists {
-					lookup[kind] = make(map[string]*Vertex)
-				}
-				v := graph.GetVertexMatch(obj)
-				if v == nil { // no match found
-					obj.Init() // initialize go channels or things won't work!!!
-					v = NewVertex(obj)
-					graph.AddVertex(v) // call standalone in case not part of an edge
-				}
-				lookup[kind][obj.GetName()] = v // used for constructing edges
-				keep = append(keep, v)          // append
-			}
-		}
-	}
-
-	// get rid of any vertices we shouldn't "keep" (that aren't in new graph)
-	for _, v := range graph.GetVertices() {
-		if !VertexContains(v, keep) {
-			// wait for exit before starting new graph!
-			v.SendEvent(eventExit, true, false)
-			graph.DeleteVertex(v)
-		}
-	}
-
-	for _, e := range config.Edges {
-		if _, ok := lookup[FirstToUpper(e.From.Kind)]; !ok {
-			return nil, fmt.Errorf("Can't find 'from' resource!")
-		}
-		if _, ok := lookup[FirstToUpper(e.To.Kind)]; !ok {
-			return nil, fmt.Errorf("Can't find 'to' resource!")
-		}
-		if _, ok := lookup[FirstToUpper(e.From.Kind)][e.From.Name]; !ok {
-			return nil, fmt.Errorf("Can't find 'from' name!")
-		}
-		if _, ok := lookup[FirstToUpper(e.To.Kind)][e.To.Name]; !ok {
-			return nil, fmt.Errorf("Can't find 'to' name!")
-		}
-		graph.AddEdge(lookup[FirstToUpper(e.From.Kind)][e.From.Name], lookup[FirstToUpper(e.To.Kind)][e.To.Name], NewEdge(e.Name))
-	}
-
-	return graph, nil
-}
-
-// add edges to the vertex in a graph based on if it matches a uuid list
-func (g *Graph) addEdgesByMatchingUUIDS(v *Vertex, uuids []ResUUID) []bool {
-	// search for edges and see what matches!
-	var result []bool
-
-	// loop through each uuid, and see if it matches any vertex
-	for _, uuid := range uuids {
-		var found = false
-		// uuid is a ResUUID object
-		for _, vv := range g.GetVertices() { // search
-			if v == vv { // skip self
-				continue
-			}
-			if DEBUG {
-				log.Printf("Compile: AutoEdge: Match: %v[%v] with UUID: %v[%v]", vv.Kind(), vv.GetName(), uuid.Kind(), uuid.GetName())
-			}
-			// we must match to an effective UUID for the resource,
-			// that is to say, the name value of a res is a helpful
-			// handle, but it is not necessarily a unique identity!
-			// remember, resources can return multiple UUID's each!
-			if UUIDExistsInUUIDs(uuid, vv.GetUUIDs()) {
-				// add edge from: vv -> v
-				if uuid.Reversed() {
-					txt := fmt.Sprintf("AutoEdge: %v[%v] -> %v[%v]", vv.Kind(), vv.GetName(), v.Kind(), v.GetName())
-					log.Printf("Compile: Adding %v", txt)
-					g.AddEdge(vv, v, NewEdge(txt))
-				} else { // edges go the "normal" way, eg: pkg resource
-					txt := fmt.Sprintf("AutoEdge: %v[%v] -> %v[%v]", v.Kind(), v.GetName(), vv.Kind(), vv.GetName())
-					log.Printf("Compile: Adding %v", txt)
-					g.AddEdge(v, vv, NewEdge(txt))
-				}
-				found = true
-				break
-			}
-		}
-		result = append(result, found)
-	}
-	return result
-}
-
-// add auto edges to graph
-func (g *Graph) AutoEdges() {
-	log.Println("Compile: Adding AutoEdges...")
-	for _, v := range g.GetVertices() { // for each vertexes autoedges
-		if !v.GetMeta().AutoEdge { // is the metaparam true?
-			continue
-		}
-		autoEdgeObj := v.AutoEdges()
-		if autoEdgeObj == nil {
-			log.Printf("%v[%v]: Config: No auto edges were found!", v.Kind(), v.GetName())
-			continue // next vertex
-		}
-
-		for { // while the autoEdgeObj has more uuids to add...
-			uuids := autoEdgeObj.Next() // get some!
-			if uuids == nil {
-				log.Printf("%v[%v]: Config: The auto edge list is empty!", v.Kind(), v.GetName())
-				break // inner loop
-			}
-			if DEBUG {
-				log.Println("Compile: AutoEdge: UUIDS:")
-				for i, u := range uuids {
-					log.Printf("Compile: AutoEdge: UUID%d: %v", i, u)
-				}
-			}
-
-			// match and add edges
-			result := g.addEdgesByMatchingUUIDS(v, uuids)
-
-			// report back, and find out if we should continue
-			if !autoEdgeObj.Test(result) {
-				break
-			}
-		}
-	}
-}
-
-// AutoGrouper is the required interface to implement for an autogroup algorithm
-type AutoGrouper interface {
-	// listed in the order these are typically called in...
-	name() string                                  // friendly identifier
-	init(*Graph) error                             // only call once
-	vertexNext() (*Vertex, *Vertex, error)         // mostly algorithmic
-	vertexCmp(*Vertex, *Vertex) error              // can we merge these ?
-	vertexMerge(*Vertex, *Vertex) (*Vertex, error) // vertex merge fn to use
-	edgeMerge(*Edge, *Edge) *Edge                  // edge merge fn to use
-	vertexTest(bool) (bool, error)                 // call until false
-}
-
-// baseGrouper is the base type for implementing the AutoGrouper interface
-type baseGrouper struct {
-	graph    *Graph    // store a pointer to the graph
-	vertices []*Vertex // cached list of vertices
-	i        int
-	j        int
-	done     bool
-}
-
-// name provides a friendly name for the logs to see
-func (ag *baseGrouper) name() string {
-	return "baseGrouper"
-}
-
-// init is called only once and before using other AutoGrouper interface methods
-// the name method is the only exception: call it any time without side effects!
-func (ag *baseGrouper) init(g *Graph) error {
-	if ag.graph != nil {
-		return fmt.Errorf("The init method has already been called!")
-	}
-	ag.graph = g                               // pointer
-	ag.vertices = ag.graph.GetVerticesSorted() // cache in deterministic order!
-	ag.i = 0
-	ag.j = 0
-	if len(ag.vertices) == 0 { // empty graph
-		ag.done = true
-		return nil
-	}
-	return nil
-}
-
-// vertexNext is a simple iterator that loops through vertex (pair) combinations
-// an intelligent algorithm would selectively offer only valid pairs of vertices
-// these should satisfy logical grouping requirements for the autogroup designs!
-// the desired algorithms can override, but keep this method as a base iterator!
-func (ag *baseGrouper) vertexNext() (v1, v2 *Vertex, err error) {
-	// this does a for v... { for w... { return v, w }} but stepwise!
-	l := len(ag.vertices)
-	if ag.i < l {
-		v1 = ag.vertices[ag.i]
-	}
-	if ag.j < l {
-		v2 = ag.vertices[ag.j]
-	}
-
-	// in case the vertex was deleted
-	if !ag.graph.HasVertex(v1) {
-		v1 = nil
-	}
-	if !ag.graph.HasVertex(v2) {
-		v2 = nil
-	}
-
-	// two nested loops...
-	if ag.j < l {
-		ag.j++
-	}
-	if ag.j == l {
-		ag.j = 0
-		if ag.i < l {
-			ag.i++
-		}
-		if ag.i == l {
-			ag.done = true
-		}
-	}
-
-	return
-}
-
-func (ag *baseGrouper) vertexCmp(v1, v2 *Vertex) error {
-	if v1 == nil || v2 == nil {
-		return fmt.Errorf("Vertex is nil!")
-	}
-	if v1 == v2 { // skip yourself
-		return fmt.Errorf("Vertices are the same!")
-	}
-	if v1.Kind() != v2.Kind() { // we must group similar kinds
-		// TODO: maybe future resources won't need this limitation?
-		return fmt.Errorf("The two resources aren't the same kind!")
-	}
-	// someone doesn't want to group!
-	if !v1.GetMeta().AutoGroup || !v2.GetMeta().AutoGroup {
-		return fmt.Errorf("One of the autogroup flags is false!")
-	}
-	if v1.Res.IsGrouped() { // already grouped!
-		return fmt.Errorf("Already grouped!")
-	}
-	if len(v2.Res.GetGroup()) > 0 { // already has children grouped!
-		return fmt.Errorf("Already has groups!")
-	}
-	if !v1.Res.GroupCmp(v2.Res) { // resource groupcmp failed!
-		return fmt.Errorf("The GroupCmp failed!")
-	}
-	return nil // success
-}
-
-func (ag *baseGrouper) vertexMerge(v1, v2 *Vertex) (v *Vertex, err error) {
-	// NOTE: it's important to use w.Res instead of w, b/c
-	// the w by itself is the *Vertex obj, not the *Res obj
-	// which is contained within it! They both satisfy the
-	// Res interface, which is why both will compile! :(
-	err = v1.Res.GroupRes(v2.Res) // GroupRes skips stupid groupings
-	return                        // success or fail, and no need to merge the actual vertices!
-}
-
-func (ag *baseGrouper) edgeMerge(e1, e2 *Edge) *Edge {
-	return e1 // noop
-}
-
-// vertexTest processes the results of the grouping for the algorithm to know
-// return an error if something went horribly wrong, and bool false to stop
-func (ag *baseGrouper) vertexTest(b bool) (bool, error) {
-	// NOTE: this particular baseGrouper version doesn't track what happens
-	// because since we iterate over every pair, we don't care which merge!
-	if ag.done {
-		return false, nil
-	}
-	return true, nil
-}
-
-// TODO: this algorithm may not be correct in all cases. replace if needed!
-type nonReachabilityGrouper struct {
-	baseGrouper // "inherit" what we want, and reimplement the rest
-}
-
-func (ag *nonReachabilityGrouper) name() string {
-	return "nonReachabilityGrouper"
-}
-
-// this algorithm relies on the observation that if there's a path from a to b,
-// then they *can't* be merged (b/c of the existing dependency) so therefore we
-// merge anything that *doesn't* satisfy this condition or that of the reverse!
-func (ag *nonReachabilityGrouper) vertexNext() (v1, v2 *Vertex, err error) {
-	for {
-		v1, v2, err = ag.baseGrouper.vertexNext() // get all iterable pairs
-		if err != nil {
-			log.Fatalf("Error running autoGroup(vertexNext): %v", err)
-		}
-
-		if v1 != v2 { // ignore self cmp early (perf optimization)
-			// if NOT reachable, they're viable...
-			out1 := ag.graph.Reachability(v1, v2)
-			out2 := ag.graph.Reachability(v2, v1)
-			if len(out1) == 0 && len(out2) == 0 {
-				return // return v1 and v2, they're viable
-			}
-		}
-
-		// if we got here, it means we're skipping over this candidate!
-		if ok, err := ag.baseGrouper.vertexTest(false); err != nil {
-			log.Fatalf("Error running autoGroup(vertexTest): %v", err)
-		} else if !ok {
-			return nil, nil, nil // done!
-		}
-
-		// the vertexTest passed, so loop and try with a new pair...
-	}
-}
-
-// autoGroup is the mechanical auto group "runner" that runs the interface spec
-func (g *Graph) autoGroup(ag AutoGrouper) chan string {
-	strch := make(chan string) // output log messages here
-	go func(strch chan string) {
-		strch <- fmt.Sprintf("Compile: Grouping: Algorithm: %v...", ag.name())
-		if err := ag.init(g); err != nil {
-			log.Fatalf("Error running autoGroup(init): %v", err)
-		}
-
-		for {
-			var v, w *Vertex
-			v, w, err := ag.vertexNext() // get pair to compare
-			if err != nil {
-				log.Fatalf("Error running autoGroup(vertexNext): %v", err)
-			}
-			merged := false
-			// save names since they change during the runs
-			vStr := fmt.Sprintf("%s", v) // valid even if it is nil
-			wStr := fmt.Sprintf("%s", w)
-
-			if err := ag.vertexCmp(v, w); err != nil { // cmp ?
-				if DEBUG {
-					strch <- fmt.Sprintf("Compile: Grouping: !GroupCmp for: %s into %s", wStr, vStr)
-				}
-
-				// remove grouped vertex and merge edges (res is safe)
-			} else if err := g.VertexMerge(v, w, ag.vertexMerge, ag.edgeMerge); err != nil { // merge...
-				strch <- fmt.Sprintf("Compile: Grouping: !VertexMerge for: %s into %s", wStr, vStr)
-
-			} else { // success!
-				strch <- fmt.Sprintf("Compile: Grouping: Success for: %s into %s", wStr, vStr)
-				merged = true // woo
-			}
-
-			// did these get used?
-			if ok, err := ag.vertexTest(merged); err != nil {
-				log.Fatalf("Error running autoGroup(vertexTest): %v", err)
-			} else if !ok {
-				break // done!
-			}
-		}
-
-		close(strch)
-		return
-	}(strch) // call function
-	return strch
-}
-
-// AutoGroup runs the auto grouping on the graph and prints out log messages
-func (g *Graph) AutoGroup() {
-	// receive log messages from channel...
-	// this allows test cases to avoid printing them when they're unwanted!
-	// TODO: this algorithm may not be correct in all cases. replace if needed!
-	for str := range g.autoGroup(&nonReachabilityGrouper{}) {
-		log.Println(str)
-	}
-}

configwatch.go

@@ -1,155 +0,0 @@
-// Mgmt
-// Copyright (C) 2013-2016+ James Shubin and the project contributors
-// Written by James Shubin <james@shubin.ca> and the project contributors
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package main
-
-import (
-	"gopkg.in/fsnotify.v1"
-	//"github.com/go-fsnotify/fsnotify" // git master of "gopkg.in/fsnotify.v1"
-	"log"
-	"math"
-	"path"
-	"strings"
-	"syscall"
-)
-
-// XXX: it would be great if we could reuse code between this and the file resource
-// XXX: patch this to submit it as part of go-fsnotify if they're interested...
-func ConfigWatch(file string) chan bool {
-	ch := make(chan bool)
-	go func() {
-		var safename = path.Clean(file) // no trailing slash
-
-		watcher, err := fsnotify.NewWatcher()
-		if err != nil {
-			log.Fatal(err)
-		}
-		defer watcher.Close()
-
-		patharray := PathSplit(safename) // tokenize the path
-		var index = len(patharray)       // starting index
-		var current string               // current "watcher" location
-		var deltaDepth int               // depth delta between watcher and event
-		var send = false                 // send event?
-
-		for {
-			current = strings.Join(patharray[0:index], "/")
-			if current == "" { // the empty string top is the root dir ("/")
-				current = "/"
-			}
-			log.Printf("Watching: %v", current) // attempting to watch...
-
-			// initialize in the loop so that we can reset on rm-ed handles
-			err = watcher.Add(current)
-			if err != nil {
-				if err == syscall.ENOENT {
-					index-- // usually not found, move up one dir
-				} else if err == syscall.ENOSPC {
-					// XXX: occasionally: no space left on device,
-					// XXX: probably due to lack of inotify watches
-					log.Printf("Out of inotify watches for config(%v)", file)
-					log.Fatal(err)
-				} else {
-					log.Printf("Unknown config(%v) error:", file)
-					log.Fatal(err)
-				}
-				index = int(math.Max(1, float64(index)))
-				continue
-			}
-
-			select {
-			case event := <-watcher.Events:
-				// the deeper you go, the bigger the deltaDepth is...
-				// this is the difference between what we're watching,
-				// and the event... doesn't mean we can't watch deeper
-				if current == event.Name {
-					deltaDepth = 0 // i was watching what i was looking for
-
-				} else if HasPathPrefix(event.Name, current) {
-					deltaDepth = len(PathSplit(current)) - len(PathSplit(event.Name)) // -1 or less
-
-				} else if HasPathPrefix(current, event.Name) {
-					deltaDepth = len(PathSplit(event.Name)) - len(PathSplit(current)) // +1 or more
-
-				} else {
-					// TODO different watchers get each others events!
-					// https://github.com/go-fsnotify/fsnotify/issues/95
-					// this happened with two values such as:
-					// event.Name: /tmp/mgmt/f3 and current: /tmp/mgmt/f2
-					continue
-				}
-				//log.Printf("The delta depth is: %v", deltaDepth)
-
-				// if we have what we wanted, awesome, send an event...
-				if event.Name == safename {
-					//log.Println("Event!")
-					send = true
-
-					// file removed, move the watch upwards
-					if deltaDepth >= 0 && (event.Op&fsnotify.Remove == fsnotify.Remove) {
-						//log.Println("Removal!")
-						watcher.Remove(current)
-						index--
-					}
-
-					// we must be a parent watcher, so descend in
-					if deltaDepth < 0 {
-						watcher.Remove(current)
-						index++
-					}
-
-					// if safename starts with event.Name, we're above, and no event should be sent
-				} else if HasPathPrefix(safename, event.Name) {
-					//log.Println("Above!")
-
-					if deltaDepth >= 0 && (event.Op&fsnotify.Remove == fsnotify.Remove) {
-						log.Println("Removal!")
-						watcher.Remove(current)
-						index--
-					}
-
-					if deltaDepth < 0 {
-						log.Println("Parent!")
-						if PathPrefixDelta(safename, event.Name) == 1 { // we're the parent dir
-							//send = true
-						}
-						watcher.Remove(current)
-						index++
-					}
-
-					// if event.Name startswith safename, send event, we're already deeper
-				} else if HasPathPrefix(event.Name, safename) {
-					//log.Println("Event2!")
-					//send = true
-				}
-
-			case err := <-watcher.Errors:
-				log.Printf("error: %v", err)
-				log.Fatal(err)
-
-			}
-
-			// do our event sending all together to avoid duplicate msgs
-			if send {
-				send = false
-				ch <- true
-			}
-		}
-		//close(ch)
-	}()
-	return ch
-}

converger/converger.go

@@ -0,0 +1,382 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// Package converger is a facility for reporting the converged state.
+package converger
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/purpleidea/mgmt/util"
+)
+
+// TODO: we could make a new function that masks out the state of certain
+// UID's, but at the moment the new Timer code has obsoleted the need...
+
+// Converger is the general interface for implementing a convergence watcher
+type Converger interface { // TODO: need a better name
+	Register() ConvergerUID
+	IsConverged(ConvergerUID) bool         // is the UID converged ?
+	SetConverged(ConvergerUID, bool) error // set the converged state of the UID
+	Unregister(ConvergerUID)
+	Start()
+	Pause()
+	Loop(bool)
+	ConvergedTimer(ConvergerUID) <-chan time.Time
+	Status() map[uint64]bool
+	Timeout() int                // returns the timeout that this was created with
+	SetStateFn(func(bool) error) // sets the stateFn
+}
+
+// ConvergerUID is the interface resources can use to notify with if converged
+// you'll need to use part of the Converger interface to Register initially too
+type ConvergerUID interface {
+	ID() uint64   // get Id
+	Name() string // get a friendly name
+	SetName(string)
+	IsValid() bool // has Id been initialized ?
+	InvalidateID() // set Id to nil
+	IsConverged() bool
+	SetConverged(bool) error
+	Unregister()
+	ConvergedTimer() <-chan time.Time
+	StartTimer() (func() error, error) // cancellable is the same as StopTimer()
+	ResetTimer() error                 // resets counter to zero
+	StopTimer() error
+}
+
+// converger is an implementation of the Converger interface
+type converger struct {
+	timeout   int              // must be zero (instant) or greater seconds to run
+	stateFn   func(bool) error // run on converged state changes with state bool
+	converged bool             // did we converge (state changes of this run Fn)
+	channel   chan struct{}    // signal here to run an isConverged check
+	control   chan bool        // control channel for start/pause
+	mutex     sync.RWMutex     // used for controlling access to status and lastid
+	lastid    uint64
+	status    map[uint64]bool
+}
+
+// convergerUID is an implementation of the ConvergerUID interface
+type convergerUID struct {
+	converger Converger
+	id        uint64
+	name      string // user defined, friendly name
+	mutex     sync.Mutex
+	timer     chan struct{}
+	running   bool // is the above timer running?
+}
+
+// NewConverger builds a new converger struct
+func NewConverger(timeout int, stateFn func(bool) error) *converger {
+	return &converger{
+		timeout: timeout,
+		stateFn: stateFn,
+		channel: make(chan struct{}),
+		control: make(chan bool),
+		lastid:  0,
+		status:  make(map[uint64]bool),
+	}
+}
+
+// Register assigns a ConvergerUID to the caller
+func (obj *converger) Register() ConvergerUID {
+	obj.mutex.Lock()
+	defer obj.mutex.Unlock()
+	obj.lastid++
+	obj.status[obj.lastid] = false // initialize as not converged
+	return &convergerUID{
+		converger: obj,
+		id:        obj.lastid,
+		name:      fmt.Sprintf("%d", obj.lastid), // some default
+		timer:     nil,
+		running:   false,
+	}
+}
+
+// IsConverged gets the converged status of a uid
+func (obj *converger) IsConverged(uid ConvergerUID) bool {
+	if !uid.IsValid() {
+		panic(fmt.Sprintf("Id of ConvergerUID(%s) is nil!", uid.Name()))
+	}
+	obj.mutex.RLock()
+	isConverged, found := obj.status[uid.ID()] // lookup
+	obj.mutex.RUnlock()
+	if !found {
+		panic("Id of ConvergerUID is unregistered!")
+	}
+	return isConverged
+}
+
+// SetConverged updates the converger with the converged state of the UID
+func (obj *converger) SetConverged(uid ConvergerUID, isConverged bool) error {
+	if !uid.IsValid() {
+		return fmt.Errorf("Id of ConvergerUID(%s) is nil!", uid.Name())
+	}
+	obj.mutex.Lock()
+	if _, found := obj.status[uid.ID()]; !found {
+		panic("Id of ConvergerUID is unregistered!")
+	}
+	obj.status[uid.ID()] = isConverged // set
+	obj.mutex.Unlock()                 // unlock *before* poke or deadlock!
+	if isConverged != obj.converged {  // only poke if it would be helpful
+		// run in a go routine so that we never block... just queue up!
+		// this allows us to send events, even if we haven't started...
+		go func() { obj.channel <- struct{}{} }()
+	}
+	return nil
+}
+
+// isConverged returns true if *every* registered uid has converged
+func (obj *converger) isConverged() bool {
+	obj.mutex.RLock() // take a read lock
+	defer obj.mutex.RUnlock()
+	for _, v := range obj.status {
+		if !v { // everyone must be converged for this to be true
+			return false
+		}
+	}
+	return true
+}
+
+// Unregister dissociates the ConvergedUID from the converged checking
+func (obj *converger) Unregister(uid ConvergerUID) {
+	if !uid.IsValid() {
+		panic(fmt.Sprintf("Id of ConvergerUID(%s) is nil!", uid.Name()))
+	}
+	obj.mutex.Lock()
+	uid.StopTimer() // ignore any errors
+	delete(obj.status, uid.ID())
+	obj.mutex.Unlock()
+	uid.InvalidateID()
+}
+
+// Start causes a Converger object to start or resume running
+func (obj *converger) Start() {
+	obj.control <- true
+}
+
+// Pause causes a Converger object to stop running temporarily
+func (obj *converger) Pause() { // FIXME: add a sync ACK on pause before return
+	obj.control <- false
+}
+
+// Loop is the main loop for a Converger object; it usually runs in a goroutine
+// TODO: we could eventually have each resource tell us as soon as it converges
+// and then keep track of the time delays here, to avoid callers needing select
+// NOTE: when we have very short timeouts, if we start before all the resources
+// have joined the map, then it might appears as if we converged before we did!
+func (obj *converger) Loop(startPaused bool) {
+	if obj.control == nil {
+		panic("Converger not initialized correctly")
+	}
+	if startPaused { // start paused without racing
+		select {
+		case e := <-obj.control:
+			if !e {
+				panic("Converger expected true!")
+			}
+		}
+	}
+	for {
+		select {
+		case e := <-obj.control: // expecting "false" which means pause!
+			if e {
+				panic("Converger expected false!")
+			}
+			// now i'm paused...
+			select {
+			case e := <-obj.control:
+				if !e {
+					panic("Converger expected true!")
+				}
+				// restart
+				// kick once to refresh the check...
+				go func() { obj.channel <- struct{}{} }()
+				continue
+			}
+
+		case <-obj.channel:
+			if !obj.isConverged() {
+				if obj.converged { // we're doing a state change
+					if obj.stateFn != nil {
+						// call an arbitrary function
+						if err := obj.stateFn(false); err != nil {
+							// FIXME: what to do on error ?
+						}
+					}
+				}
+				obj.converged = false
+				continue
+			}
+
+			// we have converged!
+			if obj.timeout >= 0 { // only run if timeout is valid
+				if !obj.converged { // we're doing a state change
+					if obj.stateFn != nil {
+						// call an arbitrary function
+						if err := obj.stateFn(true); err != nil {
+							// FIXME: what to do on error ?
+						}
+					}
+				}
+			}
+			obj.converged = true
+			// loop and wait again...
+		}
+	}
+}
+
+// ConvergedTimer adds a timeout to a select call and blocks until then
+// TODO: this means we could eventually have per resource converged timeouts
+func (obj *converger) ConvergedTimer(uid ConvergerUID) <-chan time.Time {
+	// be clever: if i'm already converged, this timeout should block which
+	// avoids unnecessary new signals being sent! this avoids fast loops if
+	// we have a low timeout, or in particular a timeout == 0
+	if uid.IsConverged() {
+		// blocks the case statement in select forever!
+		return util.TimeAfterOrBlock(-1)
+	}
+	return util.TimeAfterOrBlock(obj.timeout)
+}
+
+// Status returns a map of the converged status of each UID.
+func (obj *converger) Status() map[uint64]bool {
+	status := make(map[uint64]bool)
+	obj.mutex.RLock() // take a read lock
+	defer obj.mutex.RUnlock()
+	for k, v := range obj.status { // make a copy to avoid the mutex
+		status[k] = v
+	}
+	return status
+}
+
+// Timeout returns the timeout in seconds that converger was created with. This
+// is useful to avoid passing in the timeout value separately when you're
+// already passing in the Converger struct.
+func (obj *converger) Timeout() int {
+	return obj.timeout
+}
+
+// SetStateFn sets the state function to be run on change of converged state.
+func (obj *converger) SetStateFn(stateFn func(bool) error) {
+	obj.stateFn = stateFn
+}
+
+// Id returns the unique id of this UID object
+func (obj *convergerUID) ID() uint64 {
+	return obj.id
+}
+
+// Name returns a user defined name for the specific convergerUID.
+func (obj *convergerUID) Name() string {
+	return obj.name
+}
+
+// SetName sets a user defined name for the specific convergerUID.
+func (obj *convergerUID) SetName(name string) {
+	obj.name = name
+}
+
+// IsValid tells us if the id is valid or has already been destroyed
+func (obj *convergerUID) IsValid() bool {
+	return obj.id != 0 // an id of 0 is invalid
+}
+
+// InvalidateID marks the id as no longer valid
+func (obj *convergerUID) InvalidateID() {
+	obj.id = 0 // an id of 0 is invalid
+}
+
+// IsConverged is a helper function to the regular IsConverged method
+func (obj *convergerUID) IsConverged() bool {
+	return obj.converger.IsConverged(obj)
+}
+
+// SetConverged is a helper function to the regular SetConverged notification
+func (obj *convergerUID) SetConverged(isConverged bool) error {
+	return obj.converger.SetConverged(obj, isConverged)
+}
+
+// Unregister is a helper function to unregister myself
+func (obj *convergerUID) Unregister() {
+	obj.converger.Unregister(obj)
+}
+
+// ConvergedTimer is a helper around the regular ConvergedTimer method
+func (obj *convergerUID) ConvergedTimer() <-chan time.Time {
+	return obj.converger.ConvergedTimer(obj)
+}
+
+// StartTimer runs an invisible timer that automatically converges on timeout.
+func (obj *convergerUID) StartTimer() (func() error, error) {
+	obj.mutex.Lock()
+	if !obj.running {
+		obj.timer = make(chan struct{})
+		obj.running = true
+	} else {
+		obj.mutex.Unlock()
+		return obj.StopTimer, fmt.Errorf("Timer already started!")
+	}
+	obj.mutex.Unlock()
+	go func() {
+		for {
+			select {
+			case _, ok := <-obj.timer: // reset signal channel
+				if !ok { // channel is closed
+					return // false to exit
+				}
+				obj.SetConverged(false)
+
+			case <-obj.ConvergedTimer():
+				obj.SetConverged(true) // converged!
+				select {
+				case _, ok := <-obj.timer: // reset signal channel
+					if !ok { // channel is closed
+						return // false to exit
+					}
+				}
+			}
+		}
+	}()
+	return obj.StopTimer, nil
+}
+
+// ResetTimer resets the counter to zero if using a StartTimer internally.
+func (obj *convergerUID) ResetTimer() error {
+	obj.mutex.Lock()
+	defer obj.mutex.Unlock()
+	if obj.running {
+		obj.timer <- struct{}{} // send the reset message
+		return nil
+	}
+	return fmt.Errorf("Timer hasn't been started!")
+}
+
+// StopTimer stops the running timer permanently until a StartTimer is run.
+func (obj *convergerUID) StopTimer() error {
+	obj.mutex.Lock()
+	defer obj.mutex.Unlock()
+	if !obj.running {
+		return fmt.Errorf("Timer isn't running!")
+	}
+	close(obj.timer)
+	obj.running = false
+	return nil
+}

doc.go

@@ -15,12 +15,5 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+// Package main provides the main entrypoint for using the `mgmt` software.
 package main
-
-import (
-//"testing"
-)
-
-//func TestT1(t *testing.T) {
-
-//}

docker/Dockerfile

@@ -0,0 +1,22 @@
+FROM golang:1.6.2
+
+MAINTAINER Michał Czeraszkiewicz <contact@czerasz.com>
+
+# Set the reset cache variable
+# Read more here: http://czerasz.com/2014/11/13/docker-tip-and-tricks/#use-refreshedat-variable-for-better-cache-control
+ENV REFRESHED_AT 2016-05-10
+
+# Update the package list to be able to use required packages
+RUN apt-get update
+
+# Change the working directory
+WORKDIR /go/src/mgmt
+
+# Copy all the files to the working directory
+COPY . /go/src/mgmt
+
+# Install dependencies
+RUN make deps
+
+# Build the binary
+RUN make build

docker/Dockerfile.development

@@ -0,0 +1,31 @@
+FROM golang:1.6.2
+
+MAINTAINER Michał Czeraszkiewicz <contact@czerasz.com>
+
+# Set the reset cache variable
+# Read more here: http://czerasz.com/2014/11/13/docker-tip-and-tricks/#use-refreshedat-variable-for-better-cache-control
+ENV REFRESHED_AT 2016-05-14
+
+RUN apt-get update
+
+# Setup User to match Host User
+# Give the nre user superuser permissions
+ARG USER_ID=1000
+ARG GROUP_ID=1000
+ARG USER_NAME=mgmt
+ARG GROUP_NAME=$USER_NAME
+RUN groupadd --gid $GROUP_ID $GROUP_NAME && \
+	useradd --create-home --home /home/$USER_NAME --uid ${USER_ID} --gid $GROUP_NAME --groups sudo $USER_NAME && \
+	echo "$USER_NAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+
+# Copy all the files to the working directory
+COPY . /home/$USER_NAME/mgmt
+
+# Change working directory
+WORKDIR /home/$USER_NAME/mgmt
+
+# Install dependencies
+RUN make deps
+
+# Change user
+USER ${USER_NAME}

docker/scripts/build

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+script_directory="$( cd "$( dirname "$0" )" && pwd )"
+project_directory=$script_directory/../..
+
+# Specify the Docker image name
+image_name='purpleidea/mgmt'
+
+# Build the image which contains the compiled binary
+docker build -t $image_name \
+	--file=$project_directory/docker/Dockerfile $project_directory
+
+# Remove the container if it already exists
+docker rm -f mgmt-export 2> /dev/null
+
+# Start the container in background so we can "copy out" the binary
+docker run -d --name=mgmt-export $image_name bash -c 'while true; sleep 1000; done'
+
+# Remove the current binary
+rm $project_directory/mgmt 2> /dev/null
+
+# Get the binary from the container
+docker cp mgmt-export:/go/src/mgmt/mgmt $project_directory/mgmt
+
+# Remove the container
+docker rm -f mgmt-export 2> /dev/null

docker/scripts/build-development

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# Stop on any error
+set -e
+
+script_directory="$( cd "$( dirname "$0" )" && pwd )"
+project_directory=$script_directory/../..
+
+# Specify the Docker image name
+image_name='purpleidea/mgmt:development'
+
+# Build the image
+docker build -t $image_name \
+	--file=$project_directory/docker/Dockerfile.development $project_directory

docker/scripts/run-development

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Stop on any error
+set -e
+
+script_directory="$( cd "$( dirname "$0" )" && pwd )"
+project_directory=$script_directory/../..
+
+# Specify the Docker image name
+image_name='purpleidea/mgmt:development'
+
+# Run container in development mode
+docker run --rm --name=mgm_development --user=mgmt \
+	-v $project_directory:/home/mgmt/mgmt \
+	-it $image_name bash

docs/.gitignore

@@ -0,0 +1 @@
+mgmt-documentation.pdf

docs/documentation.md

@@ -0,0 +1,564 @@
+#mgmt
+
+<!--
+Mgmt
+Copyright (C) 2013-2016+ James Shubin and the project contributors
+Written by James Shubin <james@shubin.ca> and the project contributors
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+-->
+
+##mgmt by [James](https://ttboj.wordpress.com/)
+####Available from:
+####[https://github.com/purpleidea/mgmt/](https://github.com/purpleidea/mgmt/)
+
+####This documentation is available in: [Markdown](https://github.com/purpleidea/mgmt/blob/master/docs/documentation.md) or [PDF](https://pdfdoc-purpleidea.rhcloud.com/pdf/https://github.com/purpleidea/mgmt/blob/master/docs/documentation.md) format.
+
+####Table of Contents
+
+1. [Overview](#overview)
+2. [Project description - What the project does](#project-description)
+3. [Setup - Getting started with mgmt](#setup)
+4. [Features - All things mgmt can do](#features)
+	* [Autoedges - Automatic resource relationships](#autoedges)
+	* [Autogrouping - Automatic resource grouping](#autogrouping)
+	* [Automatic clustering - Automatic cluster management](#automatic-clustering)
+	* [Remote mode - Remote "agent-less" execution](#remote-agent-less-mode)
+	* [Puppet support - write manifest code for mgmt](#puppet-support)
+5. [Resources - All built-in primitives](#resources)
+6. [Usage/FAQ - Notes on usage and frequently asked questions](#usage-and-frequently-asked-questions)
+7. [Reference - Detailed reference](#reference)
+	* [Meta parameters](#meta-parameters)
+	* [Graph definition file](#graph-definition-file)
+	* [Command line](#command-line)
+8. [Examples - Example configurations](#examples)
+9. [Development - Background on module development and reporting bugs](#development)
+10. [Authors - Authors and contact information](#authors)
+
+##Overview
+
+The `mgmt` tool is a next generation config management prototype. It's not yet
+ready for production, but we hope to get there soon. Get involved today!
+
+##Project Description
+
+The mgmt tool is a distributed, event driven, config management tool, that
+supports parallel execution, and librarification to be used as the management
+foundation in and for, new and existing software.
+
+For more information, you may like to read some blog posts from the author:
+
+* [Next generation config mgmt](https://ttboj.wordpress.com/2016/01/18/next-generation-configuration-mgmt/)
+* [Automatic edges in mgmt](https://ttboj.wordpress.com/2016/03/14/automatic-edges-in-mgmt/)
+* [Automatic grouping in mgmt](https://ttboj.wordpress.com/2016/03/30/automatic-grouping-in-mgmt/)
+* [Automatic clustering in mgmt](https://ttboj.wordpress.com/2016/06/20/automatic-clustering-in-mgmt/)
+
+There is also an [introductory video](http://meetings-archive.debian.net/pub/debian-meetings/2016/debconf16/Next_Generation_Config_Mgmt.webm) available.
+Older videos and other material [is available](https://github.com/purpleidea/mgmt/#on-the-web).
+
+##Setup
+
+During this prototype phase, the tool can be run out of the source directory.
+You'll probably want to use ```./run.sh run --yaml examples/graph1.yaml``` to
+get started. Beware that this _can_ cause data loss. Understand what you're
+doing first, or perform these actions in a virtual environment such as the one
+provided by [Oh-My-Vagrant](https://github.com/purpleidea/oh-my-vagrant).
+
+##Features
+
+This section details the numerous features of mgmt and some caveats you might
+need to be aware of.
+
+###Autoedges
+
+Automatic edges, or AutoEdges, is the mechanism in mgmt by which it will
+automatically create dependencies for you between resources. For example,
+since mgmt can discover which files are installed by a package it will
+automatically ensure that any file resource you declare that matches a
+file installed by your package resource will only be processed after the
+package is installed.
+
+####Controlling autoedges
+
+Though autoedges is likely to be very helpful and avoid you having to declare
+all dependencies explicitly, there are cases where this behaviour is
+undesirable.
+
+Some distributions allow package installations to automatically start the
+service they ship. This can be problematic in the case of packages like MySQL
+as there are configuration options that need to be set before MySQL is ever
+started for the first time (or you'll need to wipe the data directory). In
+order to handle this situation you can disable autoedges per resource and
+explicitly declare that you want `my.cnf` to be written to disk before the
+installation of the `mysql-server` package.
+
+You can disable autoedges for a resource by setting the `autoedge` key on
+the meta attributes of that resource to `false`.
+
+####Blog post
+
+You can read the introductory blog post about this topic here:
+[https://ttboj.wordpress.com/2016/03/14/automatic-edges-in-mgmt/](https://ttboj.wordpress.com/2016/03/14/automatic-edges-in-mgmt/)
+
+###Autogrouping
+
+Automatic grouping or AutoGroup is the mechanism in mgmt by which it will
+automatically group multiple resource vertices into a single one. This is
+particularly useful for grouping multiple package resources into a single
+resource, since the multiple installations can happen together in a single
+transaction, which saves a lot of time because package resources typically have
+a large fixed cost to running (downloading and verifying the package repo) and
+if they are grouped they share this fixed cost. This grouping feature can be
+used for other use cases too.
+
+You can disable autogrouping for a resource by setting the `autogroup` key on
+the meta attributes of that resource to `false`.
+
+####Blog post
+
+You can read the introductory blog post about this topic here:
+[https://ttboj.wordpress.com/2016/03/30/automatic-grouping-in-mgmt/](https://ttboj.wordpress.com/2016/03/30/automatic-grouping-in-mgmt/)
+
+###Automatic clustering
+
+Automatic clustering is a feature by which mgmt automatically builds, scales,
+and manages the embedded etcd cluster which is compiled into mgmt itself. It is
+quite helpful for rapidly bootstrapping clusters and avoiding the extra work to
+setup etcd.
+
+If you prefer to avoid this feature. you can always opt to use an existing etcd
+cluster that is managed separately from mgmt by pointing your mgmt agents at it
+with the `--seeds` variable.
+
+####Blog post
+
+You can read the introductory blog post about this topic here:
+[https://ttboj.wordpress.com/2016/06/20/automatic-clustering-in-mgmt/](https://ttboj.wordpress.com/2016/06/20/automatic-clustering-in-mgmt/)
+
+###Remote ("agent-less") mode
+
+Remote mode is a special mode that lets you kick off mgmt runs on one or more
+remote machines which are only accessible via SSH. In this mode the initiating
+host connects over SSH, copies over the `mgmt` binary, opens an SSH tunnel, and
+runs the remote program while simultaneously passing the etcd traffic back
+through the tunnel so that the initiators etcd cluster can be used to exchange
+resource data.
+
+The interesting benefit of this architecture is that multiple hosts which can't
+connect directly use the initiator to pass the important traffic through to each
+other. Once the cluster has converged all the remote programs can shutdown
+leaving no residual agent.
+
+This mode can also be useful for bootstrapping a new host where you'd like to
+have the service run continuously and as part of an mgmt cluster normally.
+
+In particular, when combined with the `--converged-timeout` parameter, the
+entire set of running mgmt agents will need to all simultaneously converge for
+the group to exit. This is particularly useful for bootstrapping new clusters
+which need to exchange information that is only available at run time.
+
+####Blog post
+
+You can read the introductory blog post about this topic here:
+[https://ttboj.wordpress.com/2016/10/07/remote-execution-in-mgmt/](https://ttboj.wordpress.com/2016/10/07/remote-execution-in-mgmt/)
+
+###Puppet support
+
+You can supply a Puppet manifest instead of creating the (YAML) graph manually.
+Puppet must be installed and in `mgmt`'s search path. You also need the
+[ffrank-mgmtgraph Puppet module](https://forge.puppet.com/ffrank/mgmtgraph).
+
+Invoke `mgmt` with the `--puppet` switch, which supports 3 variants:
+
+1. Request the configuration from the Puppet Master (like `puppet agent` does)
+
+        mgmt run --puppet agent
+
+2. Compile a local manifest file (like `puppet apply`)
+
+        mgmt run --puppet /path/to/my/manifest.pp
+
+3. Compile an ad hoc manifest from the commandline (like `puppet apply -e`)
+
+        mgmt run --puppet 'file { "/etc/ntp.conf": ensure => file }'
+
+For more details and caveats see [Puppet.md](Puppet.md).
+
+####Blog post
+
+An introductory post on the Puppet support is on
+[Felix's blog](http://ffrank.github.io/features/2016/06/19/puppet-powered-mgmt/).
+
+##Resources
+
+This section lists all the built-in resources and their properties. The
+resource primitives in `mgmt` are typically more powerful than resources in
+other configuration management systems because they can be event based which
+lets them respond in real-time to converge to the desired state. This property
+allows you to build more complex resources that you probably hadn't considered
+in the past.
+
+In addition to the resource specific properties, there are resource properties
+(otherwise known as parameters) which can apply to every resource. These are
+called [meta parameters](#meta-parameters) and are listed separately. Certain
+meta parameters aren't very useful when combined with certain resources, but
+in general, it should be fairly obvious, such as when combining the `noop` meta
+parameter with the [Noop](#Noop) resource.
+
+* [Exec](#Exec): Execute shell commands on the system.
+* [File](#File): Manage files and directories.
+* [Hostname](#Hostname): Manages the hostname on the system.
+* [Msg](#Msg): Send log messages.
+* [Noop](#Noop): A simple resource that does nothing.
+* [Nspawn](#Nspawn): Manage systemd-machined nspawn containers.
+* [Password](#Password): Create random password strings.
+* [Pkg](#Pkg):  Manage system packages with PackageKit.
+* [Svc](#Svc): Manage system systemd services.
+* [Timer](#Timer): Manage system systemd services.
+* [Virt](#Virt): Manage virtual machines with libvirt.
+
+###Exec
+
+The exec resource can execute commands on your system.
+
+###File
+
+The file resource manages files and directories. In `mgmt`, directories are
+identified by a trailing slash in their path name. File have no such slash.
+
+####Path
+
+The path property specifies the file or directory that we are managing.
+
+####Content
+
+The content property is a string that specifies the desired file contents.
+
+####Source
+
+The source property points to a source file or directory path that we wish to
+copy over and use as the desired contents for our resource.
+
+####State
+
+The state property describes the action we'd like to apply for the resource. The
+possible values are: `exists` and `absent`.
+
+####Recurse
+
+The recurse property limits whether file resource operations should recurse into
+and monitor directory contents with a depth greater than one.
+
+####Force
+
+The force property is required if we want the file resource to be able to change
+a file into a directory or vice-versa. If such a change is needed, but the force
+property is not set to `true`, then this file resource will error.
+
+###Hostname
+
+The hostname resource manages static, transient/dynamic and pretty hostnames
+on the system and watches them for changes.
+
+#### static_hostname
+The static hostname is the one configured in /etc/hostname or a similar
+file.
+It is chosen by the local user. It is not always in sync with the current
+host name as returned by the gethostname() system call.
+
+#### transient_hostname
+The transient / dynamic hostname is the one configured via the kernel's
+sethostbyname().
+It can be different from the static hostname in case DHCP or mDNS have been
+configured to change the name based on network information.
+
+#### pretty_hostname
+The pretty hostname is a free-form UTF8 host name for presentation to the user.
+
+#### hostname
+Hostname is the fallback value for all 3 fields above, if only `hostname` is
+specified, it will set all 3 fields to this value.
+
+###Msg
+
+The msg resource sends messages to the main log, or an external service such
+as systemd's journal.
+
+###Noop
+
+The noop resource does absolutely nothing. It does have some utility in testing
+`mgmt` and also as a placeholder in the resource graph.
+
+###Nspawn
+
+The nspawn resource is used to manage systemd-machined style containers.
+
+###Password
+
+The password resource can generate a random string to be used as a password. It
+will re-generate the password if it receives a refresh notification.
+
+###Pkg
+
+The pkg resource is used to manage system packages. This resource works on many
+different distributions because it uses the underlying packagekit facility which
+supports different backends for different environments. This ensures that we
+have great Debian (deb/dpkg) and Fedora (rpm/dnf) support simultaneously.
+
+###Svc
+
+The service resource is still very WIP. Please help us my improving it!
+
+###Timer
+
+This resource needs better documentation. Please help us my improving it!
+
+###Virt
+
+The virt resource can manage virtual machines via libvirt.
+
+##Usage and frequently asked questions
+(Send your questions as a patch to this FAQ! I'll review it, merge it, and
+respond by commit with the answer.)
+
+###Why did you start this project?
+
+I wanted a next generation config management solution that didn't have all of
+the design flaws or limitations that the current generation of tools do, and no
+tool existed!
+
+###Why did you use etcd? What about consul?
+
+Etcd and consul are both written in golang, which made them the top two
+contenders for my prototype. Ultimately a choice had to be made, and etcd was
+chosen, but it was also somewhat arbitrary. If there is available interest,
+good reasoning, *and* patches, then we would consider either switching or
+supporting both, but this is not a high priority at this time.
+
+###Can I use an existing etcd cluster instead of the automatic embedded servers?
+
+Yes, it's possible to use an existing etcd cluster instead of the automatic,
+elastic embedded etcd servers. To do so, simply point to the cluster with the
+`--seeds` variable, the same way you would if you were seeding a new member to
+an existing mgmt cluster.
+
+The downside to this approach is that you won't benefit from the automatic
+elastic nature of the embedded etcd servers, and that you're responsible if you
+accidentally break your etcd cluster, or if you use an unsupported version.
+
+###What does the error message about an inconsistent dataDir mean?
+
+If you get an error message similar to:
+
+```
+Etcd: Connect: CtxError...
+Etcd: CtxError: Reason: CtxDelayErr(5s): No endpoints available yet!
+Etcd: Connect: Endpoints: []
+Etcd: The dataDir (/var/lib/mgmt/etcd) might be inconsistent or corrupt.
+```
+
+This happens when there are a series of fatal connect errors in a row. This can
+happen when you start `mgmt` using a dataDir that doesn't correspond to the
+current cluster view. As a result, the embedded etcd server never finishes
+starting up, and as a result, a default endpoint never gets added. The solution
+is to either reconcile the mistake, and if there is no important data saved, you
+can remove the etcd dataDir. This is typically `/var/lib/mgmt/etcd/member/`.
+
+###Why do resources have both a `Compare` method and an `IFF` (on the UID) method?
+
+The `Compare()` methods are for determining if two resources are effectively the
+same, which is used to make graph change delta's efficient. This is when we want
+to change from the current running graph to a new graph, but preserve the common
+vertices. Since we want to make this process efficient, we only update the parts
+that are different, and leave everything else alone. This `Compare()` method can
+tell us if two resources are the same.
+
+The `IFF()` method is part of the whole UID system, which is for discerning if a
+resource meets the requirements another expects for an automatic edge. This is
+because the automatic edge system assumes a unified UID pattern to test for
+equality. In the future it might be helpful or sane to merge the two similar
+comparison functions although for now they are separate because they are
+actually answer different questions.
+
+###Did you know that there is a band named `MGMT`?
+
+I didn't realize this when naming the project, and it is accidental. After much
+anguishing, I chose the name because it was short and I thought it was
+appropriately descriptive. If you need a less ambiguous search term or phrase,
+you can try using `mgmtconfig` or `mgmt config`.
+
+###You didn't answer my question, or I have a question!
+
+It's best to ask on [IRC](https://webchat.freenode.net/?channels=#mgmtconfig)
+to see if someone can help you. Once we get a big enough community going, we'll
+add a mailing list. If you don't get any response from the above, you can
+contact me through my [technical blog](https://ttboj.wordpress.com/contact/)
+and I'll do my best to help. If you have a good question, please add it as a
+patch to this documentation. I'll merge your question, and add a patch with the
+answer!
+
+##Reference
+Please note that there are a number of undocumented options. For more
+information on these options, please view the source at:
+[https://github.com/purpleidea/mgmt/](https://github.com/purpleidea/mgmt/).
+If you feel that a well used option needs documenting here, please patch it!
+
+###Overview of reference
+* [Meta parameters](#meta-parameters): List of available resource meta parameters.
+* [Graph definition file](#graph-definition-file): Main graph definition file.
+* [Command line](#command-line): Command line parameters.
+
+###Meta parameters
+These meta parameters are special parameters (or properties) which can apply to
+any resource. The usefulness of doing so will depend on the particular meta
+parameter and resource combination.
+
+####AutoEdge
+Boolean. Should we generate auto edges for this resource?
+
+####AutoGroup
+Boolean. Should we attempt to automatically group this resource with others?
+
+####Noop
+Boolean. Should the Apply portion of the CheckApply method of the resource
+make any changes? Noop is a concatenation of no-operation.
+
+####Retry
+Integer. The number of times to retry running the resource on error. Use -1 for
+infinite. This currently applies for both the Watch operation (which can fail)
+and for the CheckApply operation. While they could have separate values, I've
+decided to use the same ones for both until there's a proper reason to want to
+do something differently for the Watch errors.
+
+####Delay
+Integer. Number of milliseconds to wait between retries. The same value is
+shared between the Watch and CheckApply retries. This currently applies for both
+the Watch operation (which can fail) and for the CheckApply operation. While
+they could have separate values, I've decided to use the same ones for both
+until there's a proper reason to want to do something differently for the Watch
+errors.
+
+###Graph definition file
+graph.yaml is the compiled graph definition file. The format is currently
+undocumented, but by looking through the [examples/](https://github.com/purpleidea/mgmt/tree/master/examples)
+you can probably figure out most of it, as it's fairly intuitive.
+
+###Command line
+The main interface to the `mgmt` tool is the command line. For the most recent
+documentation, please run `mgmt --help`.
+
+####`--yaml <graph.yaml>`
+Point to a graph file to run.
+
+####`--converged-timeout <seconds>`
+Exit if the machine has converged for approximately this many seconds.
+
+####`--max-runtime <seconds>`
+Exit when the agent has run for approximately this many seconds. This is not
+generally recommended, but may be useful for users who know what they're doing.
+
+####`--noop`
+Globally force all resources into no-op mode. This also disables the export to
+etcd functionality, but does not disable resource collection, however all
+resources that are collected will have their individual noop settings set.
+
+####`--remote <graph.yaml>`
+Point to a graph file to run on the remote host specified within. This parameter
+can be used multiple times if you'd like to remotely run on multiple hosts in
+parallel.
+
+####`--allow-interactive`
+Allow interactive prompting for SSH passwords if there is no authentication
+method that works.
+
+####`--ssh-priv-id-rsa`
+Specify the path for finding SSH keys. This defaults to `~/.ssh/id_rsa`. To
+never use this method of authentication, set this to the empty string.
+
+####`--cconns`
+The maximum number of concurrent remote ssh connections to run. This defaults
+to `0`, which means unlimited.
+
+####`--no-caching`
+Don't allow remote caching of the remote execution binary. This will require
+the binary to be copied over for every remote execution, but it limits the
+likelihood that there is leftover information from the configuration process.
+
+####`--prefix <path>`
+Specify a path to a custom working directory prefix. This directory will get
+created if it does not exist. This usually defaults to `/var/lib/mgmt/`. This
+can't be combined with the `--tmp-prefix` option. It can be combined with the
+`--allow-tmp-prefix` option.
+
+####`--tmp-prefix`
+If this option is specified, a temporary prefix will be used instead of the
+default prefix. This can't be combined with the `--prefix` option.
+
+####`--allow-tmp-prefix`
+If this option is specified, we will attempt to fall back to a temporary prefix
+if the primary prefix couldn't be created. This is useful for avoiding failures
+in environments where the primary prefix may or may not be available, but you'd
+like to try. The canonical example is when running `mgmt` with `--remote` there
+might be a cached copy of the binary in the primary prefix, but in case there's
+no binary available continue working in a temporary directory to avoid failure.
+
+##Examples
+For example configurations, please consult the [examples/](https://github.com/purpleidea/mgmt/tree/master/examples) directory in the git
+source repository. It is available from:
+
+[https://github.com/purpleidea/mgmt/tree/master/examples](https://github.com/purpleidea/mgmt/tree/master/examples)
+
+### Systemd:
+See [`misc/mgmt.service`](misc/mgmt.service) for a sample systemd unit file.
+This unit file is part of the RPM.
+
+To specify your custom options for `mgmt` on a systemd distro:
+
+```bash
+sudo mkdir -p /etc/systemd/system/mgmt.service.d/
+
+cat > /etc/systemd/system/mgmt.service.d/env.conf <<EOF
+# Environment variables:
+MGMT_SEEDS=http://127.0.0.1:2379
+MGMT_CONVERGED_TIMEOUT=-1
+MGMT_MAX_RUNTIME=0
+
+# Other CLI options if necessary.
+#OPTS="--max-runtime=0"
+EOF
+
+sudo systemctl daemon-reload
+```
+
+##Development
+
+This is a project that I started in my free time in 2013. Development is driven
+by all of our collective patches! Dive right in, and start hacking!
+Please contact me if you'd like to invite me to speak about this at your event.
+
+You can follow along [on my technical blog](https://ttboj.wordpress.com/).
+
+To report any bugs, please file a ticket at: [https://github.com/purpleidea/mgmt/issues](https://github.com/purpleidea/mgmt/issues).
+
+##Authors
+
+Copyright (C) 2013-2016+ James Shubin and the project contributors
+
+Please see the
+[AUTHORS](https://github.com/purpleidea/mgmt/tree/master/AUTHORS) file
+for more information.
+
+* [github](https://github.com/purpleidea/)
+* [&#64;purpleidea](https://twitter.com/#!/purpleidea)
+* [https://ttboj.wordpress.com/](https://ttboj.wordpress.com/)

docs/puppet.md

@@ -0,0 +1,163 @@
+#mgmt Puppet support
+
+1. [Prerequisites](#prerequisites)
+    * [Testing the Puppet side](#testing-the-puppet-side)
+2. [Writing a suitable manifest](#writing-a-suitable-manifest)
+    * [Unsupported attributes](#unsupported-attributes)
+    * [Unsupported resources](#unsupported-resources)
+    * [Avoiding common warnings](#avoiding-common-warnings)
+3. [Configuring Puppet](#configuring-puppet)
+4. [Caveats](#caveats)
+
+`mgmt` can use Puppet as its source for the configuration graph.
+This document goes into detail on how this works, and lists
+some pitfalls and limitations.
+
+For basic instructions on how to use the Puppet support, see
+the [main documentation](documentation.md#puppet-support).
+
+##Prerequisites
+
+You need Puppet installed in your system. It is not important how you
+get it. On the most common Linux distributions, you can use packages
+from the OS maintainer, or upstream Puppet repositories. An alternative
+that will also work on OSX is the `puppet` Ruby gem. It also has the
+advantage that you can install any desired version in your home directory
+or any other location.
+
+Any release of Puppet's 3.x and 4.x series should be suitable for use with
+`mgmt`. Most importantly, make sure to install the `ffrank-mgmtgraph` Puppet
+module (referred to below as "the translator module").
+
+    puppet module install ffrank-mgmtgraph
+
+Please note that the module is not required on your Puppet master (if you
+use a master/agent setup). It's needed on the machine that runs `mgmt`.
+You can install the module on the master anyway, so that it gets distributed
+to your agents through Puppet's `pluginsync` mechanism.
+
+###Testing the Puppet side
+
+The following command should run successfully and print a YAML hash on your
+terminal:
+
+```puppet
+puppet mgmtgraph print --code 'file { "/tmp/mgmt-test": ensure => present }'
+```
+
+You can use this CLI to test any manifests before handing them straight
+to `mgmt`.
+
+##Writing a suitable manifest
+
+###Unsupported attributes
+
+`mgmt` inherited its resource module from Puppet, so by and large, it's quite
+possible to express `mgmt` graphs in terms of Puppet manifests. However,
+there isn't (and likely never will be) full feature parity between the
+respective resource types. In consequence, a manifest can have semantics that
+cannot be transferred to `mgmt`.
+
+For example, at the time of writing this, the `file` type in `mgmt` had no
+notion of permissions (the file `mode`) yet. This lead to the following
+warning (among others that will be discussed below):
+
+    $ puppet mgmtgraph print --code 'file { "/tmp/foo": mode => "0600" }'
+    Warning: cannot translate: File[/tmp/foo] { mode => "600" } (attribute is ignored)
+
+This is a heads-up for the user, because the resulting `mgmt` graph will
+in fact not pass this information to the `/tmp/foo` file resource, and
+`mgmt` will ignore this file's permissions. Including such attributes in
+manifests that are written expressly for `mgmt` is not sensible and should
+be avoided.
+
+###Unsupported resources
+
+Puppet has a fairly large number of
+[built-in types](https://docs.puppet.com/puppet/latest/reference/type.html),
+and countless more are available through
+[modules](https://forge.puppet.com/). It's unlikely that all of them will
+eventually receive native counterparts in `mgmt`.
+
+When encountering an unknown resource, the translator module will replace
+it with an `exec` resource in its output. This resource will run the equivalent
+of a `puppet resource` command to make Puppet apply the original resource
+itself. This has quite abysmal performance, because processing such a
+resource requires the forking of at least one Puppet process (two if it
+is found to be out of sync). This comes with considerable overhead.
+On most systems, starting up any Puppet command takes several seconds.
+Compared to the split second that the actual work usually takes,
+this overhead can amount to several orders of magnitude.
+
+Avoid Puppet types that `mgmt` does not implement (yet).
+
+###Avoiding common warnings
+
+Many resource parameters in Puppet take default values. For the most part,
+the translator module just ignores them. However, there are cases in which
+Puppet will default to convenient behavior that `mgmt` cannot quite replicate.
+For example, translating a plain `file` resource will lead to a warning message:
+
+    $ puppet mgmtgraph print --code 'file { "/tmp/mgmt-test": }'
+    Warning: File[/tmp/mgmt-test] uses the 'puppet' file bucket, which mgmt cannot do. There will be no backup copies!
+
+The reason is that per default, Puppet assumes the following parameter value
+(among others)
+
+```puppet
+file { "/tmp/mgmt-test":
+  backup => 'puppet',
+}
+```
+
+To avoid this, specify the parameter explicitly:
+
+    $ puppet mgmtgraph print --code 'file { "/tmp/mgmt-test": backup => false }'
+
+This is tedious in a more complex manifest. A good simplification is the
+following [resource default](https://docs.puppet.com/puppet/latest/reference/lang_defaults.html)
+anywhere on the top scope of your manifest:
+
+```puppet
+File { backup => false }
+```
+
+If you encounter similar warnings from other types and/or parameters,
+use the same approach to silence them if possible.
+
+##Configuring Puppet
+
+Since `mgmt` uses an actual Puppet CLI behind the scenes, you might
+need to tweak some of Puppet's runtime options in order to make it
+do what you want. Reasons for this could be among the following:
+
+* You use the `--puppet agent` variant and need to configure
+`servername`, `certname` and other master/agent-related options.
+* You don't want runtime information to end up in the `vardir`
+that is used by your regular `puppet agent`.
+* You install specific Puppet modules for `mgmt` in a non-standard
+location.
+
+`mgmt` exposes only one Puppet option in order to allow you to
+control all of them, through its `--puppet-conf` option. It allows
+you to specify which `puppet.conf` file should be used during
+translation.
+
+    mgmt run --puppet /opt/my-manifest.pp --puppet-conf /etc/mgmt/puppet.conf
+
+Within this file, you can just specify any needed options in the
+`[main]` section:
+
+    [main]
+    server=mgmt-master.example.net
+    vardir=/var/lib/mgmt/puppet
+
+##Caveats
+
+Please see the [README](https://github.com/ffrank/puppet-mgmtgraph/blob/master/README.md)
+of the translator module for the current state of supported and unsupported
+language features.
+
+You should probably make sure to always use the latest release of
+both `ffrank-mgmtgraph` and `ffrank-yamlresource` (the latter is
+getting pulled in as a dependency of the former).

docs/resource-guide.md

@@ -0,0 +1,534 @@
+#mgmt
+
+<!--
+Mgmt
+Copyright (C) 2013-2016+ James Shubin and the project contributors
+Written by James Shubin <james@shubin.ca> and the project contributors
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+-->
+
+##mgmt resource guide by [James](https://ttboj.wordpress.com/)
+####Available from:
+####[https://github.com/purpleidea/mgmt/](https://github.com/purpleidea/mgmt/)
+
+####This documentation is available in: [Markdown](https://github.com/purpleidea/mgmt/blob/master/docs/resource-guide.md) or [PDF](https://pdfdoc-purpleidea.rhcloud.com/pdf/https://github.com/purpleidea/mgmt/blob/master/docs/resource-guide.md) format.
+
+####Table of Contents
+
+1. [Overview](#overview)
+2. [Theory - Resource theory in mgmt](#theory)
+3. [Resource API - Getting started with mgmt](#resource-api)
+	* [Init - Initialize the resource](#init)
+	* [CheckApply - Check and apply resource state](#checkapply)
+	* [Watch - Detect resource changes](#watch)
+	* [Compare - Compare resource with another](#compare)
+4. [Further considerations - More information about resource writing](#further-considerations)
+5. [Automatic edges - Adding automatic resources dependencies](#automatic-edges)
+6. [Automatic grouping - Grouping multiple resources into one](#automatic-grouping)
+7. [Send/Recv - Communication between resources](#send-recv)
+8. [Composite resources - Importing code from one resource into another](#composite-resources)
+9. [FAQ - Frequently asked questions](#frequently-asked-questions)
+10. [Suggestions - API change suggestions](#suggestions)
+11. [Authors - Authors and contact information](#authors)
+
+##Overview
+
+The `mgmt` tool has built-in resource primitives which make up the building
+blocks of any configuration. Each instance of a resource is mapped to a single
+vertex in the resource [graph](https://en.wikipedia.org/wiki/Directed_acyclic_graph).
+This guide is meant to instruct developers on how to write a brand new resource.
+Since `mgmt` and the core resources are written in golang, some prior golang
+knowledge is assumed.
+
+##Theory
+
+Resources in `mgmt` are similar to resources in other systems in that they are
+[idempotent](https://en.wikipedia.org/wiki/Idempotence). Our resources are
+uniquely different in that they can detect when their state has changed, and as
+a result can run to revert or repair this change instantly. For some background
+on this design, please read the
+[original article](https://ttboj.wordpress.com/2016/01/18/next-generation-configuration-mgmt/)
+on the subject.
+
+##Resource API
+
+To implement a resource in `mgmt` it must satisfy the
+[`Res`](https://github.com/purpleidea/mgmt/blob/master/resources/resources.go)
+interface. What follows are each of the method signatures and a description of
+each.
+
+###Init
+```golang
+Init() error
+```
+
+This is called to initialize the resource. If something goes wrong, it should
+return an error. It should set the resource `kind`, do any resource specific
+work, and finish by calling the `Init` method of the base resource.
+
+####Example
+```golang
+// Init initializes the Foo resource.
+func (obj *FooRes) Init() error {
+	obj.BaseRes.kind = "Foo" // must set capitalized resource kind
+	// run the resource specific initialization, and error if anything fails
+	if some_error {
+		return err // something went wrong!
+	}
+	return obj.BaseRes.Init() // call the base resource init
+}
+```
+
+###CheckApply
+```golang
+CheckApply(apply bool) (checkOK bool, err error)
+```
+
+`CheckApply` is where the real _work_ is done. Under normal circumstances, this
+function should check if the state of this resource is correct, and if so, it
+should return: `(true, nil)`. If the `apply` variable is set to `true`, then
+this means that we should then proceed to run the changes required to bring the
+resource into the correct state. If the `apply` variable is set to `false`, then
+the resource is operating in _noop_ mode and _no operations_ should be executed!
+
+After having executed the necessary operations to bring the resource back into
+the desired state, or after having detected that the state was incorrect, but
+that changes can't be made because `apply` is `false`, you should then return
+`(false, nil)`.
+
+You must cause the resource to converge during a single execution of this
+function. If you cannot, then you must return an error! The exception to this
+rule is that if an external force changes the state of the resource while it is
+being remedied, it is possible to return from this function even though the
+resource isn't now converged. This is not a bug, as the resources `Watch`
+facility will detect the change, ultimately resulting in a subsequent call to
+`CheckApply`.
+
+####Example
+```golang
+// CheckApply does the idempotent work of checking and applying resource state.
+func (obj *FooRes) CheckApply(apply bool) (bool, error) {
+	// check the state
+	if state_is_okay { return true, nil } // done early! :)
+	// state was bad
+	if !apply { return false, nil } // don't apply; !stateok, nil
+	// do the apply!
+	return false, nil // after success applying
+	if any_error { return false, err } // anytime there's an err!
+}
+```
+
+The `CheckApply` function is called by the `mgmt` engine when it believes a call
+is necessary. Under certain conditions when a `Watch` call does not invalidate
+the state of the resource, and no refresh call was sent, its execution might be
+skipped. This is an engine optimization, and not a bug. It is mentioned here in
+the documentation in case you are confused as to why a debug message you've
+added to the code isn't always printed.
+
+####Refresh notifications
+Some resources may choose to support receiving refresh notifications. In general
+these should be avoided if possible, but nevertheless, they do make sense in
+certain situations. Resources that support these need to verify if one was sent
+during the CheckApply phase of execution. This is accomplished by calling the
+`Refresh() bool` method of the resource, and inspecting the return value. This
+is only necessary if you plan to perform a refresh action. Refresh actions
+should still respect the `apply` variable, and no system changes should be made
+if it is `false`. Refresh notifications are generated by any resource when an
+action is applied by that resource and are transmitted through graph edges which
+have enabled their propagation. Resources that currently perform some refresh
+action include `svc`, `timer`, and `password`.
+
+####Paired execution
+For many resources it is not uncommon to see `CheckApply` run twice in rapid
+succession. This is usually not a pathological occurrence, but rather a healthy
+pattern which is a consequence of the event system. When the state of the
+resource is incorrect, `CheckApply` will run to remedy the state. In response to
+having just changed the state, it is usually the case that this repair will
+trigger the `Watch` code! In response, a second `CheckApply` is triggered, which
+will likely find the state to now be correct.
+
+####Summary
+* Anytime an error occurs during `CheckApply`, you should return `(false, err)`.
+* If the state is correct and no changes are needed, return `(true, nil)`.
+* You should only make changes to the system if `apply` is set to `true`.
+* After checking the state and possibly applying the fix, return `(false, nil)`.
+* Returning `(true, err)` is a programming error and will cause a `Fatal`.
+
+###Watch
+```golang
+Watch(chan Event) error
+```
+
+`Watch` is a main loop that runs and sends messages when it detects that the
+state of the resource might have changed. To send a message you should write to
+the input `Event` channel using the `DoSend` helper method. The Watch function
+should run continuously until a shutdown message is received. If at any time
+something goes wrong, you should return an error, and the `mgmt` engine will
+handle possibly restarting the main loop based on the `retry` meta parameters.
+
+It is better to send an event notification which turns out to be spurious, than
+to miss a possible event. Resources which can miss events are incorrect and need
+to be re-engineered so that this isn't the case. If you have an idea for a
+resource which would fit this criteria, but you can't find a solution, please
+contact the `mgmt` maintainers so that this problem can be investigated and a
+possible system level engineering fix can be found.
+
+You may have trouble deciding how much resource state checking should happen in
+the `Watch` loop versus deferring it all to the `CheckApply` method. You may
+want to put some simple fast path checking in `Watch` to avoid generating
+obviously spurious events, but in general it's best to keep the `Watch` method
+as simple as possible. Contact the `mgmt` maintainers if you're not sure.
+
+If the resource is activated in `polling` mode, the `Watch` method will not get
+executed. As a result, the resource must still work even if the main loop is not
+running.
+
+####Select
+The lifetime of most resources `Watch` method should be spent in an infinite
+loop that is bounded by a `select` call. The `select` call is the point where
+our method hands back control to the engine (and the kernel) so that we can
+sleep until something of interest wakes us up. In this loop we must process
+events from the engine via the `<-obj.Events()` call, wait for the converged
+timeout with `<-cuid.ConvergedTimer()`, and receive events for our resource
+itself!
+
+####Events
+If we receive an internal event from the `<-obj.Events()` method, we can read it
+with the ReadEvent helper function. This function tells us if we should shutdown
+our resource, and if we should generate an event. When we want to send an event,
+we use the `DoSend` helper function. It is also important to mark the resource
+state as `dirty` if we believe it might have changed. We do this with the
+`StateOK(false)` function.
+
+####Startup
+Once the `Watch` function has finished starting up successfully, it is important
+to generate one event to notify the `mgmt` engine that we're now listening
+successfully, so that it can run an initial `CheckApply` to ensure we're safely
+tracking a healthy state and that we didn't miss anything when `Watch` was down
+or from before `mgmt` was running. It does this by calling the `Running` method.
+
+####Converged
+The engine might be asked to shutdown when the entire state of the system has
+not seen any changes for some duration of time. In order for the engine to be
+able to make this determination, each resource must report its converged state.
+To do this, the `Watch` method should get the `ConvergedUID` handle that has
+been prepared for it by the engine. This is done by calling the `Converger`
+method on the resource object. The result can be used to set the converged
+status with `SetConverged`, and to notify when the particular timeout has been
+reached by waiting on `ConvergedTimer`.
+
+Instead of interacting with the `ConvergedUID` with these two methods, we can
+instead use the `StartTimer` and `ResetTimer` methods which accomplish the same
+thing, but provide a `select`-free interface for different coding situations.
+
+####Example
+```golang
+// Watch is the listener and main loop for this resource.
+func (obj *FooRes) Watch(processChan chan event.Event) error {
+	cuid := obj.Converger() // get the converger uid used to report status
+
+	// setup the Foo resource
+	var err error
+	if err, obj.foo = OpenFoo(); err != nil {
+		return err // we couldn't startup
+	}
+	defer obj.whatever.CloseFoo() // shutdown our
+
+	// notify engine that we're running
+	if err := obj.Running(processChan); err != nil {
+		return err // bubble up a NACK...
+	}
+
+	var send = false // send event?
+	var exit = false
+	for {
+		obj.SetState(ResStateWatching) // reset
+		select {
+		case event := <-obj.Events():
+			cuid.SetConverged(false)
+			// we avoid sending events on unpause
+			if exit, send = obj.ReadEvent(&event); exit {
+				return nil // exit
+			}
+
+		// the actual events!
+		case event := <-obj.foo.Events:
+			if is_an_event {
+				send = true // used below
+				cuid.SetConverged(false)
+				obj.StateOK(false) // dirty
+			}
+
+		// event errors
+		case err := <-obj.foo.Errors:
+			cuuid.SetConverged(false)
+			return err // will cause a retry or permanent failure
+
+		case <-cuid.ConvergedTimer():
+			cuid.SetConverged(true) // converged!
+			continue
+		}
+
+		// do all our event sending all together to avoid duplicate msgs
+		if send {
+			send = false
+			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
+				return err // we exit or bubble up a NACK...
+			}
+		}
+	}
+}
+```
+
+####Summary
+* Remember to call the appropriate `converger` methods throughout the resource.
+* Remember to call `Startup` when the `Watch` is running successfully.
+* Remember to process internal events and shutdown promptly if asked to.
+* Ensure the design of your resource is well thought out.
+* Have a look at the existing resources for a rough idea of how this all works.
+
+###Compare
+```golang
+Compare(Res) bool
+```
+
+Each resource must have a `Compare` method. This takes as input another resource
+and must return whether they are identical or not. This is used for identifying
+if an existing resource can be used in place of a new one with a similar set of
+parameters. In particular, when switching from one graph to a new (possibly
+identical) graph, this avoids recomputing the state for resources which don't
+change or that are sufficiently similar that they don't need to be swapped out.
+
+In general if all the resource properties are identical, then they usually don't
+need to be changed. On occasion, not all of them need to be compared, in
+particular if they store some generated state, or if they aren't significant in
+some way.
+
+####Example
+```golang
+// Compare two resources and return if they are equivalent.
+func (obj *FooRes) Compare(res Res) bool {
+	switch res.(type) {
+	case *FooRes: // only compare to other resources of the Foo kind!
+		res := res.(*FileRes)
+		if !obj.BaseRes.Compare(res) { // call base Compare
+			return false
+		}
+		if obj.Name != res.Name {
+			return false
+		}
+		if obj.whatever != res.whatever {
+			return false
+		}
+		if obj.Flag != res.Flag {
+			return false
+		}
+	default:
+		return false // different kind of resource
+	}
+	return true // they must match!
+}
+```
+
+###Validate
+```golang
+Validate() error
+```
+
+This method is used to validate if the populated resource struct is a valid
+representation of the resource kind. If it does not conform to the resource
+specifications, it should generate an error. If you notice that this method is
+quite large, it might be an indication that you might want to reconsider the
+parameter list and interface to this resource.
+
+###GetUIDs
+```golang
+GetUIDs() []ResUID
+```
+
+The `GetUIDs` method returns a list of `ResUID` interfaces that represent the
+particular resource uniquely. This is used with the AutoEdges API to determine
+if another resource can match a dependency to this one.
+
+###AutoEdges
+```golang
+AutoEdges() AutoEdge
+```
+
+This returns a struct that implements the `AutoEdge` interface. This struct
+is used to match other resources that might be relevant dependencies for this
+resource.
+
+###CollectPattern
+```golang
+CollectPattern() string
+```
+
+This is currently a stub and will be updated once the DSL is further along.
+
+##Further considerations
+There is some additional information that any resource writer will need to know.
+Each issue is listed separately below!
+
+###Resource struct
+Each resource will implement methods as pointer receivers on a resource struct.
+The resource struct must include an anonymous reference to the `BaseRes` struct.
+The naming convention for resources is that they end with a `Res` suffix. If
+you'd like your resource to be accessible by the `YAML` graph API (GAPI), then
+you'll need to include the appropriate YAML fields as shown below.
+
+####Example
+```golang
+type FooRes struct {
+	BaseRes `yaml:",inline"` // base properties
+
+	Whatever string `yaml:"whatever"` // you pick!
+	Bar int // no yaml, used as public output value for send/recv
+	Baz bool `yaml:"baz"` // something else
+
+	something string // some private field
+}
+```
+
+###YAML
+In addition to labelling your resource struct with YAML fields, you must also
+add an entry to the internal `GraphConfig` struct. It is a fairly straight
+forward one line patch.
+
+```golang
+type GraphConfig struct {
+// [snip...]
+	Resources struct {
+		Noop []*resources.NoopRes `yaml:"noop"`
+		File []*resources.FileRes `yaml:"file"`
+		// [snip...]
+		Foo []*resources.FooRes `yaml:"foo"` // tada :)
+	}
+}
+```
+
+###Gob registration
+All resources must be registered with the `golang` _gob_ module so that they can
+be encoded and decoded. Make sure to include the following code snippet for this
+to work.
+
+```golang
+import "encoding/gob"
+func init() { // special golang method that runs once
+	gob.Register(&FooRes{}) // substitude your resource here
+}
+```
+
+##Automatic edges
+Automatic edges in `mgmt` are well described in [this article](https://ttboj.wordpress.com/2016/03/14/automatic-edges-in-mgmt/).
+The best example of this technique can be seen in the `svc` resource.
+Unfortunately no further documentation about this subject has been written. To
+expand this section, please send a patch! Please contact us if you'd like to
+work on a resource that uses this feature, or to add it to an existing one!
+
+##Automatic grouping
+Automatic grouping in `mgmt` is well described in [this article](https://ttboj.wordpress.com/2016/03/30/automatic-grouping-in-mgmt/).
+The best example of this technique can be seen in the `pkg` resource.
+Unfortunately no further documentation about this subject has been written. To
+expand this section, please send a patch! Please contact us if you'd like to
+work on a resource that uses this feature, or to add it to an existing one!
+
+
+##Send/Recv
+In `mgmt` there is a novel concept called _Send/Recv_. For some background,
+please [read the introductory article](https://ttboj.wordpress.com/2016/12/07/sendrecv-in-mgmt/).
+When using this feature, the engine will automatically send the user specified
+value to the intended destination without requiring any resource specific code.
+Any time that one of the destination values is changed, the engine automatically
+marks the resource state as `dirty`. To detect if a particular value was
+received, and if it changed (during this invocation of CheckApply) from the
+previous value, you can query the Recv parameter. It will contain a `map` of all
+the keys which can be received on, and the value has a `Changed` property which
+will indicate whether the value was updated on this particular `CheckApply`
+invocation. The type of the sending key must match that of the receiving one.
+This can _only_ be done inside of the `CheckApply` function!
+
+```golang
+// inside CheckApply, probably near the top
+if val, exists := obj.Recv["SomeKey"]; exists {
+	log.Printf("SomeKey was sent to us from: %s[%s].%s", val.Res.Kind(), val.Res.GetName(), val.Key)
+	if val.Changed {
+		log.Printf("SomeKey was just updated!")
+		// you may want to invalidate some local cache
+	}
+}
+```
+
+Astute readers will note that there isn't anything that prevents a user from
+sending an identically typed value to some arbitrary (public) key that the
+resource author hadn't considered! While this is true, resources should probably
+work within this problem space anyways. The rule of thumb is that any public
+parameter which is normally used in a resource can be used safely.
+
+One subtle scenario is that if a resource creates a local cache or stores a
+computation that depends on the value of a public parameter and will require
+invalidation should that public parameter change, then you must detect that
+scenario and invalidate the cache when it occurs. This *must* be processed
+before there is a possibility of failure in CheckApply, because if we fail (and
+possibly run again) the subsequent send->recv transfer might not have a new
+value to copy, and therefore we won't see this notification of change.
+Therefore, it is important to process these promptly, if they must not be lost,
+such as for cache invalidation.
+
+Remember, `Send/Recv` only changes your resource code if you cache state.
+
+##Composite resources
+Composite resources are resources which embed one or more existing resources.
+This is useful to prevent code duplication in higher level resource scenarios.
+The best example of this technique can be seen in the `nspawn` resource which
+can be seen to partially embed a `svc` resource, but without its `Watch`.
+Unfortunately no further documentation about this subject has been written. To
+expand this section, please send a patch! Please contact us if you'd like to
+work on a resource that uses this feature, or to add it to an existing one!
+
+##Frequently asked questions
+(Send your questions as a patch to this FAQ! I'll review it, merge it, and
+respond by commit with the answer.)
+
+###Can I write resources in a different language?
+Currently `golang` is the only supported language for built-in resources. We
+might consider allowing external resources to be imported in the future. This
+will likely require a language that can expose a C-like API, such as `python` or
+`ruby`. Custom `golang` resources are already possible when using mgmt as a lib.
+Higher level resource collections will be possible once the `mgmt` DSL is ready.
+
+###What new resource primitives need writing?
+There are still many ideas for new resources that haven't been written yet. If
+you'd like to contribute one, please contact us and tell us about your idea!
+
+###Where can I find more information about mgmt?
+Additional blog posts, videos and other material [is available!](https://github.com/purpleidea/mgmt/#on-the-web).
+
+##Suggestions
+If you have any ideas for API changes or other improvements to resource writing,
+please let us know! We're still pre 1.0 and pre 0.1 and happy to break API in
+order to get it right!
+
+##Authors
+
+Copyright (C) 2013-2016+ James Shubin and the project contributors
+
+Please see the
+[AUTHORS](https://github.com/purpleidea/mgmt/tree/master/AUTHORS) file
+for more information.
+
+* [github](https://github.com/purpleidea/)
+* [&#64;purpleidea](https://twitter.com/#!/purpleidea)
+* [https://ttboj.wordpress.com/](https://ttboj.wordpress.com/)

etcd.go

@@ -1,294 +0,0 @@
-// Mgmt
-// Copyright (C) 2013-2016+ James Shubin and the project contributors
-// Written by James Shubin <james@shubin.ca> and the project contributors
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package main
-
-import (
-	"fmt"
-	etcd "github.com/coreos/etcd/client"
-	etcd_context "golang.org/x/net/context"
-	"log"
-	"math"
-	"strings"
-	"time"
-)
-
-//go:generate stringer -type=etcdMsg -output=etcdmsg_stringer.go
-type etcdMsg int
-
-const (
-	etcdStart etcdMsg = iota
-	etcdEvent
-	etcdFoo
-	etcdBar
-)
-
-//go:generate stringer -type=etcdConvergedState -output=etcdconvergedstate_stringer.go
-type etcdConvergedState int
-
-const (
-	etcdConvergedNil etcdConvergedState = iota
-	//etcdConverged
-	etcdConvergedTimeout
-)
-
-type EtcdWObject struct { // etcd wrapper object
-	seed           string
-	ctimeout       int
-	converged      chan bool
-	kapi           etcd.KeysAPI
-	convergedState etcdConvergedState
-}
-
-func (etcdO *EtcdWObject) GetConvergedState() etcdConvergedState {
-	return etcdO.convergedState
-}
-
-func (etcdO *EtcdWObject) SetConvergedState(state etcdConvergedState) {
-	etcdO.convergedState = state
-}
-
-func (etcdO *EtcdWObject) GetKAPI() etcd.KeysAPI {
-	if etcdO.kapi != nil { // memoize
-		return etcdO.kapi
-	}
-
-	cfg := etcd.Config{
-		Endpoints: []string{etcdO.seed},
-		Transport: etcd.DefaultTransport,
-		// set timeout per request to fail fast when the target endpoint is unavailable
-		HeaderTimeoutPerRequest: time.Second,
-	}
-
-	var c etcd.Client
-	var err error
-
-	c, err = etcd.New(cfg)
-	if err != nil {
-		// XXX: not sure if this ever errors
-		if cerr, ok := err.(*etcd.ClusterError); ok {
-			// XXX: not sure if this part ever matches
-			// not running or disconnected
-			if cerr == etcd.ErrClusterUnavailable {
-				log.Fatal("XXX: etcd: ErrClusterUnavailable")
-			} else {
-				log.Fatal("XXX: etcd: Unknown")
-			}
-		}
-		log.Fatal(err) // some unhandled error
-	}
-	etcdO.kapi = etcd.NewKeysAPI(c)
-	return etcdO.kapi
-}
-
-type EtcdChannelWatchResponse struct {
-	resp *etcd.Response
-	err  error
-}
-
-// wrap the etcd watcher.Next blocking function inside of a channel
-func (etcdO *EtcdWObject) EtcdChannelWatch(watcher etcd.Watcher, context etcd_context.Context) chan *EtcdChannelWatchResponse {
-	ch := make(chan *EtcdChannelWatchResponse)
-	go func() {
-		for {
-			resp, err := watcher.Next(context) // blocks here
-			ch <- &EtcdChannelWatchResponse{resp, err}
-		}
-	}()
-	return ch
-}
-
-func (etcdO *EtcdWObject) EtcdWatch() chan etcdMsg {
-	kapi := etcdO.GetKAPI()
-	ctimeout := etcdO.ctimeout
-	converged := etcdO.converged
-	// XXX: i think we need this buffered so that when we're hanging on the
-	// channel, which is inside the EtcdWatch main loop, we still want the
-	// calls to Get/Set on etcd to succeed, so blocking them here would
-	// kill the whole thing
-	ch := make(chan etcdMsg, 1) // XXX: buffer of at least 1 is required
-	go func(ch chan etcdMsg) {
-		tmin := 500   // initial (min) delay in ms
-		t := tmin     // current time
-		tmult := 2    // multiplier for exponential delay
-		tmax := 16000 // max delay
-		watcher := kapi.Watcher("/exported/", &etcd.WatcherOptions{Recursive: true})
-		etcdch := etcdO.EtcdChannelWatch(watcher, etcd_context.Background())
-		for {
-			log.Printf("Etcd: Watching...")
-			var resp *etcd.Response // = nil by default
-			var err error
-			select {
-			case out := <-etcdch:
-				etcdO.SetConvergedState(etcdConvergedNil)
-				resp, err = out.resp, out.err
-
-			case _ = <-TimeAfterOrBlock(ctimeout):
-				etcdO.SetConvergedState(etcdConvergedTimeout)
-				converged <- true
-				continue
-			}
-
-			if err != nil {
-				if err == etcd_context.Canceled {
-					// ctx is canceled by another routine
-					log.Fatal("Canceled")
-				} else if err == etcd_context.DeadlineExceeded {
-					// ctx is attached with a deadline and it exceeded
-					log.Fatal("Deadline")
-				} else if cerr, ok := err.(*etcd.ClusterError); ok {
-					// not running or disconnected
-					// TODO: is there a better way to parse errors?
-					for _, e := range cerr.Errors {
-						if strings.HasSuffix(e.Error(), "getsockopt: connection refused") {
-							t = int(math.Min(float64(t*tmult), float64(tmax)))
-							log.Printf("Etcd: Waiting %d ms for connection...", t)
-							time.Sleep(time.Duration(t) * time.Millisecond) // sleep for t ms
-
-						} else if e.Error() == "unexpected EOF" {
-							log.Printf("Etcd: Unexpected disconnect...")
-
-						} else if e.Error() == "EOF" {
-							log.Printf("Etcd: Disconnected...")
-
-						} else if strings.HasPrefix(e.Error(), "unsupported protocol scheme") {
-							// usually a bad peer endpoint value
-							log.Fatal("Bad peer endpoint value?")
-
-						} else {
-							log.Fatal("Woops: ", e.Error())
-						}
-					}
-				} else {
-					// bad cluster endpoints, which are not etcd servers
-					log.Fatal("Woops: ", err)
-				}
-			} else {
-				//log.Print(resp)
-				//log.Printf("Watcher().Node.Value(%v): %+v", key, resp.Node.Value)
-				// FIXME: we should actually reset when the server comes back, not here on msg!
-				//XXX: can we fix this with one of these patterns?: https://blog.golang.org/go-concurrency-patterns-timing-out-and
-				t = tmin // reset timer
-
-				// don't trigger event if nothing changed
-				if n, p := resp.Node, resp.PrevNode; resp.Action == "set" && p != nil {
-					if n.Key == p.Key && n.Value == p.Value {
-						continue
-					}
-				}
-
-				// FIXME: we get events on key/res/value changes for
-				// each res directory... ignore the non final ones...
-				// IOW, ignore everything except for the value or some
-				// field which gets set last... this could be the max count field thing...
-
-				log.Printf("Etcd: Value: %v", resp.Node.Value) // event
-				ch <- etcdEvent                                // event
-			}
-
-		} // end for loop
-		//close(ch)
-	}(ch) // call go routine
-	return ch
-}
-
-// helper function to store our data in etcd
-func (etcdO *EtcdWObject) EtcdPut(hostname, key, res string, data string) bool {
-	kapi := etcdO.GetKAPI()
-	path := fmt.Sprintf("/exported/%s/resources/%s/res", hostname, key)
-	_, err := kapi.Set(etcd_context.Background(), path, res, nil)
-	// XXX validate...
-
-	path = fmt.Sprintf("/exported/%s/resources/%s/value", hostname, key)
-	resp, err := kapi.Set(etcd_context.Background(), path, data, nil)
-	if err != nil {
-		if cerr, ok := err.(*etcd.ClusterError); ok {
-			// not running or disconnected
-			for _, e := range cerr.Errors {
-				if strings.HasSuffix(e.Error(), "getsockopt: connection refused") {
-				}
-				//if e == etcd.ErrClusterUnavailable
-			}
-		}
-		log.Printf("Etcd: Could not store %v key.", key)
-		return false
-	}
-	log.Print("Etcd: ", resp) // w00t... bonus
-	return true
-}
-
-// lookup /exported/ node hierarchy
-func (etcdO *EtcdWObject) EtcdGet() (etcd.Nodes, bool) {
-	kapi := etcdO.GetKAPI()
-	// key structure is /exported/<hostname>/resources/...
-	resp, err := kapi.Get(etcd_context.Background(), "/exported/", &etcd.GetOptions{Recursive: true})
-	if err != nil {
-		return nil, false // not found
-	}
-	return resp.Node.Nodes, true
-}
-
-func (etcdO *EtcdWObject) EtcdGetProcess(nodes etcd.Nodes, res string) []string {
-	//path := fmt.Sprintf("/exported/%s/resources/", h)
-	top := "/exported/"
-	log.Printf("Etcd: Get: %+v", nodes) // Get().Nodes.Nodes
-	var output []string
-
-	for _, x := range nodes { // loop through hosts
-		if !strings.HasPrefix(x.Key, top) {
-			log.Fatal("Error!")
-		}
-		host := x.Key[len(top):]
-		//log.Printf("Get().Nodes[%v]: %+v ==> %+v", -1, host, x.Nodes)
-		//log.Printf("Get().Nodes[%v]: %+v ==> %+v", i, x.Key, x.Nodes)
-		resources, ok := EtcdGetChildNodeByKey(x, "resources")
-		if !ok {
-			continue
-		}
-		for _, y := range resources.Nodes { // loop through resources
-			//key := y.Key # UUID?
-			//log.Printf("Get(%v): RES[%v]", host, y.Key)
-			t, ok := EtcdGetChildNodeByKey(y, "res")
-			if !ok {
-				continue
-			}
-			if res != "" && res != t.Value {
-				continue
-			} // filter based on res
-
-			v, ok := EtcdGetChildNodeByKey(y, "value") // B64ToObj this
-			if !ok {
-				continue
-			}
-			log.Printf("Etcd: Hostname: %v; Get: %v", host, t.Value)
-
-			output = append(output, v.Value)
-		}
-	}
-	return output
-}
-
-// TODO: wrap this somehow so it's a method of *etcd.Node
-// helper function that returns the node for a particular key under a node
-func EtcdGetChildNodeByKey(node *etcd.Node, key string) (*etcd.Node, bool) {
-	for _, x := range node.Nodes {
-		if x.Key == fmt.Sprintf("%s/%s", node.Key, key) {
-			return x, true
-		}
-	}
-	return nil, false // not found
-}

etcd/world.go

@@ -0,0 +1,43 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package etcd
+
+import (
+	"github.com/purpleidea/mgmt/resources"
+)
+
+// World is an etcd backed implementation of the World interface.
+type World struct {
+	Hostname string // uuid for the consumer of these
+	EmbdEtcd *EmbdEtcd
+}
+
+// ResExport exports a list of resources under our hostname namespace.
+// Subsequent calls replace the previously set collection atomically.
+func (obj *World) ResExport(resourceList []resources.Res) error {
+	return EtcdSetResources(obj.EmbdEtcd, obj.Hostname, resourceList)
+}
+
+// ResCollect gets the collection of exported resources which match the filter.
+// It does this atomically so that a call always returns a complete collection.
+func (obj *World) ResCollect(hostnameFilter, kindFilter []string) ([]resources.Res, error) {
+	// XXX: should we be restricted to retrieving resources that were
+	// exported with a tag that allows or restricts our hostname? We could
+	// enforce that here if the underlying API supported it... Add this?
+	return EtcdGetResources(obj.EmbdEtcd, hostnameFilter, kindFilter)
+}

event.go

@@ -1,75 +0,0 @@
-// Mgmt
-// Copyright (C) 2013-2016+ James Shubin and the project contributors
-// Written by James Shubin <james@shubin.ca> and the project contributors
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package main
-
-//go:generate stringer -type=eventName -output=eventname_stringer.go
-type eventName int
-
-const (
-	eventNil eventName = iota
-	eventExit
-	eventStart
-	eventPause
-	eventPoke
-	eventBackPoke
-)
-
-type Resp chan bool
-
-type Event struct {
-	Name eventName
-	Resp Resp // channel to send an ack response on, nil to skip
-	//Wg   *sync.WaitGroup // receiver barrier to Wait() for everyone else on
-	Msg      string // some words for fun
-	Activity bool   // did something interesting happen?
-}
-
-// send a single acknowledgement on the channel if one was requested
-func (event *Event) ACK() {
-	if event.Resp != nil { // if they've requested an ACK
-		event.Resp <- true // send ACK
-	}
-}
-
-func (event *Event) NACK() {
-	if event.Resp != nil { // if they've requested an ACK
-		event.Resp <- false // send NACK
-	}
-}
-
-// Resp is just a helper to return the right type of response channel
-func NewResp() Resp {
-	resp := make(chan bool)
-	return resp
-}
-
-// ACKWait waits for a +ive Ack from a Resp channel
-func (resp Resp) ACKWait() {
-	for {
-		value := <-resp
-		// wait until true value
-		if value {
-			return
-		}
-	}
-}
-
-// get the activity value
-func (event *Event) GetActivity() bool {
-	return event.Activity
-}

event/event.go

@@ -0,0 +1,120 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// Package event provides some primitives that are used for message passing.
+package event
+
+import (
+	"fmt"
+)
+
+//go:generate stringer -type=EventName -output=eventname_stringer.go
+
+// EventName represents the type of event being passed.
+type EventName int
+
+// The different event names are used in different contexts.
+const (
+	EventNil EventName = iota
+	EventExit
+	EventStart
+	EventPause
+	EventPoke
+	EventBackPoke
+)
+
+// Resp is a channel to be used for boolean responses. A nil represents an ACK,
+// and a non-nil represents a NACK (false). This also lets us use custom errors.
+type Resp chan error
+
+// Event is the main struct that stores event information and responses.
+type Event struct {
+	Name EventName
+	Resp Resp // channel to send an ack response on, nil to skip
+	//Wg   *sync.WaitGroup // receiver barrier to Wait() for everyone else on
+	Msg      string // some words for fun
+	Activity bool   // did something interesting happen?
+}
+
+// ACK sends a single acknowledgement on the channel if one was requested.
+func (event *Event) ACK() {
+	if event.Resp != nil { // if they've requested an ACK
+		event.Resp.ACK()
+	}
+}
+
+// NACK sends a negative acknowledgement message on the channel if one was requested.
+func (event *Event) NACK() {
+	if event.Resp != nil { // if they've requested a NACK
+		event.Resp.NACK()
+	}
+}
+
+// ACKNACK sends a custom ACK or NACK message on the channel if one was requested.
+func (event *Event) ACKNACK(err error) {
+	if event.Resp != nil { // if they've requested a NACK
+		event.Resp.ACKNACK(err)
+	}
+}
+
+// NewResp is just a helper to return the right type of response channel.
+func NewResp() Resp {
+	resp := make(chan error)
+	return resp
+}
+
+// ACK sends a true value to resp.
+func (resp Resp) ACK() {
+	if resp != nil {
+		resp <- nil
+	}
+}
+
+// NACK sends a false value to resp.
+func (resp Resp) NACK() {
+	if resp != nil {
+		resp <- fmt.Errorf("NACK")
+	}
+}
+
+// ACKNACK sends a custom ACK or NACK. The ACK value is always nil, the NACK can
+// be any non-nil error value.
+func (resp Resp) ACKNACK(err error) {
+	if resp != nil {
+		resp <- err
+	}
+}
+
+// Wait waits for any response from a Resp channel and returns it.
+func (resp Resp) Wait() error {
+	return <-resp
+}
+
+// ACKWait waits for a +ive Ack from a Resp channel.
+func (resp Resp) ACKWait() {
+	for {
+		// wait until true value
+		if resp.Wait() == nil {
+			return
+		}
+	}
+}
+
+// GetActivity returns the activity value.
+func (event *Event) GetActivity() bool {
+	return event.Activity
+}

examples/autogroup2.yaml

@@ -0,0 +1,17 @@
+---
+graph: mygraph
+resources:
+  pkg:
+  - name: powertop
+    meta:
+      autogroup: true
+    state: installed
+  - name: sl
+    meta:
+      autogroup: true
+    state: installed
+  - name: cowsay
+    meta:
+      autogroup: true
+    state: installed
+edges: []

examples/etcd1d.yaml

@@ -0,0 +1,18 @@
+---
+graph: mygraph
+resources:
+  file:
+  - name: file1d
+    path: "/tmp/mgmtD/f1d"
+    content: |
+      i am f1
+    state: exists
+  - name: "@@file2d"
+    path: "/tmp/mgmtD/f2d"
+    content: |
+      i am f2, exported from host D
+    state: exists
+collect:
+- kind: file
+  pattern: "/tmp/mgmtD/"
+edges: []

examples/exec3.yaml

@@ -0,0 +1,59 @@
+---
+graph: parallel
+resources:
+  exec:
+  - name: pkg10
+    cmd: sleep 10s
+    shell: ''
+    timeout: 0
+    watchcmd: ''
+    watchshell: ''
+    ifcmd: ''
+    ifshell: ''
+    pollint: 0
+    state: present
+  - name: svc10
+    cmd: sleep 10s
+    shell: ''
+    timeout: 0
+    watchcmd: ''
+    watchshell: ''
+    ifcmd: ''
+    ifshell: ''
+    pollint: 0
+    state: present
+  - name: exec10
+    cmd: sleep 10s
+    shell: ''
+    timeout: 0
+    watchcmd: ''
+    watchshell: ''
+    ifcmd: ''
+    ifshell: ''
+    pollint: 0
+    state: present
+  - name: pkg15
+    cmd: sleep 15s
+    shell: ''
+    timeout: 0
+    watchcmd: ''
+    watchshell: ''
+    ifcmd: ''
+    ifshell: ''
+    pollint: 0
+    state: present
+edges:
+- name: e1
+  from:
+    kind: exec
+    name: pkg10
+  to:
+    kind: exec
+    name: svc10
+- name: e2
+  from:
+    kind: exec
+    name: svc10
+  to:
+    kind: exec
+    name: exec10

examples/file2.yaml

@@ -0,0 +1,13 @@
+---
+graph: mygraph
+resources:
+  noop:
+  - name: noop1
+  file:
+  - name: file1
+    path: "/tmp/mgmt/hello/"
+    source: "/var/lib/mgmt/files/some_dir/"
+    recurse: true
+    force: true
+    state: exists
+edges: []

examples/file3.yaml

@@ -0,0 +1,14 @@
+---
+graph: mygraph
+comment: You can test Watch and CheckApply failures with chmod ugo-r and chmod ugo-w.
+resources:
+  file:
+  - name: file1
+    path: "/tmp/mgmt/f1"
+    meta:
+      retry: 3
+      delay: 5000
+    content: |
+      i am f1
+    state: exists
+edges: []

examples/hostname.yaml

@@ -0,0 +1,7 @@
+---
+graph: mygraph
+resources:
+  hostname:
+    - name: Hostname Watcher @ TestHost
+      hostname: test.hostname.example.com
+edges: []

examples/lib/libmgmt1.go

@@ -0,0 +1,188 @@
+// libmgmt example
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/purpleidea/mgmt/gapi"
+	mgmt "github.com/purpleidea/mgmt/lib"
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/resources"
+	"github.com/purpleidea/mgmt/yamlgraph"
+)
+
+// MyGAPI implements the main GAPI interface.
+type MyGAPI struct {
+	Name     string // graph name
+	Interval uint   // refresh interval, 0 to never refresh
+
+	data        gapi.Data
+	initialized bool
+	closeChan   chan struct{}
+	wg          sync.WaitGroup // sync group for tunnel go routines
+}
+
+// NewMyGAPI creates a new MyGAPI struct and calls Init().
+func NewMyGAPI(data gapi.Data, name string, interval uint) (*MyGAPI, error) {
+	obj := &MyGAPI{
+		Name:     name,
+		Interval: interval,
+	}
+	return obj, obj.Init(data)
+}
+
+// Init initializes the MyGAPI struct.
+func (obj *MyGAPI) Init(data gapi.Data) error {
+	if obj.initialized {
+		return fmt.Errorf("Already initialized!")
+	}
+	if obj.Name == "" {
+		return fmt.Errorf("The graph name must be specified!")
+	}
+	obj.data = data // store for later
+	obj.closeChan = make(chan struct{})
+	obj.initialized = true
+	return nil
+}
+
+// Graph returns a current Graph.
+func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
+	if !obj.initialized {
+		return nil, fmt.Errorf("libmgmt: MyGAPI is not initialized")
+	}
+
+	n1, err := resources.NewNoopRes("noop1")
+	if err != nil {
+		return nil, fmt.Errorf("Can't create resource: %v", err)
+	}
+
+	// we can still build a graph via the yaml method
+	gc := &yamlgraph.GraphConfig{
+		Graph: obj.Name,
+		Resources: yamlgraph.Resources{ // must redefine anonymous struct :(
+			// in alphabetical order
+			Exec:  []*resources.ExecRes{},
+			File:  []*resources.FileRes{},
+			Msg:   []*resources.MsgRes{},
+			Noop:  []*resources.NoopRes{n1},
+			Pkg:   []*resources.PkgRes{},
+			Svc:   []*resources.SvcRes{},
+			Timer: []*resources.TimerRes{},
+			Virt:  []*resources.VirtRes{},
+		},
+		//Collector: []collectorResConfig{},
+		//Edges:     []Edge{},
+		Comment: "comment!",
+	}
+
+	g, err := gc.NewGraphFromConfig(obj.data.Hostname, obj.data.World, obj.data.Noop)
+	return g, err
+}
+
+// Next returns nil errors every time there could be a new graph.
+func (obj *MyGAPI) Next() chan error {
+	if obj.data.NoWatch || obj.Interval <= 0 {
+		return nil
+	}
+	ch := make(chan error)
+	obj.wg.Add(1)
+	go func() {
+		defer obj.wg.Done()
+		defer close(ch) // this will run before the obj.wg.Done()
+		if !obj.initialized {
+			ch <- fmt.Errorf("libmgmt: MyGAPI is not initialized")
+			return
+		}
+
+		// arbitrarily change graph every interval seconds
+		ticker := time.NewTicker(time.Duration(obj.Interval) * time.Second)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ticker.C:
+				log.Printf("libmgmt: Generating new graph...")
+				ch <- nil // trigger a run
+			case <-obj.closeChan:
+				return
+			}
+		}
+	}()
+	return ch
+}
+
+// Close shuts down the MyGAPI.
+func (obj *MyGAPI) Close() error {
+	if !obj.initialized {
+		return fmt.Errorf("libmgmt: MyGAPI is not initialized")
+	}
+	close(obj.closeChan)
+	obj.wg.Wait()
+	obj.initialized = false // closed = true
+	return nil
+}
+
+// Run runs an embedded mgmt server.
+func Run() error {
+
+	obj := &mgmt.Main{}
+	obj.Program = "libmgmt" // TODO: set on compilation
+	obj.Version = "0.0.1"   // TODO: set on compilation
+	obj.TmpPrefix = true
+	obj.IdealClusterSize = -1
+	obj.ConvergedTimeout = -1
+	obj.Noop = true
+
+	obj.GAPI = &MyGAPI{ // graph API
+		Name:     "libmgmt", // TODO: set on compilation
+		Interval: 15,        // arbitrarily change graph every 15 seconds
+	}
+
+	if err := obj.Init(); err != nil {
+		return err
+	}
+
+	// install the exit signal handler
+	exit := make(chan struct{})
+	defer close(exit)
+	go func() {
+		signals := make(chan os.Signal, 1)
+		signal.Notify(signals, os.Interrupt) // catch ^C
+		//signal.Notify(signals, os.Kill) // catch signals
+		signal.Notify(signals, syscall.SIGTERM)
+
+		select {
+		case sig := <-signals: // any signal will do
+			if sig == os.Interrupt {
+				log.Println("Interrupted by ^C")
+				obj.Exit(nil)
+				return
+			}
+			log.Println("Interrupted by signal")
+			obj.Exit(fmt.Errorf("Killed by %v", sig))
+			return
+		case <-exit:
+			return
+		}
+	}()
+
+	if err := obj.Run(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func main() {
+	log.Printf("Hello!")
+	if err := Run(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+		return
+	}
+	log.Printf("Goodbye!")
+}

examples/lib/libmgmt2.go

@@ -0,0 +1,188 @@
+// libmgmt example
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"strconv"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/purpleidea/mgmt/gapi"
+	mgmt "github.com/purpleidea/mgmt/lib"
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/resources"
+)
+
+// MyGAPI implements the main GAPI interface.
+type MyGAPI struct {
+	Name     string // graph name
+	Count    uint   // number of resources to create
+	Interval uint   // refresh interval, 0 to never refresh
+
+	data        gapi.Data
+	initialized bool
+	closeChan   chan struct{}
+	wg          sync.WaitGroup // sync group for tunnel go routines
+}
+
+// NewMyGAPI creates a new MyGAPI struct and calls Init().
+func NewMyGAPI(data gapi.Data, name string, interval uint, count uint) (*MyGAPI, error) {
+	obj := &MyGAPI{
+		Name:     name,
+		Count:    count,
+		Interval: interval,
+	}
+	return obj, obj.Init(data)
+}
+
+// Init initializes the MyGAPI struct.
+func (obj *MyGAPI) Init(data gapi.Data) error {
+	if obj.initialized {
+		return fmt.Errorf("Already initialized!")
+	}
+	if obj.Name == "" {
+		return fmt.Errorf("The graph name must be specified!")
+	}
+	obj.data = data // store for later
+	obj.closeChan = make(chan struct{})
+	obj.initialized = true
+	return nil
+}
+
+// Graph returns a current Graph.
+func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
+	if !obj.initialized {
+		return nil, fmt.Errorf("libmgmt: MyGAPI is not initialized")
+	}
+
+	g := pgraph.NewGraph(obj.Name)
+	var vertex *pgraph.Vertex
+	for i := uint(0); i < obj.Count; i++ {
+		n, err := resources.NewNoopRes(fmt.Sprintf("noop%d", i))
+		if err != nil {
+			return nil, fmt.Errorf("Can't create resource: %v", err)
+		}
+		v := pgraph.NewVertex(n)
+		g.AddVertex(v)
+		if i > 0 {
+			g.AddEdge(vertex, v, pgraph.NewEdge(fmt.Sprintf("e%d", i)))
+		}
+		vertex = v // save
+	}
+
+	//g, err := config.NewGraphFromConfig(obj.data.Hostname, obj.data.World, obj.data.Noop)
+	return g, nil
+}
+
+// Next returns nil errors every time there could be a new graph.
+func (obj *MyGAPI) Next() chan error {
+	if obj.data.NoWatch || obj.Interval <= 0 {
+		return nil
+	}
+	ch := make(chan error)
+	obj.wg.Add(1)
+	go func() {
+		defer obj.wg.Done()
+		defer close(ch) // this will run before the obj.wg.Done()
+		if !obj.initialized {
+			ch <- fmt.Errorf("libmgmt: MyGAPI is not initialized")
+			return
+		}
+
+		// arbitrarily change graph every interval seconds
+		ticker := time.NewTicker(time.Duration(obj.Interval) * time.Second)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ticker.C:
+				log.Printf("libmgmt: Generating new graph...")
+				ch <- nil // trigger a run
+			case <-obj.closeChan:
+				return
+			}
+		}
+	}()
+	return ch
+}
+
+// Close shuts down the MyGAPI.
+func (obj *MyGAPI) Close() error {
+	if !obj.initialized {
+		return fmt.Errorf("libmgmt: MyGAPI is not initialized")
+	}
+	close(obj.closeChan)
+	obj.wg.Wait()
+	obj.initialized = false // closed = true
+	return nil
+}
+
+// Run runs an embedded mgmt server.
+func Run(count uint) error {
+
+	obj := &mgmt.Main{}
+	obj.Program = "libmgmt" // TODO: set on compilation
+	obj.Version = "0.0.1"   // TODO: set on compilation
+	obj.TmpPrefix = true
+	obj.IdealClusterSize = -1
+	obj.ConvergedTimeout = -1
+	obj.Noop = true
+
+	obj.GAPI = &MyGAPI{ // graph API
+		Name:     "libmgmt", // TODO: set on compilation
+		Count:    count,     // number of vertices to add
+		Interval: 15,        // arbitrarily change graph every 15 seconds
+	}
+
+	if err := obj.Init(); err != nil {
+		return err
+	}
+
+	// install the exit signal handler
+	exit := make(chan struct{})
+	defer close(exit)
+	go func() {
+		signals := make(chan os.Signal, 1)
+		signal.Notify(signals, os.Interrupt) // catch ^C
+		//signal.Notify(signals, os.Kill) // catch signals
+		signal.Notify(signals, syscall.SIGTERM)
+
+		select {
+		case sig := <-signals: // any signal will do
+			if sig == os.Interrupt {
+				log.Println("Interrupted by ^C")
+				obj.Exit(nil)
+				return
+			}
+			log.Println("Interrupted by signal")
+			obj.Exit(fmt.Errorf("Killed by %v", sig))
+			return
+		case <-exit:
+			return
+		}
+	}()
+
+	if err := obj.Run(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func main() {
+	log.Printf("Hello!")
+	var count uint = 1 // default
+	if len(os.Args) == 2 {
+		if i, err := strconv.Atoi(os.Args[1]); err == nil && i > 0 {
+			count = uint(i)
+		}
+	}
+	if err := Run(count); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+		return
+	}
+	log.Printf("Goodbye!")
+}

examples/lib/libmgmt3.go

@@ -0,0 +1,225 @@
+// libmgmt example of send->recv
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/purpleidea/mgmt/gapi"
+	mgmt "github.com/purpleidea/mgmt/lib"
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/resources"
+)
+
+// MyGAPI implements the main GAPI interface.
+type MyGAPI struct {
+	Name     string // graph name
+	Interval uint   // refresh interval, 0 to never refresh
+
+	data        gapi.Data
+	initialized bool
+	closeChan   chan struct{}
+	wg          sync.WaitGroup // sync group for tunnel go routines
+}
+
+// NewMyGAPI creates a new MyGAPI struct and calls Init().
+func NewMyGAPI(data gapi.Data, name string, interval uint) (*MyGAPI, error) {
+	obj := &MyGAPI{
+		Name:     name,
+		Interval: interval,
+	}
+	return obj, obj.Init(data)
+}
+
+// Init initializes the MyGAPI struct.
+func (obj *MyGAPI) Init(data gapi.Data) error {
+	if obj.initialized {
+		return fmt.Errorf("Already initialized!")
+	}
+	if obj.Name == "" {
+		return fmt.Errorf("The graph name must be specified!")
+	}
+	obj.data = data // store for later
+	obj.closeChan = make(chan struct{})
+	obj.initialized = true
+	return nil
+}
+
+// Graph returns a current Graph.
+func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
+	if !obj.initialized {
+		return nil, fmt.Errorf("libmgmt: MyGAPI is not initialized")
+	}
+
+	g := pgraph.NewGraph(obj.Name)
+
+	content := "Delete me to trigger a notification!\n"
+	f0 := &resources.FileRes{
+		BaseRes: resources.BaseRes{
+			Name: "README",
+		},
+		Path:    "/tmp/mgmt/README",
+		Content: &content,
+		State:   "present",
+	}
+
+	v0 := pgraph.NewVertex(f0)
+	g.AddVertex(v0)
+
+	p1 := &resources.PasswordRes{
+		BaseRes: resources.BaseRes{
+			Name: "password1",
+		},
+		Length: 8,    // generated string will have this many characters
+		Saved:  true, // this causes passwords to be stored in plain text!
+	}
+	v1 := pgraph.NewVertex(p1)
+	g.AddVertex(v1)
+
+	f1 := &resources.FileRes{
+		BaseRes: resources.BaseRes{
+			Name: "file1",
+			// send->recv!
+			Recv: map[string]*resources.Send{
+				"Content": {Res: p1, Key: "Password"},
+			},
+		},
+		Path: "/tmp/mgmt/secret",
+		//Content:  p1.Password, // won't work
+		State: "present",
+	}
+
+	v2 := pgraph.NewVertex(f1)
+	g.AddVertex(v2)
+
+	n1 := &resources.NoopRes{
+		BaseRes: resources.BaseRes{
+			Name: "noop1",
+		},
+	}
+
+	v3 := pgraph.NewVertex(n1)
+	g.AddVertex(v3)
+
+	e0 := pgraph.NewEdge("e0")
+	e0.Notify = true // send a notification from v0 to v1
+	g.AddEdge(v0, v1, e0)
+
+	g.AddEdge(v1, v2, pgraph.NewEdge("e1"))
+
+	e2 := pgraph.NewEdge("e2")
+	e2.Notify = true // send a notification from v2 to v3
+	g.AddEdge(v2, v3, e2)
+
+	//g, err := config.NewGraphFromConfig(obj.data.Hostname, obj.data.World, obj.data.Noop)
+	return g, nil
+}
+
+// Next returns nil errors every time there could be a new graph.
+func (obj *MyGAPI) Next() chan error {
+	if obj.data.NoWatch || obj.Interval <= 0 {
+		return nil
+	}
+	ch := make(chan error)
+	obj.wg.Add(1)
+	go func() {
+		defer obj.wg.Done()
+		defer close(ch) // this will run before the obj.wg.Done()
+		if !obj.initialized {
+			ch <- fmt.Errorf("libmgmt: MyGAPI is not initialized")
+			return
+		}
+
+		// arbitrarily change graph every interval seconds
+		ticker := time.NewTicker(time.Duration(obj.Interval) * time.Second)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ticker.C:
+				log.Printf("libmgmt: Generating new graph...")
+				ch <- nil // trigger a run
+			case <-obj.closeChan:
+				return
+			}
+		}
+	}()
+	return ch
+}
+
+// Close shuts down the MyGAPI.
+func (obj *MyGAPI) Close() error {
+	if !obj.initialized {
+		return fmt.Errorf("libmgmt: MyGAPI is not initialized")
+	}
+	close(obj.closeChan)
+	obj.wg.Wait()
+	obj.initialized = false // closed = true
+	return nil
+}
+
+// Run runs an embedded mgmt server.
+func Run() error {
+
+	obj := &mgmt.Main{}
+	obj.Program = "libmgmt" // TODO: set on compilation
+	obj.Version = "0.0.1"   // TODO: set on compilation
+	obj.TmpPrefix = true    // disable for easy debugging
+	//prefix := "/tmp/testprefix/"
+	//obj.Prefix = &p // enable for easy debugging
+	obj.IdealClusterSize = -1
+	obj.ConvergedTimeout = -1
+	obj.Noop = false // FIXME: careful!
+
+	obj.GAPI = &MyGAPI{ // graph API
+		Name:     "libmgmt", // TODO: set on compilation
+		Interval: 60 * 10,   // arbitrarily change graph every 15 seconds
+	}
+
+	if err := obj.Init(); err != nil {
+		return err
+	}
+
+	// install the exit signal handler
+	exit := make(chan struct{})
+	defer close(exit)
+	go func() {
+		signals := make(chan os.Signal, 1)
+		signal.Notify(signals, os.Interrupt) // catch ^C
+		//signal.Notify(signals, os.Kill) // catch signals
+		signal.Notify(signals, syscall.SIGTERM)
+
+		select {
+		case sig := <-signals: // any signal will do
+			if sig == os.Interrupt {
+				log.Println("Interrupted by ^C")
+				obj.Exit(nil)
+				return
+			}
+			log.Println("Interrupted by signal")
+			obj.Exit(fmt.Errorf("Killed by %v", sig))
+			return
+		case <-exit:
+			return
+		}
+	}()
+
+	if err := obj.Run(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func main() {
+	log.Printf("Hello!")
+	if err := Run(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+		return
+	}
+	log.Printf("Goodbye!")
+}

examples/msg1.yaml

@@ -0,0 +1,19 @@
+---
+graph: mygraph
+comment: timer example
+resources:
+  timer:
+  - name: timer1
+    interval: 30
+  msg:
+  - name: msg1
+    body: mgmt logged this message
+    journal: true
+edges:
+- name: e1
+  from:
+    kind: timer
+    name: timer1
+  to:
+    kind: msg
+    name: msg1

examples/noop1.yaml

@@ -0,0 +1,24 @@
+---
+graph: mygraph
+comment: noop example
+resources:
+  noop:
+  - name: noop1
+    meta:
+      noop: true
+  file:
+  - name: file1
+    path: "/tmp/mgmt-hello-noop"
+    content: |
+      hello world from @purpleidea
+    state: exists
+    meta:
+      noop: true
+edges:
+- name: e1
+  from:
+    kind: noop
+    name: noop1
+  to:
+    kind: file
+    name: file1

examples/nspawn1.yaml

@@ -0,0 +1,7 @@
+---
+graph: mygraph
+resources:
+  nspawn:
+  - name: mgmt-nspawn1
+    state: running
+edges: []

examples/nspawn2.yaml

@@ -0,0 +1,7 @@
+---
+graph: mygraph
+resources:
+  nspawn:
+  - name: mgmt-nspawn2
+    state: stopped
+edges: []

examples/remote1.yaml

@@ -0,0 +1,23 @@
+---
+graph: mygraph
+comment: remote noop example
+resources:
+  noop:
+  - name: noop1
+    meta:
+      noop: true
+  file:
+  - name: file1
+    path: "/tmp/mgmt-remote-hello"
+    content: |
+      hello world from @purpleidea
+    state: exists
+edges:
+- name: e1
+  from:
+    kind: noop
+    name: noop1
+  to:
+    kind: file
+    name: file1
+remote: "ssh://root:password@hostname:22"

examples/remote2a.yaml

@@ -0,0 +1,20 @@
+---
+graph: mygraph
+comment: remote noop example
+resources:
+  file:
+  - name: file1a
+    path: "/tmp/file1a"
+    content: |
+      i am file1a
+    state: exists
+  - name: "@@file2a"
+    path: "/tmp/file2a"
+    content: |
+      i am file2a, exported from host a
+    state: exists
+collect:
+- kind: file
+  pattern: "/tmp/"
+edges: []
+remote: ssh://root:vagrant@192.168.121.201:22

examples/remote2b.yaml

@@ -0,0 +1,20 @@
+---
+graph: mygraph
+comment: remote noop example
+resources:
+  file:
+  - name: file1b
+    path: "/tmp/file1b"
+    content: |
+      i am file1b
+    state: exists
+  - name: "@@file2b"
+    path: "/tmp/file2b"
+    content: |
+      i am file2b, exported from host b
+    state: exists
+collect:
+- kind: file
+  pattern: "/tmp/"
+edges: []
+remote: ssh://root:vagrant@192.168.121.202:22

examples/timer1.yaml

@@ -0,0 +1,25 @@
+---
+graph: mygraph
+comment: timer example
+resources:
+  timer:
+  - name: timer1
+    interval: 30
+  exec:
+  - name: exec1
+    cmd: echo hello world
+    timeout: 0
+    watchcmd: ''
+    watchshell: ''
+    ifcmd: ''
+    ifshell: ''
+    pollint: 0
+    state: present
+edges:
+- name: e1
+  from:
+    kind: timer
+    name: timer1
+  to:
+    kind: exec
+    name: exec1

examples/timer2.yaml

@@ -0,0 +1,43 @@
+---
+graph: mygraph
+comment: example of multiple timers
+resources:
+  timer:
+  - name: timer1
+    interval: 30
+  - name: timer2
+    interval: 60
+  exec:
+  - name: exec1
+    cmd: echo hello world 30
+    timeout: 0
+    watchcmd: ''
+    watchshell: ''
+    ifcmd: ''
+    ifshell: ''
+    pollint: 0
+    state: present
+  - name: exec2
+    cmd: echo hello world 60
+    timeout: 0
+    watchcmd: ''
+    watchshell: ''
+    ifcmd: ''
+    ifshell: ''
+    pollint: 0
+    state: present
+edges:
+- name: e1
+  from:
+    kind: timer
+    name: timer1
+  to:
+    kind: exec
+    name: exec1
+- name: e2
+  from:
+    kind: timer
+    name: timer2
+  to:
+    kind: exec
+    name: exec2

examples/virt1.yaml

@@ -0,0 +1,11 @@
+---
+graph: mygraph
+resources:
+  virt:
+  - name: mgmt1
+    uri: 'qemu:///session'
+    cpus: 1
+    memory: 524288
+    state: shutoff
+    transient: true
+edges: []

examples/virt2.yaml

@@ -0,0 +1,11 @@
+---
+graph: mygraph
+resources:
+  virt:
+  - name: mgmt2
+    uri: 'qemu:///session'
+    cpus: 1
+    memory: 524288
+    state: shutoff
+    transient: false
+edges: []

examples/virt3.yaml

@@ -0,0 +1,11 @@
+---
+graph: mygraph
+resources:
+  virt:
+  - name: mgmt3
+    uri: 'qemu:///session'
+    cpus: 1
+    memory: 524288
+    state: running
+    transient: false
+edges: []

file.go

@@ -1,502 +0,0 @@
-// Mgmt
-// Copyright (C) 2013-2016+ James Shubin and the project contributors
-// Written by James Shubin <james@shubin.ca> and the project contributors
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package main
-
-import (
-	"crypto/sha256"
-	"encoding/hex"
-	"gopkg.in/fsnotify.v1"
-	//"github.com/go-fsnotify/fsnotify" // git master of "gopkg.in/fsnotify.v1"
-	"encoding/gob"
-	"io"
-	"log"
-	"math"
-	"os"
-	"path"
-	"strings"
-	"syscall"
-)
-
-func init() {
-	gob.Register(&FileRes{})
-}
-
-type FileRes struct {
-	BaseRes   `yaml:",inline"`
-	Path      string `yaml:"path"` // path variable (should default to name)
-	Dirname   string `yaml:"dirname"`
-	Basename  string `yaml:"basename"`
-	Content   string `yaml:"content"`
-	State     string `yaml:"state"` // state: exists/present?, absent, (undefined?)
-	sha256sum string
-}
-
-func NewFileRes(name, path, dirname, basename, content, state string) *FileRes {
-	// FIXME if path = nil, path = name ...
-	obj := &FileRes{
-		BaseRes: BaseRes{
-			Name: name,
-		},
-		Path:      path,
-		Dirname:   dirname,
-		Basename:  basename,
-		Content:   content,
-		State:     state,
-		sha256sum: "",
-	}
-	obj.Init()
-	return obj
-}
-
-func (obj *FileRes) Init() {
-	obj.BaseRes.kind = "File"
-	obj.BaseRes.Init() // call base init, b/c we're overriding
-}
-
-func (obj *FileRes) GetPath() string {
-	d := Dirname(obj.Path)
-	b := Basename(obj.Path)
-	if !obj.Validate() || (obj.Dirname == "" && obj.Basename == "") {
-		return obj.Path
-	} else if obj.Dirname == "" {
-		return d + obj.Basename
-	} else if obj.Basename == "" {
-		return obj.Dirname + b
-	} else { // if obj.dirname != "" && obj.basename != "" {
-		return obj.Dirname + obj.Basename
-	}
-}
-
-// validate if the params passed in are valid data
-func (obj *FileRes) Validate() bool {
-	if obj.Dirname != "" {
-		// must end with /
-		if obj.Dirname[len(obj.Dirname)-1:] != "/" {
-			return false
-		}
-	}
-	if obj.Basename != "" {
-		// must not start with /
-		if obj.Basename[0:1] == "/" {
-			return false
-		}
-	}
-	return true
-}
-
-// File watcher for files and directories
-// Modify with caution, probably important to write some test cases first!
-// obj.GetPath(): file or directory
-func (obj *FileRes) Watch(processChan chan Event) {
-	if obj.IsWatching() {
-		return
-	}
-	obj.SetWatching(true)
-	defer obj.SetWatching(false)
-
-	//var recursive bool = false
-	//var isdir = (obj.GetPath()[len(obj.GetPath())-1:] == "/") // dirs have trailing slashes
-	//log.Printf("IsDirectory: %v", isdir)
-	//vertex := obj.GetVertex()         // stored with SetVertex
-	var safename = path.Clean(obj.GetPath()) // no trailing slash
-
-	watcher, err := fsnotify.NewWatcher()
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer watcher.Close()
-
-	patharray := PathSplit(safename) // tokenize the path
-	var index = len(patharray)       // starting index
-	var current string               // current "watcher" location
-	var deltaDepth int               // depth delta between watcher and event
-	var send = false                 // send event?
-	var exit = false
-	var dirty = false
-
-	for {
-		current = strings.Join(patharray[0:index], "/")
-		if current == "" { // the empty string top is the root dir ("/")
-			current = "/"
-		}
-		if DEBUG {
-			log.Printf("File[%v]: Watching: %v", obj.GetName(), current) // attempting to watch...
-		}
-		// initialize in the loop so that we can reset on rm-ed handles
-		err = watcher.Add(current)
-		if err != nil {
-			if DEBUG {
-				log.Printf("File[%v]: watcher.Add(%v): Error: %v", obj.GetName(), current, err)
-			}
-			if err == syscall.ENOENT {
-				index-- // usually not found, move up one dir
-			} else if err == syscall.ENOSPC {
-				// XXX: occasionally: no space left on device,
-				// XXX: probably due to lack of inotify watches
-				log.Printf("%v[%v]: Out of inotify watches!", obj.Kind(), obj.GetName())
-				log.Fatal(err)
-			} else {
-				log.Printf("Unknown file[%v] error:", obj.Name)
-				log.Fatal(err)
-			}
-			index = int(math.Max(1, float64(index)))
-			continue
-		}
-
-		obj.SetState(resStateWatching) // reset
-		select {
-		case event := <-watcher.Events:
-			if DEBUG {
-				log.Printf("File[%v]: Watch(%v), Event(%v): %v", obj.GetName(), current, event.Name, event.Op)
-			}
-			obj.SetConvergedState(resConvergedNil) // XXX: technically i can detect if the event is erroneous or not first
-			// the deeper you go, the bigger the deltaDepth is...
-			// this is the difference between what we're watching,
-			// and the event... doesn't mean we can't watch deeper
-			if current == event.Name {
-				deltaDepth = 0 // i was watching what i was looking for
-
-			} else if HasPathPrefix(event.Name, current) {
-				deltaDepth = len(PathSplit(current)) - len(PathSplit(event.Name)) // -1 or less
-
-			} else if HasPathPrefix(current, event.Name) {
-				deltaDepth = len(PathSplit(event.Name)) - len(PathSplit(current)) // +1 or more
-
-			} else {
-				// TODO different watchers get each others events!
-				// https://github.com/go-fsnotify/fsnotify/issues/95
-				// this happened with two values such as:
-				// event.Name: /tmp/mgmt/f3 and current: /tmp/mgmt/f2
-				continue
-			}
-			//log.Printf("The delta depth is: %v", deltaDepth)
-
-			// if we have what we wanted, awesome, send an event...
-			if event.Name == safename {
-				//log.Println("Event!")
-				// FIXME: should all these below cases trigger?
-				send = true
-				dirty = true
-
-				// file removed, move the watch upwards
-				if deltaDepth >= 0 && (event.Op&fsnotify.Remove == fsnotify.Remove) {
-					//log.Println("Removal!")
-					watcher.Remove(current)
-					index--
-				}
-
-				// we must be a parent watcher, so descend in
-				if deltaDepth < 0 {
-					watcher.Remove(current)
-					index++
-				}
-
-				// if safename starts with event.Name, we're above, and no event should be sent
-			} else if HasPathPrefix(safename, event.Name) {
-				//log.Println("Above!")
-
-				if deltaDepth >= 0 && (event.Op&fsnotify.Remove == fsnotify.Remove) {
-					log.Println("Removal!")
-					watcher.Remove(current)
-					index--
-				}
-
-				if deltaDepth < 0 {
-					log.Println("Parent!")
-					if PathPrefixDelta(safename, event.Name) == 1 { // we're the parent dir
-						send = true
-						dirty = true
-					}
-					watcher.Remove(current)
-					index++
-				}
-
-				// if event.Name startswith safename, send event, we're already deeper
-			} else if HasPathPrefix(event.Name, safename) {
-				//log.Println("Event2!")
-				send = true
-				dirty = true
-			}
-
-		case err := <-watcher.Errors:
-			obj.SetConvergedState(resConvergedNil) // XXX ?
-			log.Printf("error: %v", err)
-			log.Fatal(err)
-			//obj.events <- fmt.Sprintf("file: %v", "error") // XXX: how should we handle errors?
-
-		case event := <-obj.events:
-			obj.SetConvergedState(resConvergedNil)
-			if exit, send = obj.ReadEvent(&event); exit {
-				return // exit
-			}
-			//dirty = false // these events don't invalidate state
-
-		case _ = <-TimeAfterOrBlock(obj.ctimeout):
-			obj.SetConvergedState(resConvergedTimeout)
-			obj.converged <- true
-			continue
-		}
-
-		// do all our event sending all together to avoid duplicate msgs
-		if send {
-			send = false
-			// only invalid state on certain types of events
-			if dirty {
-				dirty = false
-				obj.isStateOK = false // something made state dirty
-			}
-			resp := NewResp()
-			processChan <- Event{eventNil, resp, "", true} // trigger process
-			resp.ACKWait()                                 // wait for the ACK()
-		}
-	}
-}
-
-func (obj *FileRes) HashSHA256fromContent() string {
-	if obj.sha256sum != "" { // return if already computed
-		return obj.sha256sum
-	}
-
-	hash := sha256.New()
-	hash.Write([]byte(obj.Content))
-	obj.sha256sum = hex.EncodeToString(hash.Sum(nil))
-	return obj.sha256sum
-}
-
-func (obj *FileRes) FileHashSHA256Check() (bool, error) {
-	if PathIsDir(obj.GetPath()) { // assert
-		log.Fatal("This should only be called on a File resource.")
-	}
-	// run a diff, and return true if it needs changing
-	hash := sha256.New()
-	f, err := os.Open(obj.GetPath())
-	if err != nil {
-		if e, ok := err.(*os.PathError); ok && (e.Err.(syscall.Errno) == syscall.ENOENT) {
-			return false, nil // no "error", file is just absent
-		}
-		return false, err
-	}
-	defer f.Close()
-	if _, err := io.Copy(hash, f); err != nil {
-		return false, err
-	}
-	sha256sum := hex.EncodeToString(hash.Sum(nil))
-	//log.Printf("sha256sum: %v", sha256sum)
-	if obj.HashSHA256fromContent() == sha256sum {
-		return true, nil
-	}
-	return false, nil
-}
-
-func (obj *FileRes) FileApply() error {
-	if PathIsDir(obj.GetPath()) {
-		log.Fatal("This should only be called on a File resource.")
-	}
-
-	if obj.State == "absent" {
-		log.Printf("About to remove: %v", obj.GetPath())
-		err := os.Remove(obj.GetPath())
-		return err // either nil or not, for success or failure
-	}
-
-	f, err := os.Create(obj.GetPath())
-	if err != nil {
-		return nil
-	}
-	defer f.Close()
-
-	_, err = io.WriteString(f, obj.Content)
-	if err != nil {
-		return err
-	}
-
-	return nil // success
-}
-
-func (obj *FileRes) CheckApply(apply bool) (stateok bool, err error) {
-	log.Printf("%v[%v]: CheckApply(%t)", obj.Kind(), obj.GetName(), apply)
-
-	if obj.isStateOK { // cache the state
-		return true, nil
-	}
-
-	if _, err = os.Stat(obj.GetPath()); os.IsNotExist(err) {
-		// no such file or directory
-		if obj.State == "absent" {
-			// missing file should be missing, phew :)
-			obj.isStateOK = true
-			return true, nil
-		}
-	}
-	err = nil // reset
-
-	// FIXME: add file mode check here...
-
-	if PathIsDir(obj.GetPath()) {
-		log.Fatal("Not implemented!") // XXX
-	} else {
-		ok, err := obj.FileHashSHA256Check()
-		if err != nil {
-			return false, err
-		}
-		if ok {
-			obj.isStateOK = true
-			return true, nil
-		}
-		// if no err, but !ok, then we continue on...
-	}
-
-	// state is not okay, no work done, exit, but without error
-	if !apply {
-		return false, nil
-	}
-
-	// apply portion
-	log.Printf("%v[%v]: Apply", obj.Kind(), obj.GetName())
-	if PathIsDir(obj.GetPath()) {
-		log.Fatal("Not implemented!") // XXX
-	} else {
-		err = obj.FileApply()
-		if err != nil {
-			return false, err
-		}
-	}
-
-	obj.isStateOK = true
-	return false, nil // success
-}
-
-type FileUUID struct {
-	BaseUUID
-	path string
-}
-
-// if and only if they are equivalent, return true
-// if they are not equivalent, return false
-func (obj *FileUUID) IFF(uuid ResUUID) bool {
-	res, ok := uuid.(*FileUUID)
-	if !ok {
-		return false
-	}
-	return obj.path == res.path
-}
-
-type FileResAutoEdges struct {
-	data    []ResUUID
-	pointer int
-	found   bool
-}
-
-func (obj *FileResAutoEdges) Next() []ResUUID {
-	if obj.found {
-		log.Fatal("Shouldn't be called anymore!")
-	}
-	if len(obj.data) == 0 { // check length for rare scenarios
-		return nil
-	}
-	value := obj.data[obj.pointer]
-	obj.pointer++
-	return []ResUUID{value} // we return one, even though api supports N
-}
-
-// get results of the earlier Next() call, return if we should continue!
-func (obj *FileResAutoEdges) Test(input []bool) bool {
-	// if there aren't any more remaining
-	if len(obj.data) <= obj.pointer {
-		return false
-	}
-	if obj.found { // already found, done!
-		return false
-	}
-	if len(input) != 1 { // in case we get given bad data
-		log.Fatal("Expecting a single value!")
-	}
-	if input[0] { // if a match is found, we're done!
-		obj.found = true // no more to find!
-		return false
-	}
-	return true // keep going
-}
-
-// generate a simple linear sequence of each parent directory from bottom up!
-func (obj *FileRes) AutoEdges() AutoEdge {
-	var data []ResUUID                             // store linear result chain here...
-	values := PathSplitFullReversed(obj.GetPath()) // build it
-	_, values = values[0], values[1:]              // get rid of first value which is me!
-	for _, x := range values {
-		var reversed = true // cheat by passing a pointer
-		data = append(data, &FileUUID{
-			BaseUUID: BaseUUID{
-				name:     obj.GetName(),
-				kind:     obj.Kind(),
-				reversed: &reversed,
-			},
-			path: x, // what matters
-		}) // build list
-	}
-	return &FileResAutoEdges{
-		data:    data,
-		pointer: 0,
-		found:   false,
-	}
-}
-
-func (obj *FileRes) GetUUIDs() []ResUUID {
-	x := &FileUUID{
-		BaseUUID: BaseUUID{name: obj.GetName(), kind: obj.Kind()},
-		path:     obj.GetPath(),
-	}
-	return []ResUUID{x}
-}
-
-func (obj *FileRes) GroupCmp(r Res) bool {
-	_, ok := r.(*FileRes)
-	if !ok {
-		return false
-	}
-	// TODO: we might be able to group directory children into a single
-	// recursive watcher in the future, thus saving fanotify watches
-	return false // not possible atm
-}
-
-func (obj *FileRes) Compare(res Res) bool {
-	switch res.(type) {
-	case *FileRes:
-		res := res.(*FileRes)
-		if obj.Name != res.Name {
-			return false
-		}
-		if obj.GetPath() != res.Path {
-			return false
-		}
-		if obj.Content != res.Content {
-			return false
-		}
-		if obj.State != res.State {
-			return false
-		}
-	default:
-		return false
-	}
-	return true
-}
-
-func (obj *FileRes) CollectPattern(pattern string) {
-	// XXX: currently the pattern for files can only override the Dirname variable :P
-	obj.Dirname = pattern // XXX: simplistic for now
-}

gapi/gapi.go

@@ -0,0 +1,50 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// Package gapi defines the interface that graph API generators must meet.
+package gapi
+
+import (
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/resources"
+)
+
+// World is an interface to the rest of the different graph state. It allows
+// the GAPI to store state and exchange information throughout the cluster. It
+// is the interface each machine uses to communicate with the rest of the world.
+type World interface { // TODO: is there a better name for this interface?
+	ResExport([]resources.Res) error
+	// FIXME: should this method take a "filter" data struct instead of many args?
+	ResCollect(hostnameFilter, kindFilter []string) ([]resources.Res, error)
+}
+
+// Data is the set of input values passed into the GAPI structs via Init.
+type Data struct {
+	Hostname string // uuid for the host, required for GAPI
+	World    World
+	Noop     bool
+	NoWatch  bool
+	// NOTE: we can add more fields here if needed by GAPI endpoints
+}
+
+// GAPI is a Graph API that represents incoming graphs and change streams.
+type GAPI interface {
+	Init(Data) error               // initializes the GAPI and passes in useful data
+	Graph() (*pgraph.Graph, error) // returns the most recent pgraph
+	Next() chan error              // returns a stream of switch events
+	Close() error                  // shutdown the GAPI
+}

gopath/.gitignore

@@ -0,0 +1,2 @@
+bin/
+pkg/

gopath/src

@@ -0,0 +1 @@
+../vendor

lib/cli.go

@@ -0,0 +1,328 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package lib
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/purpleidea/mgmt/puppet"
+	"github.com/purpleidea/mgmt/yamlgraph"
+
+	"github.com/urfave/cli"
+)
+
+// run is the main run target.
+func run(c *cli.Context) error {
+
+	obj := &Main{}
+
+	obj.Program = c.App.Name
+	obj.Version = c.App.Version
+	if val, exists := c.App.Metadata["flags"]; exists {
+		if flags, ok := val.(Flags); ok {
+			obj.Flags = flags
+		}
+	}
+
+	if h := c.String("hostname"); c.IsSet("hostname") && h != "" {
+		obj.Hostname = &h
+	}
+
+	if s := c.String("prefix"); c.IsSet("prefix") && s != "" {
+		obj.Prefix = &s
+	}
+	obj.TmpPrefix = c.Bool("tmp-prefix")
+	obj.AllowTmpPrefix = c.Bool("allow-tmp-prefix")
+
+	if _ = c.String("code"); c.IsSet("code") {
+		if obj.GAPI != nil {
+			return fmt.Errorf("Can't combine code GAPI with existing GAPI.")
+		}
+		// TODO: implement DSL GAPI
+		//obj.GAPI = &dsl.GAPI{
+		//	Code: &s,
+		//}
+		return fmt.Errorf("The Code GAPI is not implemented yet!") // TODO: DSL
+	}
+	if y := c.String("yaml"); c.IsSet("yaml") {
+		if obj.GAPI != nil {
+			return fmt.Errorf("Can't combine YAML GAPI with existing GAPI.")
+		}
+		obj.GAPI = &yamlgraph.GAPI{
+			File: &y,
+		}
+	}
+	if p := c.String("puppet"); c.IsSet("puppet") {
+		if obj.GAPI != nil {
+			return fmt.Errorf("Can't combine puppet GAPI with existing GAPI.")
+		}
+		obj.GAPI = &puppet.GAPI{
+			PuppetParam: &p,
+			PuppetConf:  c.String("puppet-conf"),
+		}
+	}
+	obj.Remotes = c.StringSlice("remote") // FIXME: GAPI-ify somehow?
+
+	obj.NoWatch = c.Bool("no-watch")
+	obj.Noop = c.Bool("noop")
+	obj.Graphviz = c.String("graphviz")
+	obj.GraphvizFilter = c.String("graphviz-filter")
+	obj.ConvergedTimeout = c.Int("converged-timeout")
+	obj.MaxRuntime = uint(c.Int("max-runtime"))
+
+	obj.Seeds = c.StringSlice("seeds")
+	obj.ClientURLs = c.StringSlice("client-urls")
+	obj.ServerURLs = c.StringSlice("server-urls")
+	obj.IdealClusterSize = c.Int("ideal-cluster-size")
+	obj.NoServer = c.Bool("no-server")
+
+	obj.CConns = uint16(c.Int("cconns"))
+	obj.AllowInteractive = c.Bool("allow-interactive")
+	obj.SSHPrivIDRsa = c.String("ssh-priv-id-rsa")
+	obj.NoCaching = c.Bool("no-caching")
+	obj.Depth = uint16(c.Int("depth"))
+
+	obj.NoPgp = c.Bool("no-pgp")
+
+	if kp := c.String("pgp-key-path"); c.IsSet("pgp-key-path") {
+		obj.PgpKeyPath = &kp
+	}
+
+	if us := c.String("pgp-identity"); c.IsSet("pgp-identity") {
+		obj.PgpIdentity = &us
+	}
+
+	if err := obj.Init(); err != nil {
+		return err
+	}
+
+	// install the exit signal handler
+	exit := make(chan struct{})
+	defer close(exit)
+	go func() {
+		signals := make(chan os.Signal, 1)
+		signal.Notify(signals, os.Interrupt) // catch ^C
+		//signal.Notify(signals, os.Kill) // catch signals
+		signal.Notify(signals, syscall.SIGTERM)
+
+		select {
+		case sig := <-signals: // any signal will do
+			if sig == os.Interrupt {
+				log.Println("Interrupted by ^C")
+				obj.Exit(nil)
+				return
+			}
+			log.Println("Interrupted by signal")
+			obj.Exit(fmt.Errorf("Killed by %v", sig))
+			return
+		case <-exit:
+			return
+		}
+	}()
+
+	if err := obj.Run(); err != nil {
+		return err
+		//return cli.NewExitError(err.Error(), 1) // TODO: ?
+		//return cli.NewExitError("", 1) // TODO: ?
+	}
+	return nil
+}
+
+// CLI is the entry point for using mgmt normally from the CLI.
+func CLI(program, version string, flags Flags) error {
+
+	// test for sanity
+	if program == "" || version == "" {
+		return fmt.Errorf("Program was not compiled correctly. Please see Makefile.")
+	}
+	app := cli.NewApp()
+	app.Name = program // App.name and App.version pass these values through
+	app.Version = version
+	app.Usage = "next generation config management"
+	app.Metadata = map[string]interface{}{ // additional flags
+		"flags": flags,
+	}
+	//app.Action = ... // without a default action, help runs
+
+	app.Commands = []cli.Command{
+		{
+			Name:    "run",
+			Aliases: []string{"r"},
+			Usage:   "run",
+			Action:  run,
+			Flags: []cli.Flag{
+				// useful for testing multiple instances on same machine
+				cli.StringFlag{
+					Name:  "hostname",
+					Value: "",
+					Usage: "hostname to use",
+				},
+
+				cli.StringFlag{
+					Name:   "prefix",
+					Usage:  "specify a path to the working prefix directory",
+					EnvVar: "MGMT_PREFIX",
+				},
+				cli.BoolFlag{
+					Name:  "tmp-prefix",
+					Usage: "request a pseudo-random, temporary prefix to be used",
+				},
+				cli.BoolFlag{
+					Name:  "allow-tmp-prefix",
+					Usage: "allow creation of a new temporary prefix if main prefix is unavailable",
+				},
+
+				cli.StringFlag{
+					Name:  "code, c",
+					Value: "",
+					Usage: "code definition to run",
+				},
+				cli.StringFlag{
+					Name:  "yaml",
+					Value: "",
+					Usage: "yaml graph definition to run",
+				},
+				cli.StringFlag{
+					Name:  "puppet, p",
+					Value: "",
+					Usage: "load graph from puppet, optionally takes a manifest or path to manifest file",
+				},
+				cli.StringFlag{
+					Name:  "puppet-conf",
+					Value: "",
+					Usage: "the path to an alternate puppet.conf file",
+				},
+				cli.StringSliceFlag{
+					Name:  "remote",
+					Value: &cli.StringSlice{},
+					Usage: "list of remote graph definitions to run",
+				},
+
+				cli.BoolFlag{
+					Name:  "no-watch",
+					Usage: "do not update graph on stream switch events",
+				},
+				cli.BoolFlag{
+					Name:  "noop",
+					Usage: "globally force all resources into no-op mode",
+				},
+				cli.StringFlag{
+					Name:  "graphviz, g",
+					Value: "",
+					Usage: "output file for graphviz data",
+				},
+				cli.StringFlag{
+					Name:  "graphviz-filter, gf",
+					Value: "dot", // directed graph default
+					Usage: "graphviz filter to use",
+				},
+				cli.IntFlag{
+					Name:   "converged-timeout, t",
+					Value:  -1,
+					Usage:  "exit after approximately this many seconds in a converged state",
+					EnvVar: "MGMT_CONVERGED_TIMEOUT",
+				},
+				cli.IntFlag{
+					Name:   "max-runtime",
+					Value:  0,
+					Usage:  "exit after a maximum of approximately this many seconds",
+					EnvVar: "MGMT_MAX_RUNTIME",
+				},
+
+				// if empty, it will startup a new server
+				cli.StringSliceFlag{
+					Name:   "seeds, s",
+					Value:  &cli.StringSlice{}, // empty slice
+					Usage:  "default etc client endpoint",
+					EnvVar: "MGMT_SEEDS",
+				},
+				// port 2379 and 4001 are common
+				cli.StringSliceFlag{
+					Name:   "client-urls",
+					Value:  &cli.StringSlice{},
+					Usage:  "list of URLs to listen on for client traffic",
+					EnvVar: "MGMT_CLIENT_URLS",
+				},
+				// port 2380 and 7001 are common
+				cli.StringSliceFlag{
+					Name:   "server-urls, peer-urls",
+					Value:  &cli.StringSlice{},
+					Usage:  "list of URLs to listen on for server (peer) traffic",
+					EnvVar: "MGMT_SERVER_URLS",
+				},
+				cli.IntFlag{
+					Name:   "ideal-cluster-size",
+					Value:  -1,
+					Usage:  "ideal number of server peers in cluster; only read by initial server",
+					EnvVar: "MGMT_IDEAL_CLUSTER_SIZE",
+				},
+				cli.BoolFlag{
+					Name:  "no-server",
+					Usage: "do not let other servers peer with me",
+				},
+
+				cli.IntFlag{
+					Name:   "cconns",
+					Value:  0,
+					Usage:  "number of maximum concurrent remote ssh connections to run; 0 for unlimited",
+					EnvVar: "MGMT_CCONNS",
+				},
+				cli.BoolFlag{
+					Name:  "allow-interactive",
+					Usage: "allow interactive prompting, such as for remote passwords",
+				},
+				cli.StringFlag{
+					Name:   "ssh-priv-id-rsa",
+					Value:  "~/.ssh/id_rsa",
+					Usage:  "default path to ssh key file, set empty to never touch",
+					EnvVar: "MGMT_SSH_PRIV_ID_RSA",
+				},
+				cli.BoolFlag{
+					Name:  "no-caching",
+					Usage: "don't allow remote caching of remote execution binary",
+				},
+				cli.IntFlag{
+					Name:   "depth",
+					Hidden: true, // internal use only
+					Value:  0,
+					Usage:  "specify depth in remote hierarchy",
+				},
+				cli.BoolFlag{
+					Name:  "no-pgp",
+					Usage: "don't create pgp keys",
+				},
+				cli.StringFlag{
+					Name:  "pgp-key-path",
+					Value: "",
+					Usage: "path for instance key pair",
+				},
+				cli.StringFlag{
+					Name:  "pgp-identity",
+					Value: "",
+					Usage: "default identity used for generation",
+				},
+			},
+		},
+	}
+	app.EnableBashCompletion = true
+	return app.Run(os.Args)
+}

lib/main.go

@@ -0,0 +1,561 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package lib
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+	"time"
+
+	"github.com/purpleidea/mgmt/converger"
+	"github.com/purpleidea/mgmt/etcd"
+	"github.com/purpleidea/mgmt/gapi"
+	"github.com/purpleidea/mgmt/pgp"
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/recwatch"
+	"github.com/purpleidea/mgmt/remote"
+	"github.com/purpleidea/mgmt/resources"
+	"github.com/purpleidea/mgmt/util"
+
+	etcdtypes "github.com/coreos/etcd/pkg/types"
+	"github.com/coreos/pkg/capnslog"
+	multierr "github.com/hashicorp/go-multierror"
+	errwrap "github.com/pkg/errors"
+)
+
+// Flags are some constant flags which are used throughout the program.
+type Flags struct {
+	Debug   bool // add additional log messages
+	Trace   bool // add execution flow log messages
+	Verbose bool // add extra log message output
+}
+
+// Main is the main struct for running the mgmt logic.
+type Main struct {
+	Program string // the name of this program, usually set at compile time
+	Version string // the version of this program, usually set at compile time
+
+	Flags Flags // static global flags that are set at compile time
+
+	Hostname *string // hostname to use; nil if undefined
+
+	Prefix         *string // prefix passed in; nil if undefined
+	TmpPrefix      bool    // request a pseudo-random, temporary prefix to be used
+	AllowTmpPrefix bool    // allow creation of a new temporary prefix if main prefix is unavailable
+
+	GAPI    gapi.GAPI // graph API interface struct
+	Remotes []string  // list of remote graph definitions to run
+
+	NoWatch          bool   // do not update graph on watched graph definition file changes
+	Noop             bool   // globally force all resources into no-op mode
+	Graphviz         string // output file for graphviz data
+	GraphvizFilter   string // graphviz filter to use
+	ConvergedTimeout int    // exit after approximately this many seconds in a converged state; -1 to disable
+	MaxRuntime       uint   // exit after a maximum of approximately this many seconds
+
+	Seeds            []string // default etc client endpoint
+	ClientURLs       []string // list of URLs to listen on for client traffic
+	ServerURLs       []string // list of URLs to listen on for server (peer) traffic
+	IdealClusterSize int      // ideal number of server peers in cluster; only read by initial server
+	NoServer         bool     // do not let other servers peer with me
+
+	CConns           uint16 // number of maximum concurrent remote ssh connections to run, 0 for unlimited
+	AllowInteractive bool   // allow interactive prompting, such as for remote passwords
+	SSHPrivIDRsa     string // default path to ssh key file, set empty to never touch
+	NoCaching        bool   // don't allow remote caching of remote execution binary
+	Depth            uint16 // depth in remote hierarchy; for internal use only
+
+	seeds            etcdtypes.URLs // processed seeds value
+	clientURLs       etcdtypes.URLs // processed client urls value
+	serverURLs       etcdtypes.URLs // processed server urls value
+	idealClusterSize uint16         // processed ideal cluster size value
+
+	NoPgp       bool    // disallow pgp functionality
+	PgpKeyPath  *string // import a pre-made key pair
+	PgpIdentity *string
+	pgpKeys     *pgp.PGP // agent key pair
+
+	exit chan error // exit signal
+}
+
+// Init initializes the main struct after it performs some validation.
+func (obj *Main) Init() error {
+
+	if obj.Program == "" || obj.Version == "" {
+		return fmt.Errorf("You must set the Program and Version strings!")
+	}
+
+	if obj.Prefix != nil && obj.TmpPrefix {
+		return fmt.Errorf("Choosing a prefix and the request for a tmp prefix is illogical!")
+	}
+
+	obj.idealClusterSize = uint16(obj.IdealClusterSize)
+	if obj.IdealClusterSize < 0 { // value is undefined, set to the default
+		obj.idealClusterSize = etcd.DefaultIdealClusterSize
+	}
+
+	if obj.idealClusterSize < 1 {
+		return fmt.Errorf("IdealClusterSize should be at least one!")
+	}
+
+	if obj.NoServer && len(obj.Remotes) > 0 {
+		// TODO: in this case, we won't be able to tunnel stuff back to
+		// here, so if we're okay with every remote graph running in an
+		// isolated mode, then this is okay. Improve on this if there's
+		// someone who really wants to be able to do this.
+		return fmt.Errorf("The Server is required when using Remotes!")
+	}
+
+	if obj.CConns < 0 {
+		return fmt.Errorf("The CConns value should be at least zero!")
+	}
+
+	if obj.ConvergedTimeout >= 0 && obj.CConns > 0 && len(obj.Remotes) > int(obj.CConns) {
+		return fmt.Errorf("You can't converge if you have more remotes than available connections!")
+	}
+
+	if obj.Depth < 0 { // user should not be using this argument manually
+		return fmt.Errorf("Negative values for Depth are not permitted!")
+	}
+
+	// transform the url list inputs into etcd typed lists
+	var err error
+	obj.seeds, err = etcdtypes.NewURLs(
+		util.FlattenListWithSplit(obj.Seeds, []string{",", ";", " "}),
+	)
+	if err != nil && len(obj.Seeds) > 0 {
+		return fmt.Errorf("Seeds didn't parse correctly!")
+	}
+	obj.clientURLs, err = etcdtypes.NewURLs(
+		util.FlattenListWithSplit(obj.ClientURLs, []string{",", ";", " "}),
+	)
+	if err != nil && len(obj.ClientURLs) > 0 {
+		return fmt.Errorf("ClientURLs didn't parse correctly!")
+	}
+	obj.serverURLs, err = etcdtypes.NewURLs(
+		util.FlattenListWithSplit(obj.ServerURLs, []string{",", ";", " "}),
+	)
+	if err != nil && len(obj.ServerURLs) > 0 {
+		return fmt.Errorf("ServerURLs didn't parse correctly!")
+	}
+
+	obj.exit = make(chan error)
+	return nil
+}
+
+// Exit causes a safe shutdown. This is often attached to the ^C signal handler.
+func (obj *Main) Exit(err error) {
+	obj.exit <- err // trigger an exit!
+}
+
+// Run is the main execution entrypoint to run mgmt.
+func (obj *Main) Run() error {
+
+	var start = time.Now().UnixNano()
+
+	var flags int
+	if obj.Flags.Debug || true { // TODO: remove || true
+		flags = log.LstdFlags | log.Lshortfile
+	}
+	flags = (flags - log.Ldate) // remove the date for now
+	log.SetFlags(flags)
+
+	// un-hijack from capnslog...
+	log.SetOutput(os.Stderr)
+	if obj.Flags.Verbose {
+		capnslog.SetFormatter(capnslog.NewLogFormatter(os.Stderr, "(etcd) ", flags))
+	} else {
+		capnslog.SetFormatter(capnslog.NewNilFormatter())
+	}
+
+	log.Printf("This is: %s, version: %s", obj.Program, obj.Version)
+	log.Printf("Main: Start: %v", start)
+
+	hostname, err := os.Hostname() // a sensible default
+	// allow passing in the hostname, instead of using the system setting
+	if h := obj.Hostname; h != nil && *h != "" { // override by cli
+		hostname = *h
+	} else if err != nil {
+		return errwrap.Wrapf(err, "Can't get default hostname!")
+	}
+	if hostname == "" { // safety check
+		return fmt.Errorf("Hostname cannot be empty!")
+	}
+
+	var prefix = fmt.Sprintf("/var/lib/%s/", obj.Program) // default prefix
+	if p := obj.Prefix; p != nil {
+		prefix = *p
+	}
+	// make sure the working directory prefix exists
+	if obj.TmpPrefix || os.MkdirAll(prefix, 0770) != nil {
+		if obj.TmpPrefix || obj.AllowTmpPrefix {
+			var err error
+			if prefix, err = ioutil.TempDir("", obj.Program+"-"+hostname+"-"); err != nil {
+				return fmt.Errorf("Main: Error: Can't create temporary prefix!")
+			}
+			log.Println("Main: Warning: Working prefix directory is temporary!")
+
+		} else {
+			return fmt.Errorf("Main: Error: Can't create prefix!")
+		}
+	}
+	log.Printf("Main: Working prefix is: %s", prefix)
+	pgraphPrefix := fmt.Sprintf("%s/", path.Join(prefix, "pgraph")) // pgraph namespace
+	if err := os.MkdirAll(pgraphPrefix, 0770); err != nil {
+		return errwrap.Wrapf(err, "Can't create pgraph prefix")
+	}
+
+	if !obj.NoPgp {
+		pgpPrefix := fmt.Sprintf("%s/", path.Join(prefix, "pgp"))
+		if err := os.MkdirAll(pgpPrefix, 0770); err != nil {
+			return errwrap.Wrapf(err, "Can't create pgp prefix")
+		}
+
+		pgpKeyringPath := path.Join(pgpPrefix, pgp.DefaultKeyringFile) // default path
+
+		if p := obj.PgpKeyPath; p != nil {
+			pgpKeyringPath = *p
+		}
+
+		var err error
+		if obj.pgpKeys, err = pgp.Import(pgpKeyringPath); err != nil && !os.IsNotExist(err) {
+			return errwrap.Wrapf(err, "Can't import pgp key")
+		}
+
+		if obj.pgpKeys == nil {
+
+			identity := fmt.Sprintf("%s <%s> %s", obj.Program, "root@"+hostname, "generated by "+obj.Program)
+			if p := obj.PgpIdentity; p != nil {
+				identity = *p
+			}
+
+			name, comment, email, err := pgp.ParseIdentity(identity)
+			if err != nil {
+				return errwrap.Wrapf(err, "Can't parse user string")
+
+			}
+
+			// TODO: Make hash configurable
+			if obj.pgpKeys, err = pgp.Generate(name, comment, email, nil); err != nil {
+				return errwrap.Wrapf(err, "Can't creating pgp key")
+			}
+
+			if err := obj.pgpKeys.SaveKey(pgpKeyringPath); err != nil {
+				return errwrap.Wrapf(err, "Can't save pgp key")
+			}
+		}
+
+		// TODO: Import admin key
+	}
+
+	var G, oldGraph *pgraph.Graph
+
+	// exit after `max-runtime` seconds for no reason at all...
+	if i := obj.MaxRuntime; i > 0 {
+		go func() {
+			time.Sleep(time.Duration(i) * time.Second)
+			obj.Exit(nil)
+		}()
+	}
+
+	// setup converger
+	converger := converger.NewConverger(
+		obj.ConvergedTimeout,
+		nil, // stateFn gets added in by EmbdEtcd
+	)
+	go converger.Loop(true) // main loop for converger, true to start paused
+
+	// embedded etcd
+	if len(obj.seeds) == 0 {
+		log.Printf("Main: Seeds: No seeds specified!")
+	} else {
+		log.Printf("Main: Seeds(%d): %v", len(obj.seeds), obj.seeds)
+	}
+	EmbdEtcd := etcd.NewEmbdEtcd(
+		hostname,
+		obj.seeds,
+		obj.clientURLs,
+		obj.serverURLs,
+		obj.NoServer,
+		obj.idealClusterSize,
+		etcd.Flags{
+			Debug:   obj.Flags.Debug,
+			Trace:   obj.Flags.Trace,
+			Verbose: obj.Flags.Verbose,
+		},
+		prefix,
+		converger,
+	)
+	if EmbdEtcd == nil {
+		// TODO: verify EmbdEtcd is not nil below...
+		obj.Exit(fmt.Errorf("Main: Etcd: Creation failed!"))
+	} else if err := EmbdEtcd.Startup(); err != nil { // startup (returns when etcd main loop is running)
+		obj.Exit(fmt.Errorf("Main: Etcd: Startup failed: %v", err))
+	}
+	convergerStateFn := func(b bool) error {
+		// exit if we are using the converged timeout and we are the
+		// root node. otherwise, if we are a child node in a remote
+		// execution hierarchy, we should only notify our converged
+		// state and wait for the parent to trigger the exit.
+		if t := obj.ConvergedTimeout; obj.Depth == 0 && t >= 0 {
+			if b {
+				log.Printf("Converged for %d seconds, exiting!", t)
+				obj.Exit(nil) // trigger an exit!
+			}
+			return nil
+		}
+		// send our individual state into etcd for others to see
+		return etcd.EtcdSetHostnameConverged(EmbdEtcd, hostname, b) // TODO: what should happen on error?
+	}
+	if EmbdEtcd != nil {
+		converger.SetStateFn(convergerStateFn)
+	}
+
+	var gapiChan chan error // stream events are nil errors
+	if obj.GAPI != nil {
+		data := gapi.Data{
+			Hostname: hostname,
+			// NOTE: alternate implementations can be substituted in
+			World: &etcd.World{
+				Hostname: hostname,
+				EmbdEtcd: EmbdEtcd,
+			},
+			Noop:    obj.Noop,
+			NoWatch: obj.NoWatch,
+		}
+		if err := obj.GAPI.Init(data); err != nil {
+			obj.Exit(fmt.Errorf("Main: GAPI: Init failed: %v", err))
+		} else if !obj.NoWatch {
+			gapiChan = obj.GAPI.Next() // stream of graph switch events!
+		}
+	}
+
+	exitchan := make(chan struct{}) // exit on close
+	go func() {
+		startChan := make(chan struct{}) // start signal
+		go func() { startChan <- struct{}{} }()
+
+		log.Println("Etcd: Starting...")
+		etcdChan := etcd.EtcdWatch(EmbdEtcd)
+		first := true // first loop or not
+		for {
+			log.Println("Main: Waiting...")
+			select {
+			case <-startChan: // kick the loop once at start
+				// pass
+
+			case b := <-etcdChan:
+				if !b { // ignore the message
+					continue
+				}
+				// everything else passes through to cause a compile!
+
+			case err, ok := <-gapiChan:
+				if !ok { // channel closed
+					if obj.Flags.Debug {
+						log.Printf("Main: GAPI exited")
+					}
+					gapiChan = nil // disable it
+					continue
+				}
+				if err != nil {
+					obj.Exit(err) // trigger exit
+					continue
+					//return // TODO: return or wait for exitchan?
+				}
+				if obj.NoWatch { // extra safety for bad GAPI's
+					log.Printf("Main: GAPI stream should be quiet with NoWatch!") // fix the GAPI!
+					continue                                                      // no stream events should be sent
+				}
+
+			case <-exitchan:
+				return
+			}
+
+			if obj.GAPI == nil {
+				log.Printf("Config: GAPI is empty!")
+				continue
+			}
+
+			// we need the vertices to be paused to work on them, so
+			// run graph vertex LOCK...
+			if !first { // TODO: we can flatten this check out I think
+				converger.Pause() // FIXME: add sync wait?
+				G.Pause()         // sync
+
+				//G.UnGroup() // FIXME: implement me if needed!
+			}
+
+			// make the graph from yaml, lib, puppet->yaml, or dsl!
+			newGraph, err := obj.GAPI.Graph() // generate graph!
+			if err != nil {
+				log.Printf("Config: Error creating new graph: %v", err)
+				// unpause!
+				if !first {
+					G.Start(first)    // sync
+					converger.Start() // after G.Start()
+				}
+				continue
+			}
+			newGraph.Flags = pgraph.Flags{Debug: obj.Flags.Debug}
+			// pass in the information we need
+			newGraph.AssociateData(&resources.Data{
+				Converger: converger,
+				Prefix:    pgraphPrefix,
+				Debug:     obj.Flags.Debug,
+			})
+
+			// apply the global noop parameter if requested
+			if obj.Noop {
+				for _, m := range newGraph.GraphMetas() {
+					m.Noop = obj.Noop
+				}
+			}
+
+			// FIXME: make sure we "UnGroup()" any semi-destructive
+			// changes to the resources so our efficient GraphSync
+			// will be able to re-use and cmp to the old graph.
+			newFullGraph, err := newGraph.GraphSync(oldGraph)
+			if err != nil {
+				log.Printf("Config: Error running graph sync: %v", err)
+				// unpause!
+				if !first {
+					G.Start(first)    // sync
+					converger.Start() // after G.Start()
+				}
+				continue
+			}
+			oldGraph = newFullGraph // save old graph
+			G = oldGraph.Copy()     // copy to active graph
+
+			G.AutoEdges() // add autoedges; modifies the graph
+			G.AutoGroup() // run autogroup; modifies the graph
+			// TODO: do we want to do a transitive reduction?
+			// FIXME: run a type checker that verifies all the send->recv relationships
+
+			log.Printf("Graph: %v", G) // show graph
+			if obj.GraphvizFilter != "" {
+				if err := G.ExecGraphviz(obj.GraphvizFilter, obj.Graphviz); err != nil {
+					log.Printf("Graphviz: %v", err)
+				} else {
+					log.Printf("Graphviz: Successfully generated graph!")
+				}
+			}
+			// G.Start(...) needs to be synchronous or wait,
+			// because if half of the nodes are started and
+			// some are not ready yet and the EtcdWatch
+			// loops, we'll cause G.Pause(...) before we
+			// even got going, thus causing nil pointer errors
+			G.Start(first)    // sync
+			converger.Start() // after G.Start()
+			first = false
+		}
+	}()
+
+	configWatcher := recwatch.NewConfigWatcher()
+	configWatcher.Flags = recwatch.Flags{Debug: obj.Flags.Debug}
+	events := configWatcher.Events()
+	if !obj.NoWatch {
+		configWatcher.Add(obj.Remotes...) // add all the files...
+	} else {
+		events = nil // signal that no-watch is true
+	}
+	go func() {
+		select {
+		case err := <-configWatcher.Error():
+			obj.Exit(err) // trigger an exit!
+
+		case <-exitchan:
+			return
+		}
+	}()
+
+	// initialize the add watcher, which calls the f callback on map changes
+	convergerCb := func(f func(map[string]bool) error) (func(), error) {
+		return etcd.EtcdAddHostnameConvergedWatcher(EmbdEtcd, f)
+	}
+
+	// build remotes struct for remote ssh
+	remotes := remote.NewRemotes(
+		EmbdEtcd.LocalhostClientURLs().StringSlice(),
+		[]string{etcd.DefaultClientURL},
+		obj.Noop,
+		obj.Remotes, // list of files
+		events,      // watch for file changes
+		obj.CConns,
+		obj.AllowInteractive,
+		obj.SSHPrivIDRsa,
+		!obj.NoCaching,
+		obj.Depth,
+		prefix,
+		converger,
+		convergerCb,
+		remote.Flags{
+			Program: obj.Program,
+			Debug:   obj.Flags.Debug,
+		},
+	)
+
+	// TODO: is there any benefit to running the remotes above in the loop?
+	// wait for etcd to be running before we remote in, which we do above!
+	go remotes.Run()
+
+	if obj.GAPI == nil {
+		converger.Start() // better start this for empty graphs
+	}
+	log.Println("Main: Running...")
+
+	reterr := <-obj.exit // wait for exit signal
+
+	log.Println("Destroy...")
+
+	if obj.GAPI != nil {
+		if err := obj.GAPI.Close(); err != nil {
+			err = errwrap.Wrapf(err, "GAPI closed poorly!")
+			reterr = multierr.Append(reterr, err) // list of errors
+		}
+	}
+
+	configWatcher.Close()                  // stop sending file changes to remotes
+	if err := remotes.Exit(); err != nil { // tell all the remote connections to shutdown; waits!
+		err = errwrap.Wrapf(err, "Remote exited poorly!")
+		reterr = multierr.Append(reterr, err) // list of errors
+	}
+
+	// tell inner main loop to exit
+	close(exitchan)
+
+	G.Exit() // tell all the children to exit, and waits for them to do so
+
+	// cleanup etcd main loop last so it can process everything first
+	if err := EmbdEtcd.Destroy(); err != nil { // shutdown and cleanup etcd
+		err = errwrap.Wrapf(err, "Etcd exited poorly!")
+		reterr = multierr.Append(reterr, err) // list of errors
+	}
+
+	if obj.Flags.Debug {
+		log.Printf("Main: Graph: %v", G)
+	}
+
+	// TODO: wait for each vertex to exit...
+	log.Println("Goodbye!")
+	return reterr
+}

main.go

@@ -18,13 +18,17 @@
 package main
 
 import (
-	"github.com/codegangsta/cli"
-	"log"
+	"fmt"
 	"os"
-	"os/signal"
-	"sync"
-	"syscall"
-	"time"
+
+	mgmt "github.com/purpleidea/mgmt/lib"
+)
+
+// These constants are some global variables that are used throughout the code.
+const (
+	DEBUG   = false // add additional log messages
+	TRACE   = false // add execution flow log messages
+	VERBOSE = false // add extra log message output
 )
 
 // set at compile time
@@ -33,262 +37,15 @@ var (
 	version string
 )
 
-const (
-	DEBUG = false
-)
-
-// signal handler
-func waitForSignal(exit chan bool) {
-	signals := make(chan os.Signal, 1)
-	signal.Notify(signals, os.Interrupt) // catch ^C
-	//signal.Notify(signals, os.Kill) // catch signals
-	signal.Notify(signals, syscall.SIGTERM)
-
-	select {
-	case e := <-signals: // any signal will do
-		if e == os.Interrupt {
-			log.Println("Interrupted by ^C")
-		} else {
-			log.Println("Interrupted by signal")
-		}
-	case <-exit: // or a manual signal
-		log.Println("Interrupted by exit signal")
-	}
-}
-
-func run(c *cli.Context) {
-	var start = time.Now().UnixNano()
-	var wg sync.WaitGroup
-	exit := make(chan bool)      // exit signal
-	converged := make(chan bool) // converged signal
-	log.Printf("This is: %v, version: %v", program, version)
-	log.Printf("Main: Start: %v", start)
-	var G, fullGraph *Graph
-
-	// exit after `max-runtime` seconds for no reason at all...
-	if i := c.Int("max-runtime"); i > 0 {
-		go func() {
-			time.Sleep(time.Duration(i) * time.Second)
-			exit <- true
-		}()
-	}
-
-	// initial etcd peer endpoint
-	seed := c.String("seed")
-	if seed == "" {
-		// XXX: start up etcd server, others will join me!
-		seed = "http://127.0.0.1:2379" // thus we use the local server!
-	}
-	// then, connect to `seed` as a client
-
-	// FIXME: validate seed, or wait for it to fail in etcd init?
-
-	// etcd
-	etcdO := &EtcdWObject{
-		seed:      seed,
-		ctimeout:  c.Int("converged-timeout"),
-		converged: converged,
-	}
-
-	hostname := c.String("hostname")
-	if hostname == "" {
-		hostname, _ = os.Hostname() // etcd watch key // XXX: this is not the correct key name this is the set key name... WOOPS
-	}
-	go func() {
-		startchan := make(chan struct{}) // start signal
-		go func() { startchan <- struct{}{} }()
-		file := c.String("file")
-		configchan := make(chan bool)
-		if !c.Bool("no-watch") {
-			configchan = ConfigWatch(file)
-		}
-		log.Println("Etcd: Starting...")
-		etcdchan := etcdO.EtcdWatch()
-		first := true // first loop or not
-		for {
-			log.Println("Main: Waiting...")
-			select {
-			case _ = <-startchan: // kick the loop once at start
-				// pass
-			case msg := <-etcdchan:
-				switch msg {
-				// some types of messages we ignore...
-				case etcdFoo, etcdBar:
-					continue
-				// while others passthrough and cause a compile!
-				case etcdStart, etcdEvent:
-					// pass
-				default:
-					log.Fatal("Etcd: Unhandled message: ", msg)
-				}
-			case msg := <-configchan:
-				if c.Bool("no-watch") || !msg {
-					continue // not ready to read config
-				}
-				//case compile_event: XXX
-			}
-
-			config := ParseConfigFromFile(file)
-			if config == nil {
-				log.Printf("Config parse failure")
-				continue
-			}
-
-			// run graph vertex LOCK...
-			if !first { // TODO: we can flatten this check out I think
-				G.Pause() // sync
-			}
-
-			// build graph from yaml file on events (eg: from etcd)
-			// we need the vertices to be paused to work on them
-			if newFullgraph, err := fullGraph.NewGraphFromConfig(config, etcdO, hostname); err == nil { // keep references to all original elements
-				fullGraph = newFullgraph
-			} else {
-				log.Printf("Config: Error making new graph from config: %v", err)
-				// unpause!
-				if !first {
-					G.Start(&wg, first) // sync
-				}
-				continue
-			}
-
-			G = fullGraph.Copy() // copy to active graph
-			// XXX: do etcd transaction out here...
-			G.AutoEdges() // add autoedges; modifies the graph
-			G.AutoGroup() // run autogroup; modifies the graph
-			// TODO: do we want to do a transitive reduction?
-
-			log.Printf("Graph: %v", G) // show graph
-			err := G.ExecGraphviz(c.String("graphviz-filter"), c.String("graphviz"))
-			if err != nil {
-				log.Printf("Graphviz: %v", err)
-			} else {
-				log.Printf("Graphviz: Successfully generated graph!")
-			}
-			G.SetVertex()
-			G.SetConvergedCallback(c.Int("converged-timeout"), converged)
-			// G.Start(...) needs to be synchronous or wait,
-			// because if half of the nodes are started and
-			// some are not ready yet and the EtcdWatch
-			// loops, we'll cause G.Pause(...) before we
-			// even got going, thus causing nil pointer errors
-			G.Start(&wg, first) // sync
-			first = false
-		}
-	}()
-
-	if i := c.Int("converged-timeout"); i >= 0 {
-		go func() {
-		ConvergedLoop:
-			for {
-				<-converged // when anyone says they have converged
-
-				if etcdO.GetConvergedState() != etcdConvergedTimeout {
-					continue
-				}
-				for v := range G.GetVerticesChan() {
-					if v.Res.GetConvergedState() != resConvergedTimeout {
-						continue ConvergedLoop
-					}
-				}
-
-				// if all have converged, exit
-				log.Printf("Converged for %d seconds, exiting!", i)
-				exit <- true
-				for {
-					<-converged
-				} // unblock/drain
-				//return
-			}
-		}()
-	}
-
-	log.Println("Main: Running...")
-
-	waitForSignal(exit) // pass in exit channel to watch
-
-	G.Exit() // tell all the children to exit
-
-	if DEBUG {
-		log.Printf("Graph: %v", G)
-	}
-
-	wg.Wait() // wait for primary go routines to exit
-
-	// TODO: wait for each vertex to exit...
-	log.Println("Goodbye!")
-}
-
 func main() {
-	//if DEBUG {
-	log.SetFlags(log.LstdFlags | log.Lshortfile)
-	//}
-	log.SetFlags(log.Flags() - log.Ldate) // remove the date for now
-	if program == "" || version == "" {
-		log.Fatal("Program was not compiled correctly. Please see Makefile.")
+	flags := mgmt.Flags{
+		Debug:   DEBUG,
+		Trace:   TRACE,
+		Verbose: VERBOSE,
 	}
-	app := cli.NewApp()
-	app.Name = program
-	app.Usage = "next generation config management"
-	app.Version = version
-	//app.Action = ... // without a default action, help runs
-
-	app.Commands = []cli.Command{
-		{
-			Name:    "run",
-			Aliases: []string{"r"},
-			Usage:   "run",
-			Action:  run,
-			Flags: []cli.Flag{
-				cli.StringFlag{
-					Name:  "file, f",
-					Value: "",
-					Usage: "graph definition to run",
-				},
-				cli.BoolFlag{
-					Name:  "no-watch",
-					Usage: "do not update graph on watched graph definition file changes",
-				},
-				cli.StringFlag{
-					Name:  "code, c",
-					Value: "",
-					Usage: "code definition to run",
-				},
-				cli.StringFlag{
-					Name:  "graphviz, g",
-					Value: "",
-					Usage: "output file for graphviz data",
-				},
-				cli.StringFlag{
-					Name:  "graphviz-filter, gf",
-					Value: "dot", // directed graph default
-					Usage: "graphviz filter to use",
-				},
-				// useful for testing multiple instances on same machine
-				cli.StringFlag{
-					Name:  "hostname",
-					Value: "",
-					Usage: "hostname to use",
-				},
-				// if empty, it will startup a new server
-				cli.StringFlag{
-					Name:  "seed, s",
-					Value: "",
-					Usage: "default etc peer endpoint",
-				},
-				cli.IntFlag{
-					Name:  "converged-timeout, t",
-					Value: -1,
-					Usage: "exit after approximately this many seconds in a converged state",
-				},
-				cli.IntFlag{
-					Name:  "max-runtime",
-					Value: 0,
-					Usage: "exit after a maximum of approximately this many seconds",
-				},
-			},
-		},
+	if err := mgmt.CLI(program, version, flags); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+		return
 	}
-	app.EnableBashCompletion = true
-	app.Run(os.Args)
 }

misc/go

@@ -0,0 +1,15 @@
+#!/bin/bash
+# hack around stupid $GOPATH semantics, with ~/bin/go helper
+# thanks to Nilium in #go-nuts for 1/3 of the idea
+[ -z "$GOPATH" ] && echo '$GOPATH is not set!' && exit 1
+GO="$(which -a go | sed -e '2q;d')"	# TODO: pick /usr/bin/go in a better way
+if [ "$1" = "generate" ]; then
+	exec $GO "$@"	# go generate is stupid and gets confused by $GOPATH
+fi
+# the idea is to have $project/gopath/src/ be a symlink to ../vendor but you put
+# all of your vendored things in vendor/ but with this gopath can be per project
+if [ -d "$PWD/vendor/" ] && [ -d "$PWD/gopath/" ] && [ "`readlink $PWD/gopath/src`" = "../vendor" ] ; then
+	GOPATH="$PWD/gopath/:$GOPATH" $GO "$@"
+else
+	$GO "$@"
+fi

misc/make-deps.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # setup a simple go environment
 XPWD=`pwd`
 ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"	# dir!
@@ -9,43 +9,60 @@ if env | grep -q '^TRAVIS=true$'; then
 	travis=1
 fi
 
-if [ $travis -eq 0 ]; then
-	YUM=`which yum 2>/dev/null`
-	APT=`which apt-get 2>/dev/null`
-	if [ -z "$YUM" -a -z "$APT" ]; then
+sudo_command=$(which sudo)
+
+YUM=`which yum 2>/dev/null`
+DNF=`which dnf 2>/dev/null`
+APT=`which apt-get 2>/dev/null`
+
+# if DNF is available use it
+if [ -x "$DNF" ]; then
+	YUM=$DNF
+fi
+
+if [ -z "$YUM" -a -z "$APT" ]; then
 	echo "The package managers can't be found."
 	exit 1
-	fi
+fi
+
+if [ ! -z "$YUM" ]; then
+	$sudo_command $YUM install -y libvirt-devel
+
+fi
+if [ ! -z "$APT" ]; then
+	$sudo_command $APT install -y libvirt-dev || true
+	$sudo_command $APT install -y libpcap0.8-dev || true
+fi
+
+if [ $travis -eq 0 ]; then
 	if [ ! -z "$YUM" ]; then
 		# some go dependencies are stored in mercurial
-		sudo $YUM install -y golang golang-googlecode-tools-stringer hg
+		$sudo_command $YUM install -y golang golang-googlecode-tools-stringer hg
 
 	fi
 	if [ ! -z "$APT" ]; then
-		sudo $APT update
-		sudo $APT install -y golang make gcc packagekit mercurial
+		$sudo_command $APT update
+		$sudo_command $APT install -y golang make gcc packagekit mercurial
 		# one of these two golang tools packages should work on debian
-		sudo $APT install -y golang-golang-x-tools || true
-		sudo $APT install -y golang-go.tools || true
+		$sudo_command $APT install -y golang-golang-x-tools || true
+		$sudo_command $APT install -y golang-go.tools || true
 	fi
 fi
 
-# build etcd
-git clone --recursive https://github.com/coreos/etcd/ && cd etcd
-goversion=$(go version)
-# if 'go version' contains string 'devel', then use git master of etcd...
-if [ "${goversion#*devel}" == "$goversion" ]; then
-	git checkout v2.2.4	# TODO: update to newer versions as needed
+# if golang is too old, we don't want to fail with an obscure error later
+if go version | grep 'go1\.[0123]\.'; then
+	echo "mgmt requires go1.4 or higher."
+	exit 1
 fi
-[ -x build ] && ./build
-mkdir -p ~/bin/
-cp bin/etcd ~/bin/
-cd - >/dev/null
-rm -rf etcd	# clean up to avoid failing on upstream gofmt errors
 
-go get ./...	# get all the go dependencies
+go get -d ./...	# get all the go dependencies
 [ -e "$GOBIN/mgmt" ] && rm -f "$GOBIN/mgmt"	# the `go get` version has no -X
-go get golang.org/x/tools/cmd/vet	# add in `go vet` for travis
+# vet is built-in in go 1.6 - we check for go vet command
+go vet 1> /dev/null 2>&1
+ret=$?
+if [[ $ret != 0 ]]; then
+	go get golang.org/x/tools/cmd/vet      # add in `go vet` for travis
+fi
 go get golang.org/x/tools/cmd/stringer	# for automatic stringer-ing
 go get github.com/golang/lint/golint	# for `golint`-ing
 cd "$XPWD" >/dev/null

misc/mgmt.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Run mgmt configuration management
+Documentation=https://github.com/purpleidea/mgmt/
+After=systemd-networkd.service
+Requires=systemd-networkd.service
+
+[Service]
+ExecStart=/usr/bin/mgmt run ${OPTS}
+RestartSec=5s
+Restart=always
+
+[Install]
+WantedBy=multi-user.target

pgp/pgp.go

@@ -0,0 +1,230 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgp
+
+import (
+	"bufio"
+	"bytes"
+	"crypto"
+	"encoding/base64"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+
+	errwrap "github.com/pkg/errors"
+	"golang.org/x/crypto/openpgp"
+	"golang.org/x/crypto/openpgp/packet"
+)
+
+// DefaultKeyringFile is the default file name for keyrings.
+const DefaultKeyringFile = "keyring.pgp"
+
+// CONFIG set default Hash.
+var CONFIG packet.Config
+
+func init() {
+	CONFIG.DefaultHash = crypto.SHA256
+}
+
+// PGP contains base entity.
+type PGP struct {
+	Entity *openpgp.Entity
+}
+
+// Import private key from defined path.
+func Import(privKeyPath string) (*PGP, error) {
+
+	privKeyFile, err := os.Open(privKeyPath)
+	if err != nil {
+		return nil, err
+	}
+	defer privKeyFile.Close()
+
+	file := packet.NewReader(bufio.NewReader(privKeyFile))
+	entity, err := openpgp.ReadEntity(file)
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "can't read entity from path")
+	}
+
+	obj := &PGP{
+		Entity: entity,
+	}
+
+	log.Printf("PGP: Imported key: %s", obj.Entity.PrivateKey.KeyIdShortString())
+	return obj, nil
+}
+
+// Generate creates new key pair. This key pair must be saved or it will be lost.
+func Generate(name, comment, email string, hash *crypto.Hash) (*PGP, error) {
+	if hash != nil {
+		CONFIG.DefaultHash = *hash
+	}
+	// generate a new public/private key pair
+	entity, err := openpgp.NewEntity(name, comment, email, &CONFIG)
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "can't generate entity")
+	}
+
+	obj := &PGP{
+		Entity: entity,
+	}
+
+	log.Printf("PGP: Created key: %s", obj.Entity.PrivateKey.KeyIdShortString())
+	return obj, nil
+}
+
+// SaveKey writes the whole entity (including private key!) to a .gpg file.
+func (obj *PGP) SaveKey(path string) error {
+	f, err := os.Create(path)
+	if err != nil {
+		return errwrap.Wrapf(err, "can't create file from given path")
+	}
+
+	w := bufio.NewWriter(f)
+	if err != nil {
+		return errwrap.Wrapf(err, "can't create writer")
+	}
+
+	if err := obj.Entity.SerializePrivate(w, &CONFIG); err != nil {
+		return errwrap.Wrapf(err, "can't serialize private key")
+	}
+
+	for _, ident := range obj.Entity.Identities {
+		for _, sig := range ident.Signatures {
+			if err := sig.Serialize(w); err != nil {
+				return errwrap.Wrapf(err, "can't serialize signature")
+			}
+		}
+	}
+
+	if err := w.Flush(); err != nil {
+		return errwrap.Wrapf(err, "enable to flush writer")
+	}
+
+	return nil
+}
+
+// WriteFile from given buffer in specified path.
+func (obj *PGP) WriteFile(path string, buff *bytes.Buffer) error {
+	w, err := createWriter(path)
+	if err != nil {
+		return errwrap.Wrapf(err, "can't create writer")
+	}
+	buff.WriteTo(w)
+
+	if err := w.Flush(); err != nil {
+		return errwrap.Wrapf(err, "can't flush buffered data")
+	}
+	return nil
+}
+
+// CreateWriter remove duplicate function.
+func createWriter(path string) (*bufio.Writer, error) {
+	f, err := os.Create(path)
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "can't create file from given path")
+	}
+	return bufio.NewWriter(f), nil
+}
+
+// Encrypt message for specified entity.
+func (obj *PGP) Encrypt(to *openpgp.Entity, msg string) (string, error) {
+	buf, err := obj.EncryptMsg(to, msg)
+	if err != nil {
+		return "", errwrap.Wrapf(err, "can't encrypt message")
+	}
+
+	// encode to base64
+	bytes, err := ioutil.ReadAll(buf)
+	if err != nil {
+		return "", errwrap.Wrapf(err, "can't read unverified body")
+	}
+	return base64.StdEncoding.EncodeToString(bytes), nil
+}
+
+// EncryptMsg encrypts the message.
+func (obj *PGP) EncryptMsg(to *openpgp.Entity, msg string) (*bytes.Buffer, error) {
+	ents := []*openpgp.Entity{to}
+
+	buf := new(bytes.Buffer)
+	w, err := openpgp.Encrypt(buf, ents, obj.Entity, nil, nil)
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "can't encrypt message")
+	}
+
+	_, err = w.Write([]byte(msg))
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "can't write to buffer")
+	}
+
+	if err = w.Close(); err != nil {
+		return nil, errwrap.Wrapf(err, "can't close writer")
+	}
+	return buf, nil
+}
+
+// Decrypt an encrypted msg.
+func (obj *PGP) Decrypt(encString string) (string, error) {
+	entityList := openpgp.EntityList{obj.Entity}
+
+	// decode the base64 string
+	dec, err := base64.StdEncoding.DecodeString(encString)
+	if err != nil {
+		return "", errwrap.Wrapf(err, "fail at decoding encrypted string")
+	}
+
+	// decrypt it with the contents of the private key
+	md, err := openpgp.ReadMessage(bytes.NewBuffer(dec), entityList, nil, nil)
+	if err != nil {
+		return "", errwrap.Wrapf(err, "can't read message")
+	}
+
+	bytes, err := ioutil.ReadAll(md.UnverifiedBody)
+	if err != nil {
+		return "", errwrap.Wrapf(err, "can't read unverified body")
+	}
+	return string(bytes), nil
+}
+
+// GetIdentities return the first identities from current object.
+func (obj *PGP) GetIdentities() (string, error) {
+	identities := []*openpgp.Identity{}
+
+	for _, v := range obj.Entity.Identities {
+		identities = append(identities, v)
+	}
+	return identities[0].Name, nil
+}
+
+// ParseIdentity parses an identity into name, comment and email components.
+func ParseIdentity(identity string) (name, comment, email string, err error) {
+	// get name
+	n := strings.Split(identity, " <")
+	if len(n) != 2 {
+		return "", "", "", errwrap.Wrap(err, "user string mal formated")
+	}
+
+	// get email and comment
+	ec := strings.Split(n[1], "> ")
+	if len(ec) != 2 {
+		return "", "", "", errwrap.Wrap(err, "user string mal formated")
+	}
+
+	return n[0], ec[1], ec[0], nil
+}

pgraph.go

@@ -1,883 +0,0 @@
-// Mgmt
-// Copyright (C) 2013-2016+ James Shubin and the project contributors
-// Written by James Shubin <james@shubin.ca> and the project contributors
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-// Pgraph (Pointer Graph)
-package main
-
-import (
-	"errors"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"os"
-	"os/exec"
-	"sort"
-	"strconv"
-	"sync"
-	"syscall"
-	"time"
-)
-
-//go:generate stringer -type=graphState -output=graphstate_stringer.go
-type graphState int
-
-const (
-	graphStateNil graphState = iota
-	graphStateStarting
-	graphStateStarted
-	graphStatePausing
-	graphStatePaused
-)
-
-// The graph abstract data type (ADT) is defined as follows:
-// * the directed graph arrows point from left to right ( -> )
-// * the arrows point away from their dependencies (eg: arrows mean "before")
-// * IOW, you might see package -> file -> service (where package runs first)
-// * This is also the direction that the notify should happen in...
-type Graph struct {
-	Name      string
-	Adjacency map[*Vertex]map[*Vertex]*Edge // *Vertex -> *Vertex (edge)
-	state     graphState
-	mutex     sync.Mutex // used when modifying graph State variable
-}
-
-type Vertex struct {
-	Res             // anonymous field
-	timestamp int64 // last updated timestamp ?
-}
-
-type Edge struct {
-	Name string
-}
-
-func NewGraph(name string) *Graph {
-	return &Graph{
-		Name:      name,
-		Adjacency: make(map[*Vertex]map[*Vertex]*Edge),
-		state:     graphStateNil,
-	}
-}
-
-func NewVertex(r Res) *Vertex {
-	return &Vertex{
-		Res: r,
-	}
-}
-
-func NewEdge(name string) *Edge {
-	return &Edge{
-		Name: name,
-	}
-}
-
-// Copy makes a copy of the graph struct
-func (g *Graph) Copy() *Graph {
-	newGraph := &Graph{
-		Name:      g.Name,
-		Adjacency: make(map[*Vertex]map[*Vertex]*Edge, len(g.Adjacency)),
-		state:     g.state,
-	}
-	for k, v := range g.Adjacency {
-		newGraph.Adjacency[k] = v // copy
-	}
-	return newGraph
-}
-
-// returns the name of the graph
-func (g *Graph) GetName() string {
-	return g.Name
-}
-
-// set name of the graph
-func (g *Graph) SetName(name string) {
-	g.Name = name
-}
-
-func (g *Graph) GetState() graphState {
-	//g.mutex.Lock()
-	//defer g.mutex.Unlock()
-	return g.state
-}
-
-// set graph state and return previous state
-func (g *Graph) SetState(state graphState) graphState {
-	g.mutex.Lock()
-	defer g.mutex.Unlock()
-	prev := g.GetState()
-	g.state = state
-	return prev
-}
-
-// store a pointer in the resource to it's parent vertex
-func (g *Graph) SetVertex() {
-	for v := range g.GetVerticesChan() {
-		v.Res.SetVertex(v)
-	}
-}
-
-// AddVertex uses variadic input to add all listed vertices to the graph
-func (g *Graph) AddVertex(xv ...*Vertex) {
-	for _, v := range xv {
-		if _, exists := g.Adjacency[v]; !exists {
-			g.Adjacency[v] = make(map[*Vertex]*Edge)
-		}
-	}
-}
-
-func (g *Graph) DeleteVertex(v *Vertex) {
-	delete(g.Adjacency, v)
-	for k := range g.Adjacency {
-		delete(g.Adjacency[k], v)
-	}
-}
-
-// adds a directed edge to the graph from v1 to v2
-func (g *Graph) AddEdge(v1, v2 *Vertex, e *Edge) {
-	// NOTE: this doesn't allow more than one edge between two vertexes...
-	g.AddVertex(v1, v2) // supports adding N vertices now
-	// TODO: check if an edge exists to avoid overwriting it!
-	// NOTE: VertexMerge() depends on overwriting it at the moment...
-	g.Adjacency[v1][v2] = e
-}
-
-func (g *Graph) GetVertexMatch(obj Res) *Vertex {
-	for k := range g.Adjacency {
-		if k.Res.Compare(obj) {
-			return k
-		}
-	}
-	return nil
-}
-
-func (g *Graph) HasVertex(v *Vertex) bool {
-	if _, exists := g.Adjacency[v]; exists {
-		return true
-	}
-	return false
-}
-
-// number of vertices in the graph
-func (g *Graph) NumVertices() int {
-	return len(g.Adjacency)
-}
-
-// number of edges in the graph
-func (g *Graph) NumEdges() int {
-	count := 0
-	for k := range g.Adjacency {
-		count += len(g.Adjacency[k])
-	}
-	return count
-}
-
-// GetVertices returns a randomly sorted slice of all vertices in the graph
-// The order is random, because the map implementation is intentionally so!
-func (g *Graph) GetVertices() []*Vertex {
-	var vertices []*Vertex
-	for k := range g.Adjacency {
-		vertices = append(vertices, k)
-	}
-	return vertices
-}
-
-// returns a channel of all vertices in the graph
-func (g *Graph) GetVerticesChan() chan *Vertex {
-	ch := make(chan *Vertex)
-	go func(ch chan *Vertex) {
-		for k := range g.Adjacency {
-			ch <- k
-		}
-		close(ch)
-	}(ch)
-	return ch
-}
-
-type VertexSlice []*Vertex
-
-func (vs VertexSlice) Len() int           { return len(vs) }
-func (vs VertexSlice) Swap(i, j int)      { vs[i], vs[j] = vs[j], vs[i] }
-func (vs VertexSlice) Less(i, j int) bool { return vs[i].String() < vs[j].String() }
-
-// GetVerticesSorted returns a sorted slice of all vertices in the graph
-// The order is sorted by String() to avoid the non-determinism in the map type
-func (g *Graph) GetVerticesSorted() []*Vertex {
-	var vertices []*Vertex
-	for k := range g.Adjacency {
-		vertices = append(vertices, k)
-	}
-	sort.Sort(VertexSlice(vertices)) // add determinism
-	return vertices
-}
-
-// make the graph pretty print
-func (g *Graph) String() string {
-	return fmt.Sprintf("Vertices(%d), Edges(%d)", g.NumVertices(), g.NumEdges())
-}
-
-// String returns the canonical form for a vertex
-func (v *Vertex) String() string {
-	return fmt.Sprintf("%s[%s]", v.Res.Kind(), v.Res.GetName())
-}
-
-// output the graph in graphviz format
-// https://en.wikipedia.org/wiki/DOT_%28graph_description_language%29
-func (g *Graph) Graphviz() (out string) {
-	//digraph g {
-	//	label="hello world";
-	//	node [shape=box];
-	//	A [label="A"];
-	//	B [label="B"];
-	//	C [label="C"];
-	//	D [label="D"];
-	//	E [label="E"];
-	//	A -> B [label=f];
-	//	B -> C [label=g];
-	//	D -> E [label=h];
-	//}
-	out += fmt.Sprintf("digraph %v {\n", g.GetName())
-	out += fmt.Sprintf("\tlabel=\"%v\";\n", g.GetName())
-	//out += "\tnode [shape=box];\n"
-	str := ""
-	for i := range g.Adjacency { // reverse paths
-		out += fmt.Sprintf("\t%v [label=\"%v[%v]\"];\n", i.GetName(), i.Kind(), i.GetName())
-		for j := range g.Adjacency[i] {
-			k := g.Adjacency[i][j]
-			// use str for clearer output ordering
-			str += fmt.Sprintf("\t%v -> %v [label=%v];\n", i.GetName(), j.GetName(), k.Name)
-		}
-	}
-	out += str
-	out += "}\n"
-	return
-}
-
-// write out the graphviz data and run the correct graphviz filter command
-func (g *Graph) ExecGraphviz(program, filename string) error {
-
-	switch program {
-	case "dot", "neato", "twopi", "circo", "fdp":
-	default:
-		return errors.New("Invalid graphviz program selected!")
-	}
-
-	if filename == "" {
-		return errors.New("No filename given!")
-	}
-
-	// run as a normal user if possible when run with sudo
-	uid, err1 := strconv.Atoi(os.Getenv("SUDO_UID"))
-	gid, err2 := strconv.Atoi(os.Getenv("SUDO_GID"))
-
-	err := ioutil.WriteFile(filename, []byte(g.Graphviz()), 0644)
-	if err != nil {
-		return errors.New("Error writing to filename!")
-	}
-
-	if err1 == nil && err2 == nil {
-		if err := os.Chown(filename, uid, gid); err != nil {
-			return errors.New("Error changing file owner!")
-		}
-	}
-
-	path, err := exec.LookPath(program)
-	if err != nil {
-		return errors.New("Graphviz is missing!")
-	}
-
-	out := fmt.Sprintf("%v.png", filename)
-	cmd := exec.Command(path, "-Tpng", fmt.Sprintf("-o%v", out), filename)
-
-	if err1 == nil && err2 == nil {
-		cmd.SysProcAttr = &syscall.SysProcAttr{}
-		cmd.SysProcAttr.Credential = &syscall.Credential{
-			Uid: uint32(uid),
-			Gid: uint32(gid),
-		}
-	}
-	_, err = cmd.Output()
-	if err != nil {
-		return errors.New("Error writing to image!")
-	}
-	return nil
-}
-
-// return an array (slice) of all directed vertices to vertex v (??? -> v)
-// OKTimestamp should use this
-func (g *Graph) IncomingGraphEdges(v *Vertex) []*Vertex {
-	// TODO: we might be able to implement this differently by reversing
-	// the Adjacency graph and then looping through it again...
-	var s []*Vertex
-	for k := range g.Adjacency { // reverse paths
-		for w := range g.Adjacency[k] {
-			if w == v {
-				s = append(s, k)
-			}
-		}
-	}
-	return s
-}
-
-// return an array (slice) of all vertices that vertex v points to (v -> ???)
-// poke should use this
-func (g *Graph) OutgoingGraphEdges(v *Vertex) []*Vertex {
-	var s []*Vertex
-	for k := range g.Adjacency[v] { // forward paths
-		s = append(s, k)
-	}
-	return s
-}
-
-// return an array (slice) of all vertices that connect to vertex v
-func (g *Graph) GraphEdges(v *Vertex) []*Vertex {
-	var s []*Vertex
-	s = append(s, g.IncomingGraphEdges(v)...)
-	s = append(s, g.OutgoingGraphEdges(v)...)
-	return s
-}
-
-func (g *Graph) DFS(start *Vertex) []*Vertex {
-	var d []*Vertex // discovered
-	var s []*Vertex // stack
-	if _, exists := g.Adjacency[start]; !exists {
-		return nil // TODO: error
-	}
-	v := start
-	s = append(s, v)
-	for len(s) > 0 {
-		v, s = s[len(s)-1], s[:len(s)-1] // s.pop()
-
-		if !VertexContains(v, d) { // if not discovered
-			d = append(d, v) // label as discovered
-
-			for _, w := range g.GraphEdges(v) {
-				s = append(s, w)
-			}
-		}
-	}
-	return d
-}
-
-// build a new graph containing only vertices from the list...
-func (g *Graph) FilterGraph(name string, vertices []*Vertex) *Graph {
-	newgraph := NewGraph(name)
-	for k1, x := range g.Adjacency {
-		for k2, e := range x {
-			//log.Printf("Filter: %v -> %v # %v", k1.Name, k2.Name, e.Name)
-			if VertexContains(k1, vertices) || VertexContains(k2, vertices) {
-				newgraph.AddEdge(k1, k2, e)
-			}
-		}
-	}
-	return newgraph
-}
-
-// return a channel containing the N disconnected graphs in our main graph
-// we can then process each of these in parallel
-func (g *Graph) GetDisconnectedGraphs() chan *Graph {
-	ch := make(chan *Graph)
-	go func() {
-		var start *Vertex
-		var d []*Vertex // discovered
-		c := g.NumVertices()
-		for len(d) < c {
-
-			// get an undiscovered vertex to start from
-			for _, s := range g.GetVertices() {
-				if !VertexContains(s, d) {
-					start = s
-				}
-			}
-
-			// dfs through the graph
-			dfs := g.DFS(start)
-			// filter all the collected elements into a new graph
-			newgraph := g.FilterGraph(g.Name, dfs)
-
-			// add number of elements found to found variable
-			d = append(d, dfs...) // extend
-
-			// return this new graph to the channel
-			ch <- newgraph
-
-			// if we've found all the elements, then we're done
-			// otherwise loop through to continue...
-		}
-		close(ch)
-	}()
-	return ch
-}
-
-// return the indegree for the graph, IOW the count of vertices that point to me
-// NOTE: this returns the values for all vertices in one big lookup table
-func (g *Graph) InDegree() map[*Vertex]int {
-	result := make(map[*Vertex]int)
-	for k := range g.Adjacency {
-		result[k] = 0 // initialize
-	}
-
-	for k := range g.Adjacency {
-		for z := range g.Adjacency[k] {
-			result[z]++
-		}
-	}
-	return result
-}
-
-// return the outdegree for the graph, IOW the count of vertices that point away
-// NOTE: this returns the values for all vertices in one big lookup table
-func (g *Graph) OutDegree() map[*Vertex]int {
-	result := make(map[*Vertex]int)
-
-	for k := range g.Adjacency {
-		result[k] = 0 // initialize
-		for _ = range g.Adjacency[k] {
-			result[k]++
-		}
-	}
-	return result
-}
-
-// returns a topological sort for the graph
-// based on descriptions and code from wikipedia and rosetta code
-// TODO: add memoization, and cache invalidation to speed this up :)
-func (g *Graph) TopologicalSort() (result []*Vertex, ok bool) { // kahn's algorithm
-	var L []*Vertex                    // empty list that will contain the sorted elements
-	var S []*Vertex                    // set of all nodes with no incoming edges
-	remaining := make(map[*Vertex]int) // amount of edges remaining
-
-	for v, d := range g.InDegree() {
-		if d == 0 {
-			// accumulate set of all nodes with no incoming edges
-			S = append(S, v)
-		} else {
-			// initialize remaining edge count from indegree
-			remaining[v] = d
-		}
-	}
-
-	for len(S) > 0 {
-		last := len(S) - 1 // remove a node v from S
-		v := S[last]
-		S = S[:last]
-		L = append(L, v) // add v to tail of L
-		for n := range g.Adjacency[v] {
-			// for each node n remaining in the graph, consume from
-			// remaining, so for remaining[n] > 0
-			if remaining[n] > 0 {
-				remaining[n]--         // remove edge from the graph
-				if remaining[n] == 0 { // if n has no other incoming edges
-					S = append(S, n) // insert n into S
-				}
-			}
-		}
-	}
-
-	// if graph has edges, eg if any value in rem is > 0
-	for c, in := range remaining {
-		if in > 0 {
-			for n := range g.Adjacency[c] {
-				if remaining[n] > 0 {
-					return nil, false // not a dag!
-				}
-			}
-		}
-	}
-
-	return L, true
-}
-
-// Reachability finds the shortest path in a DAG from a to b, and returns the
-// slice of vertices that matched this particular path including both a and b.
-// It returns nil if a or b is nil, and returns empty list if no path is found.
-// Since there could be more than one possible result for this operation, we
-// arbitrarily choose one of the shortest possible. As a result, this should
-// actually return a tree if we cared about correctness.
-// This operates by a recursive algorithm; a more efficient version is likely.
-// If you don't give this function a DAG, you might cause infinite recursion!
-func (g *Graph) Reachability(a, b *Vertex) []*Vertex {
-	if a == nil || b == nil {
-		return nil
-	}
-	vertices := g.OutgoingGraphEdges(a) // what points away from a ?
-	if len(vertices) == 0 {
-		return []*Vertex{} // nope
-	}
-	if VertexContains(b, vertices) {
-		return []*Vertex{a, b} // found
-	}
-	// TODO: parallelize this with go routines?
-	var collected = make([][]*Vertex, len(vertices))
-	pick := -1
-	for i, v := range vertices {
-		collected[i] = g.Reachability(v, b) // find b by recursion
-		if l := len(collected[i]); l > 0 {
-			// pick shortest path
-			// TODO: technically i should return a tree
-			if pick < 0 || l < len(collected[pick]) {
-				pick = i
-			}
-		}
-	}
-	if pick < 0 {
-		return []*Vertex{} // nope
-	}
-	result := []*Vertex{a} // tack on a
-	result = append(result, collected[pick]...)
-	return result
-}
-
-// VertexMerge merges v2 into v1 by reattaching the edges where appropriate,
-// and then by deleting v2 from the graph. Since more than one edge between two
-// vertices is not allowed, duplicate edges are merged as well. an edge merge
-// function can be provided if you'd like to control how you merge the edges!
-func (g *Graph) VertexMerge(v1, v2 *Vertex, vertexMergeFn func(*Vertex, *Vertex) (*Vertex, error), edgeMergeFn func(*Edge, *Edge) *Edge) error {
-	// methodology
-	// 1) edges between v1 and v2 are removed
-	//Loop:
-	for k1 := range g.Adjacency {
-		for k2 := range g.Adjacency[k1] {
-			// v1 -> v2 || v2 -> v1
-			if (k1 == v1 && k2 == v2) || (k1 == v2 && k2 == v1) {
-				delete(g.Adjacency[k1], k2) // delete map & edge
-				// NOTE: if we assume this is a DAG, then we can
-				// assume only v1 -> v2 OR v2 -> v1 exists, and
-				// we can break out of these loops immediately!
-				//break Loop
-				break
-			}
-		}
-	}
-
-	// 2) edges that point towards v2 from X now point to v1 from X (no dupes)
-	for _, x := range g.IncomingGraphEdges(v2) { // all to vertex v (??? -> v)
-		e := g.Adjacency[x][v2] // previous edge
-		r := g.Reachability(x, v1)
-		// merge e with ex := g.Adjacency[x][v1] if it exists!
-		if ex, exists := g.Adjacency[x][v1]; exists && edgeMergeFn != nil && len(r) == 0 {
-			e = edgeMergeFn(e, ex)
-		}
-		if len(r) == 0 { // if not reachable, add it
-			g.AddEdge(x, v1, e) // overwrite edge
-		} else if edgeMergeFn != nil { // reachable, merge e through...
-			prev := x // initial condition
-			for i, next := range r {
-				if i == 0 {
-					// next == prev, therefore skip
-					continue
-				}
-				// this edge is from: prev, to: next
-				ex, _ := g.Adjacency[prev][next] // get
-				ex = edgeMergeFn(ex, e)
-				g.Adjacency[prev][next] = ex // set
-				prev = next
-			}
-		}
-		delete(g.Adjacency[x], v2) // delete old edge
-	}
-
-	// 3) edges that point from v2 to X now point from v1 to X (no dupes)
-	for _, x := range g.OutgoingGraphEdges(v2) { // all from vertex v (v -> ???)
-		e := g.Adjacency[v2][x] // previous edge
-		r := g.Reachability(v1, x)
-		// merge e with ex := g.Adjacency[v1][x] if it exists!
-		if ex, exists := g.Adjacency[v1][x]; exists && edgeMergeFn != nil && len(r) == 0 {
-			e = edgeMergeFn(e, ex)
-		}
-		if len(r) == 0 {
-			g.AddEdge(v1, x, e) // overwrite edge
-		} else if edgeMergeFn != nil { // reachable, merge e through...
-			prev := v1 // initial condition
-			for i, next := range r {
-				if i == 0 {
-					// next == prev, therefore skip
-					continue
-				}
-				// this edge is from: prev, to: next
-				ex, _ := g.Adjacency[prev][next]
-				ex = edgeMergeFn(ex, e)
-				g.Adjacency[prev][next] = ex
-				prev = next
-			}
-		}
-		delete(g.Adjacency[v2], x)
-	}
-
-	// 4) merge and then remove the (now merged/grouped) vertex
-	if vertexMergeFn != nil { // run vertex merge function
-		if v, err := vertexMergeFn(v1, v2); err != nil {
-			return err
-		} else if v != nil { // replace v1 with the "merged" version...
-			v1 = v // XXX: will this replace v1 the way we want?
-		}
-	}
-	g.DeleteVertex(v2) // remove grouped vertex
-
-	// 5) creation of a cyclic graph should throw an error
-	if _, dag := g.TopologicalSort(); !dag { // am i a dag or not?
-		return fmt.Errorf("Graph is not a dag!")
-	}
-	return nil // success
-}
-
-func HeisenbergCount(ch chan *Vertex) int {
-	c := 0
-	for x := range ch {
-		_ = x
-		c++
-	}
-	return c
-}
-
-// GetTimestamp returns the timestamp of a vertex
-func (v *Vertex) GetTimestamp() int64 {
-	return v.timestamp
-}
-
-// UpdateTimestamp updates the timestamp on a vertex and returns the new value
-func (v *Vertex) UpdateTimestamp() int64 {
-	v.timestamp = time.Now().UnixNano() // update
-	return v.timestamp
-}
-
-// can this element run right now?
-func (g *Graph) OKTimestamp(v *Vertex) bool {
-	// these are all the vertices pointing TO v, eg: ??? -> v
-	for _, n := range g.IncomingGraphEdges(v) {
-		// if the vertex has a greater timestamp than any pre-req (n)
-		// then we can't run right now...
-		// if they're equal (eg: on init of 0) then we also can't run
-		// b/c we should let our pre-req's go first...
-		x, y := v.GetTimestamp(), n.GetTimestamp()
-		if DEBUG {
-			log.Printf("%v[%v]: OKTimestamp: (%v) >= %v[%v](%v): !%v", v.Kind(), v.GetName(), x, n.Kind(), n.GetName(), y, x >= y)
-		}
-		if x >= y {
-			return false
-		}
-	}
-	return true
-}
-
-// notify nodes after me in the dependency graph that they need refreshing...
-// NOTE: this assumes that this can never fail or need to be rescheduled
-func (g *Graph) Poke(v *Vertex, activity bool) {
-	// these are all the vertices pointing AWAY FROM v, eg: v -> ???
-	for _, n := range g.OutgoingGraphEdges(v) {
-		// XXX: if we're in state event and haven't been cancelled by
-		// apply, then we can cancel a poke to a child, right? XXX
-		// XXX: if n.Res.GetState() != resStateEvent { // is this correct?
-		if true { // XXX
-			if DEBUG {
-				log.Printf("%v[%v]: Poke: %v[%v]", v.Kind(), v.GetName(), n.Kind(), n.GetName())
-			}
-			n.SendEvent(eventPoke, false, activity) // XXX: can this be switched to sync?
-		} else {
-			if DEBUG {
-				log.Printf("%v[%v]: Poke: %v[%v]: Skipped!", v.Kind(), v.GetName(), n.Kind(), n.GetName())
-			}
-		}
-	}
-}
-
-// poke the pre-requisites that are stale and need to run before I can run...
-func (g *Graph) BackPoke(v *Vertex) {
-	// these are all the vertices pointing TO v, eg: ??? -> v
-	for _, n := range g.IncomingGraphEdges(v) {
-		x, y, s := v.GetTimestamp(), n.GetTimestamp(), n.Res.GetState()
-		// if the parent timestamp needs poking AND it's not in state
-		// resStateEvent, then poke it. If the parent is in resStateEvent it
-		// means that an event is pending, so we'll be expecting a poke
-		// back soon, so we can safely discard the extra parent poke...
-		// TODO: implement a stateLT (less than) to tell if something
-		// happens earlier in the state cycle and that doesn't wrap nil
-		if x >= y && (s != resStateEvent && s != resStateCheckApply) {
-			if DEBUG {
-				log.Printf("%v[%v]: BackPoke: %v[%v]", v.Kind(), v.GetName(), n.Kind(), n.GetName())
-			}
-			n.SendEvent(eventBackPoke, false, false) // XXX: can this be switched to sync?
-		} else {
-			if DEBUG {
-				log.Printf("%v[%v]: BackPoke: %v[%v]: Skipped!", v.Kind(), v.GetName(), n.Kind(), n.GetName())
-			}
-		}
-	}
-}
-
-// XXX: rename this function
-func (g *Graph) Process(v *Vertex) {
-	obj := v.Res
-	if DEBUG {
-		log.Printf("%v[%v]: Process()", obj.Kind(), obj.GetName())
-	}
-	obj.SetState(resStateEvent)
-	var ok = true
-	var apply = false // did we run an apply?
-	// is it okay to run dependency wise right now?
-	// if not, that's okay because when the dependency runs, it will poke
-	// us back and we will run if needed then!
-	if g.OKTimestamp(v) {
-		if DEBUG {
-			log.Printf("%v[%v]: OKTimestamp(%v)", obj.Kind(), obj.GetName(), v.GetTimestamp())
-		}
-
-		obj.SetState(resStateCheckApply)
-		// if this fails, don't UpdateTimestamp()
-		stateok, err := obj.CheckApply(true)
-		if stateok && err != nil { // should never return this way
-			log.Fatalf("%v[%v]: CheckApply(): %t, %+v", obj.Kind(), obj.GetName(), stateok, err)
-		}
-		if DEBUG {
-			log.Printf("%v[%v]: CheckApply(): %t, %v", obj.Kind(), obj.GetName(), stateok, err)
-		}
-
-		if !stateok { // if state *was* not ok, we had to have apply'ed
-			if err != nil { // error during check or apply
-				ok = false
-			} else {
-				apply = true
-			}
-		}
-
-		if ok {
-			// update this timestamp *before* we poke or the poked
-			// nodes might fail due to having a too old timestamp!
-			v.UpdateTimestamp()          // this was touched...
-			obj.SetState(resStatePoking) // can't cancel parent poke
-			g.Poke(v, apply)
-		}
-		// poke at our pre-req's instead since they need to refresh/run...
-	} else {
-		// only poke at the pre-req's that need to run
-		go g.BackPoke(v)
-	}
-}
-
-// main kick to start the graph
-func (g *Graph) Start(wg *sync.WaitGroup, first bool) { // start or continue
-	log.Printf("State: %v -> %v", g.SetState(graphStateStarting), g.GetState())
-	defer log.Printf("State: %v -> %v", g.SetState(graphStateStarted), g.GetState())
-	t, _ := g.TopologicalSort()
-	// TODO: only calculate indegree if `first` is true to save resources
-	indegree := g.InDegree() // compute all of the indegree's
-	for _, v := range Reverse(t) {
-
-		if !v.Res.IsWatching() { // if Watch() is not running...
-			wg.Add(1)
-			// must pass in value to avoid races...
-			// see: https://ttboj.wordpress.com/2015/07/27/golang-parallelism-issues-causing-too-many-open-files-error/
-			go func(vv *Vertex) {
-				defer wg.Done()
-				// listen for chan events from Watch() and run
-				// the Process() function when they're received
-				// this avoids us having to pass the data into
-				// the Watch() function about which graph it is
-				// running on, which isolates things nicely...
-				chanProcess := make(chan Event)
-				go func() {
-					for event := range chanProcess {
-						// this has to be synchronous,
-						// because otherwise the Res
-						// event loop will keep running
-						// and change state, causing the
-						// converged timeout to fire!
-						g.Process(vv)
-						event.ACK() // sync
-					}
-				}()
-				vv.Res.Watch(chanProcess) // i block until i end
-				close(chanProcess)
-				log.Printf("%v[%v]: Exited", vv.Kind(), vv.GetName())
-			}(v)
-		}
-
-		// selective poke: here we reduce the number of initial pokes
-		// to the minimum required to activate every vertex in the
-		// graph, either by direct action, or by getting poked by a
-		// vertex that was previously activated. if we poke each vertex
-		// that has no incoming edges, then we can be sure to reach the
-		// whole graph. Please note: this may mask certain optimization
-		// failures, such as any poke limiting code in Poke() or
-		// BackPoke(). You might want to disable this selective start
-		// when experimenting with and testing those elements.
-		// if we are unpausing (since it's not the first run of this
-		// function) we need to poke to *unpause* every graph vertex,
-		// and not just selectively the subset with no indegree.
-		if (!first) || indegree[v] == 0 {
-			// ensure state is started before continuing on to next vertex
-			for !v.SendEvent(eventStart, true, false) {
-				if DEBUG {
-					// if SendEvent fails, we aren't up yet
-					log.Printf("%v[%v]: Retrying SendEvent(Start)", v.Kind(), v.GetName())
-					// sleep here briefly or otherwise cause
-					// a different goroutine to be scheduled
-					time.Sleep(1 * time.Millisecond)
-				}
-			}
-		}
-	}
-}
-
-func (g *Graph) Pause() {
-	log.Printf("State: %v -> %v", g.SetState(graphStatePausing), g.GetState())
-	defer log.Printf("State: %v -> %v", g.SetState(graphStatePaused), g.GetState())
-	t, _ := g.TopologicalSort()
-	for _, v := range t { // squeeze out the events...
-		v.SendEvent(eventPause, true, false)
-	}
-}
-
-func (g *Graph) Exit() {
-	if g == nil {
-		return
-	} // empty graph that wasn't populated yet
-	t, _ := g.TopologicalSort()
-	for _, v := range t { // squeeze out the events...
-		// turn off the taps...
-		// XXX: do this by sending an exit signal, and then returning
-		// when we hit the 'default' in the select statement!
-		// XXX: we can do this to quiesce, but it's not necessary now
-
-		v.SendEvent(eventExit, true, false)
-	}
-}
-
-func (g *Graph) SetConvergedCallback(ctimeout int, converged chan bool) {
-	for v := range g.GetVerticesChan() {
-		v.Res.SetConvergedCallback(ctimeout, converged)
-	}
-}
-
-// in array function to test *Vertex in a slice of *Vertices
-func VertexContains(needle *Vertex, haystack []*Vertex) bool {
-	for _, v := range haystack {
-		if needle == v {
-			return true
-		}
-	}
-	return false
-}
-
-// reverse a list of vertices
-func Reverse(vs []*Vertex) []*Vertex {
-	//var out []*Vertex       // XXX: golint suggests, but it fails testing
-	out := make([]*Vertex, 0) // empty list
-	l := len(vs)
-	for i := range vs {
-		out = append(out, vs[l-i-1])
-	}
-	return out
-}

pgraph/actions.go

@@ -0,0 +1,535 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgraph
+
+import (
+	"fmt"
+	"log"
+	"math"
+	"sync"
+	"time"
+
+	"github.com/purpleidea/mgmt/event"
+	"github.com/purpleidea/mgmt/resources"
+
+	errwrap "github.com/pkg/errors"
+)
+
+// GetTimestamp returns the timestamp of a vertex
+func (v *Vertex) GetTimestamp() int64 {
+	return v.timestamp
+}
+
+// UpdateTimestamp updates the timestamp on a vertex and returns the new value
+func (v *Vertex) UpdateTimestamp() int64 {
+	v.timestamp = time.Now().UnixNano() // update
+	return v.timestamp
+}
+
+// OKTimestamp returns true if this element can run right now?
+func (g *Graph) OKTimestamp(v *Vertex) bool {
+	// these are all the vertices pointing TO v, eg: ??? -> v
+	for _, n := range g.IncomingGraphVertices(v) {
+		// if the vertex has a greater timestamp than any pre-req (n)
+		// then we can't run right now...
+		// if they're equal (eg: on init of 0) then we also can't run
+		// b/c we should let our pre-req's go first...
+		x, y := v.GetTimestamp(), n.GetTimestamp()
+		if g.Flags.Debug {
+			log.Printf("%s[%s]: OKTimestamp: (%v) >= %s[%s](%v): !%v", v.Kind(), v.GetName(), x, n.Kind(), n.GetName(), y, x >= y)
+		}
+		if x >= y {
+			return false
+		}
+	}
+	return true
+}
+
+// Poke notifies nodes after me in the dependency graph that they need refreshing...
+// NOTE: this assumes that this can never fail or need to be rescheduled
+func (g *Graph) Poke(v *Vertex, activity bool) error {
+	var wg sync.WaitGroup
+	// these are all the vertices pointing AWAY FROM v, eg: v -> ???
+	for _, n := range g.OutgoingGraphVertices(v) {
+		// XXX: if we're in state event and haven't been cancelled by
+		// apply, then we can cancel a poke to a child, right? XXX
+		// XXX: if n.Res.getState() != resources.ResStateEvent || activity { // is this correct?
+		if true || activity { // XXX: ???
+			if g.Flags.Debug {
+				log.Printf("%s[%s]: Poke: %s[%s]", v.Kind(), v.GetName(), n.Kind(), n.GetName())
+			}
+			wg.Add(1)
+			go func(nn *Vertex) error {
+				defer wg.Done()
+				edge := g.Adjacency[v][nn] // lookup
+				notify := edge.Notify && edge.Refresh()
+
+				// FIXME: is it okay that this is sync?
+				nn.SendEvent(event.EventPoke, true, notify)
+				// TODO: check return value?
+				return nil // never error for now...
+			}(n)
+
+		} else {
+			if g.Flags.Debug {
+				log.Printf("%s[%s]: Poke: %s[%s]: Skipped!", v.Kind(), v.GetName(), n.Kind(), n.GetName())
+			}
+		}
+	}
+	wg.Wait() // wait for all the pokes to complete
+	return nil
+}
+
+// BackPoke pokes the pre-requisites that are stale and need to run before I can run.
+func (g *Graph) BackPoke(v *Vertex) {
+	// these are all the vertices pointing TO v, eg: ??? -> v
+	for _, n := range g.IncomingGraphVertices(v) {
+		x, y, s := v.GetTimestamp(), n.GetTimestamp(), n.Res.GetState()
+		// if the parent timestamp needs poking AND it's not in state
+		// ResStateEvent, then poke it. If the parent is in ResStateEvent it
+		// means that an event is pending, so we'll be expecting a poke
+		// back soon, so we can safely discard the extra parent poke...
+		// TODO: implement a stateLT (less than) to tell if something
+		// happens earlier in the state cycle and that doesn't wrap nil
+		if x >= y && (s != resources.ResStateEvent && s != resources.ResStateCheckApply) {
+			if g.Flags.Debug {
+				log.Printf("%s[%s]: BackPoke: %s[%s]", v.Kind(), v.GetName(), n.Kind(), n.GetName())
+			}
+			// FIXME: is it okay that this is sync?
+			n.SendEvent(event.EventBackPoke, true, false)
+		} else {
+			if g.Flags.Debug {
+				log.Printf("%s[%s]: BackPoke: %s[%s]: Skipped!", v.Kind(), v.GetName(), n.Kind(), n.GetName())
+			}
+		}
+	}
+}
+
+// RefreshPending determines if any previous nodes have a refresh pending here.
+// If this is true, it means I am expected to apply a refresh when I next run.
+func (g *Graph) RefreshPending(v *Vertex) bool {
+	var refresh bool
+	for _, edge := range g.IncomingGraphEdges(v) {
+		// if we asked for a notify *and* if one is pending!
+		if edge.Notify && edge.Refresh() {
+			refresh = true
+			break
+		}
+	}
+	return refresh
+}
+
+// SetUpstreamRefresh sets the refresh value to any upstream vertices.
+func (g *Graph) SetUpstreamRefresh(v *Vertex, b bool) {
+	for _, edge := range g.IncomingGraphEdges(v) {
+		if edge.Notify {
+			edge.SetRefresh(b)
+		}
+	}
+}
+
+// SetDownstreamRefresh sets the refresh value to any downstream vertices.
+func (g *Graph) SetDownstreamRefresh(v *Vertex, b bool) {
+	for _, edge := range g.OutgoingGraphEdges(v) {
+		// if we asked for a notify *and* if one is pending!
+		if edge.Notify {
+			edge.SetRefresh(b)
+		}
+	}
+}
+
+// Process is the primary function to execute for a particular vertex in the graph.
+func (g *Graph) Process(v *Vertex) error {
+	obj := v.Res
+	if g.Flags.Debug {
+		log.Printf("%s[%s]: Process()", obj.Kind(), obj.GetName())
+	}
+	obj.SetState(resources.ResStateEvent)
+	var ok = true
+	var applied = false // did we run an apply?
+	// is it okay to run dependency wise right now?
+	// if not, that's okay because when the dependency runs, it will poke
+	// us back and we will run if needed then!
+	if g.OKTimestamp(v) {
+		if g.Flags.Debug {
+			log.Printf("%s[%s]: OKTimestamp(%v)", obj.Kind(), obj.GetName(), v.GetTimestamp())
+		}
+
+		obj.SetState(resources.ResStateCheckApply)
+
+		// connect any senders to receivers and detect if values changed
+		if updated, err := obj.SendRecv(obj); err != nil {
+			return errwrap.Wrapf(err, "could not SendRecv in Process")
+		} else if len(updated) > 0 {
+			for _, changed := range updated {
+				if changed { // at least one was updated
+					obj.StateOK(false) // invalidate cache, mark as dirty
+					break
+				}
+			}
+		}
+
+		var noop = obj.Meta().Noop // lookup the noop value
+		var refresh bool
+		var checkOK bool
+		var err error
+
+		if g.Flags.Debug {
+			log.Printf("%s[%s]: CheckApply(%t)", obj.Kind(), obj.GetName(), !noop)
+		}
+
+		// lookup the refresh (notification) variable
+		refresh = g.RefreshPending(v) // do i need to perform a refresh?
+		obj.SetRefresh(refresh)       // tell the resource
+
+		// check cached state, to skip CheckApply; can't skip if refreshing
+		if !refresh && obj.IsStateOK() {
+			checkOK, err = true, nil
+
+			// NOTE: technically this block is wrong because we don't know
+			// if the resource implements refresh! If it doesn't, we could
+			// skip this, but it doesn't make a big difference under noop!
+		} else if noop && refresh { // had a refresh to do w/ noop!
+			checkOK, err = false, nil // therefore the state is wrong
+
+			// run the CheckApply!
+		} else {
+			// if this fails, don't UpdateTimestamp()
+			checkOK, err = obj.CheckApply(!noop)
+		}
+
+		if checkOK && err != nil { // should never return this way
+			log.Fatalf("%s[%s]: CheckApply(): %t, %+v", obj.Kind(), obj.GetName(), checkOK, err)
+		}
+		if g.Flags.Debug {
+			log.Printf("%s[%s]: CheckApply(): %t, %v", obj.Kind(), obj.GetName(), checkOK, err)
+		}
+
+		// if CheckApply ran without noop and without error, state should be good
+		if !noop && err == nil { // aka !noop || checkOK
+			obj.StateOK(true) // reset
+			if refresh {
+				g.SetUpstreamRefresh(v, false) // refresh happened, clear the request
+				obj.SetRefresh(false)
+			}
+		}
+
+		if !checkOK { // if state *was* not ok, we had to have apply'ed
+			if err != nil { // error during check or apply
+				ok = false
+			} else {
+				applied = true
+			}
+		}
+
+		// when noop is true we always want to update timestamp
+		if noop && err == nil {
+			ok = true
+		}
+
+		if ok {
+			// did we actually do work?
+			activity := applied
+			if noop {
+				activity = false // no we didn't do work...
+			}
+
+			if activity { // add refresh flag to downstream edges...
+				g.SetDownstreamRefresh(v, true)
+			}
+
+			// update this timestamp *before* we poke or the poked
+			// nodes might fail due to having a too old timestamp!
+			v.UpdateTimestamp()                    // this was touched...
+			obj.SetState(resources.ResStatePoking) // can't cancel parent poke
+			if err := g.Poke(v, activity); err != nil {
+				return errwrap.Wrapf(err, "the Poke() failed")
+			}
+		}
+		// poke at our pre-req's instead since they need to refresh/run...
+		return errwrap.Wrapf(err, "could not Process() successfully")
+	}
+	// else... only poke at the pre-req's that need to run
+	go g.BackPoke(v)
+	return nil
+}
+
+// SentinelErr is a sentinal as an error type that wraps an arbitrary error.
+type SentinelErr struct {
+	err error
+}
+
+// Error is the required method to fulfill the error type.
+func (obj *SentinelErr) Error() string {
+	return obj.err.Error()
+}
+
+// Worker is the common run frontend of the vertex. It handles all of the retry
+// and retry delay common code, and ultimately returns the final status of this
+// vertex execution.
+func (g *Graph) Worker(v *Vertex) error {
+	// listen for chan events from Watch() and run
+	// the Process() function when they're received
+	// this avoids us having to pass the data into
+	// the Watch() function about which graph it is
+	// running on, which isolates things nicely...
+	obj := v.Res
+	// TODO: is there a better system for the `Watching` flag?
+	obj.SetWatching(true)
+	defer obj.SetWatching(false)
+	processChan := make(chan event.Event)
+	go func() {
+		running := false
+		var timer = time.NewTimer(time.Duration(math.MaxInt64)) // longest duration
+		if !timer.Stop() {
+			<-timer.C // unnecessary, shouldn't happen
+		}
+		var delay = time.Duration(v.Meta().Delay) * time.Millisecond
+		var retry = v.Meta().Retry // number of tries left, -1 for infinite
+		var saved event.Event
+	Loop:
+		for {
+			// this has to be synchronous, because otherwise the Res
+			// event loop will keep running and change state,
+			// causing the converged timeout to fire!
+			select {
+			case event, ok := <-processChan: // must use like this
+				if running && ok {
+					// we got an event that wasn't a close,
+					// while we were waiting for the timer!
+					// if this happens, it might be a bug:(
+					log.Fatalf("%s[%s]: Worker: Unexpected event: %+v", v.Kind(), v.GetName(), event)
+				}
+				if !ok { // processChan closed, let's exit
+					break Loop // no event, so no ack!
+				}
+
+				// the above mentioned synchronous part, is the
+				// running of this function, paired with an ack.
+				if e := g.Process(v); e != nil {
+					saved = event
+					log.Printf("%s[%s]: CheckApply errored: %v", v.Kind(), v.GetName(), e)
+					if retry == 0 {
+						// wrap the error in the sentinel
+						event.ACKNACK(&SentinelErr{e}) // fail the Watch()
+						break Loop
+					}
+					if retry > 0 { // don't decrement the -1
+						retry--
+					}
+					log.Printf("%s[%s]: CheckApply: Retrying after %.4f seconds (%d left)", v.Kind(), v.GetName(), delay.Seconds(), retry)
+					// start the timer...
+					timer.Reset(delay)
+					running = true
+					continue
+				}
+				retry = v.Meta().Retry // reset on success
+				event.ACK()            // sync
+
+			case <-timer.C:
+				if !timer.Stop() {
+					//<-timer.C // blocks, docs are wrong!
+				}
+				running = false
+				log.Printf("%s[%s]: CheckApply delay expired!", v.Kind(), v.GetName())
+				// re-send this failed event, to trigger a CheckApply()
+				go func() { processChan <- saved }()
+				// TODO: should we send a fake event instead?
+				//saved = nil
+			}
+		}
+	}()
+	var err error // propagate the error up (this is a permanent BAD error!)
+	// the watch delay runs inside of the Watch resource loop, so that it
+	// can still process signals and exit if needed. It shouldn't run any
+	// resource specific code since this is supposed to be a retry delay.
+	// NOTE: we're using the same retry and delay metaparams that CheckApply
+	// uses. This is for practicality. We can separate them later if needed!
+	var watchDelay time.Duration
+	var watchRetry = v.Meta().Retry // number of tries left, -1 for infinite
+	// watch blocks until it ends, & errors to retry
+	for {
+		// TODO: do we have to stop the converged-timeout when in this block (perhaps we're in the delay block!)
+		// TODO: should we setup/manage some of the converged timeout stuff in here anyways?
+
+		// if a retry-delay was requested, wait, but don't block our events!
+		if watchDelay > 0 {
+			//var pendingSendEvent bool
+			timer := time.NewTimer(watchDelay)
+		Loop:
+			for {
+				select {
+				case <-timer.C: // the wait is over
+					break Loop // critical
+
+				// TODO: resources could have a separate exit channel to avoid this complexity!?
+				case event := <-obj.Events():
+					// NOTE: this code should match the similar Res code!
+					//cuid.SetConverged(false) // TODO: ?
+					if exit, send := obj.ReadEvent(&event); exit {
+						return nil // exit
+					} else if send {
+						// if we dive down this rabbit hole, our
+						// timer.C won't get seen until we get out!
+						// in this situation, the Watch() is blocked
+						// from performing until CheckApply returns
+						// successfully, or errors out. This isn't
+						// so bad, but we should document it. Is it
+						// possible that some resource *needs* Watch
+						// to run to be able to execute a CheckApply?
+						// That situation shouldn't be common, and
+						// should probably not be allowed. Can we
+						// avoid it though?
+						//if exit, err := doSend(); exit || err != nil {
+						//	return err // we exit or bubble up a NACK...
+						//}
+						// Instead of doing the above, we can
+						// add events to a pending list, and
+						// when we finish the delay, we can run
+						// them.
+						//pendingSendEvent = true // all events are identical for now...
+					}
+				}
+			}
+			timer.Stop() // it's nice to cleanup
+			log.Printf("%s[%s]: Watch delay expired!", v.Kind(), v.GetName())
+			// NOTE: we can avoid the send if running Watch guarantees
+			// one CheckApply event on startup!
+			//if pendingSendEvent { // TODO: should this become a list in the future?
+			//	if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
+			//		return err // we exit or bubble up a NACK...
+			//	}
+			//}
+		}
+
+		// TODO: reset the watch retry count after some amount of success
+		v.Res.RegisterConverger()
+		e := v.Res.Watch(processChan)
+		v.Res.UnregisterConverger()
+		if e == nil { // exit signal
+			err = nil // clean exit
+			break
+		}
+		if sentinelErr, ok := e.(*SentinelErr); ok { // unwrap the sentinel
+			err = sentinelErr.err
+			break // sentinel means, perma-exit
+		}
+		log.Printf("%s[%s]: Watch errored: %v", v.Kind(), v.GetName(), e)
+		if watchRetry == 0 {
+			err = fmt.Errorf("Permanent watch error: %v", e)
+			break
+		}
+		if watchRetry > 0 { // don't decrement the -1
+			watchRetry--
+		}
+		watchDelay = time.Duration(v.Meta().Delay) * time.Millisecond
+		log.Printf("%s[%s]: Watch: Retrying after %.4f seconds (%d left)", v.Kind(), v.GetName(), watchDelay.Seconds(), watchRetry)
+		// We need to trigger a CheckApply after Watch restarts, so that
+		// we catch any lost events that happened while down. We do this
+		// by getting the Watch resource to send one event once it's up!
+		//v.SendEvent(eventPoke, false, false)
+	}
+	close(processChan)
+	return err
+}
+
+// Start is a main kick to start the graph. It goes through in reverse topological
+// sort order so that events can't hit un-started vertices.
+func (g *Graph) Start(first bool) { // start or continue
+	log.Printf("State: %v -> %v", g.setState(graphStateStarting), g.getState())
+	defer log.Printf("State: %v -> %v", g.setState(graphStateStarted), g.getState())
+	var wg sync.WaitGroup
+	t, _ := g.TopologicalSort()
+	// TODO: only calculate indegree if `first` is true to save resources
+	indegree := g.InDegree() // compute all of the indegree's
+	for _, v := range Reverse(t) {
+		// selective poke: here we reduce the number of initial pokes
+		// to the minimum required to activate every vertex in the
+		// graph, either by direct action, or by getting poked by a
+		// vertex that was previously activated. if we poke each vertex
+		// that has no incoming edges, then we can be sure to reach the
+		// whole graph. Please note: this may mask certain optimization
+		// failures, such as any poke limiting code in Poke() or
+		// BackPoke(). You might want to disable this selective start
+		// when experimenting with and testing those elements.
+		// if we are unpausing (since it's not the first run of this
+		// function) we need to poke to *unpause* every graph vertex,
+		// and not just selectively the subset with no indegree.
+		if (!first) || indegree[v] == 0 {
+			v.Res.Starter(true) // let the startup code know to poke
+		}
+
+		if !v.Res.IsWatching() { // if Watch() is not running...
+			g.wg.Add(1)
+			// must pass in value to avoid races...
+			// see: https://ttboj.wordpress.com/2015/07/27/golang-parallelism-issues-causing-too-many-open-files-error/
+			go func(vv *Vertex) {
+				defer g.wg.Done()
+				// TODO: if a sufficient number of workers error,
+				// should something be done? Will these restart
+				// after perma-failure if we have a graph change?
+				if err := g.Worker(vv); err != nil { // contains the Watch and CheckApply loops
+					log.Printf("%s[%s]: Exited with failure: %v", vv.Kind(), vv.GetName(), err)
+					return
+				}
+				log.Printf("%s[%s]: Exited", vv.Kind(), vv.GetName())
+			}(v)
+		}
+
+		// let the vertices run their startup code in parallel
+		wg.Add(1)
+		go func(vv *Vertex) {
+			defer wg.Done()
+			vv.Res.Started() // block until started
+		}(v)
+
+		if !first { // unpause!
+			v.Res.SendEvent(event.EventStart, true, false) // sync!
+		}
+	}
+
+	wg.Wait() // wait for everyone
+}
+
+// Pause sends pause events to the graph in a topological sort order.
+func (g *Graph) Pause() {
+	log.Printf("State: %v -> %v", g.setState(graphStatePausing), g.getState())
+	defer log.Printf("State: %v -> %v", g.setState(graphStatePaused), g.getState())
+	t, _ := g.TopologicalSort()
+	for _, v := range t { // squeeze out the events...
+		v.SendEvent(event.EventPause, true, false)
+	}
+}
+
+// Exit sends exit events to the graph in a topological sort order.
+func (g *Graph) Exit() {
+	if g == nil {
+		return
+	} // empty graph that wasn't populated yet
+	t, _ := g.TopologicalSort()
+	for _, v := range t { // squeeze out the events...
+		// turn off the taps...
+		// XXX: consider instead doing this by closing the Res.events channel instead?
+		// XXX: do this by sending an exit signal, and then returning
+		// when we hit the 'default' in the select statement!
+		// XXX: we can do this to quiesce, but it's not necessary now
+
+		v.SendEvent(event.EventExit, true, false)
+	}
+	g.wg.Wait() // for now, this doesn't need to be a separate Wait() method
+}

pgraph/autoedge.go

@@ -0,0 +1,103 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// Package pgraph represents the internal "pointer graph" that we use.
+package pgraph
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/purpleidea/mgmt/resources"
+)
+
+// add edges to the vertex in a graph based on if it matches a uid list
+func (g *Graph) addEdgesByMatchingUIDS(v *Vertex, uids []resources.ResUID) []bool {
+	// search for edges and see what matches!
+	var result []bool
+
+	// loop through each uid, and see if it matches any vertex
+	for _, uid := range uids {
+		var found = false
+		// uid is a ResUID object
+		for _, vv := range g.GetVertices() { // search
+			if v == vv { // skip self
+				continue
+			}
+			if g.Flags.Debug {
+				log.Printf("Compile: AutoEdge: Match: %v[%v] with UID: %v[%v]", vv.Kind(), vv.GetName(), uid.Kind(), uid.GetName())
+			}
+			// we must match to an effective UID for the resource,
+			// that is to say, the name value of a res is a helpful
+			// handle, but it is not necessarily a unique identity!
+			// remember, resources can return multiple UID's each!
+			if resources.UIDExistsInUIDs(uid, vv.GetUIDs()) {
+				// add edge from: vv -> v
+				if uid.Reversed() {
+					txt := fmt.Sprintf("AutoEdge: %v[%v] -> %v[%v]", vv.Kind(), vv.GetName(), v.Kind(), v.GetName())
+					log.Printf("Compile: Adding %v", txt)
+					g.AddEdge(vv, v, NewEdge(txt))
+				} else { // edges go the "normal" way, eg: pkg resource
+					txt := fmt.Sprintf("AutoEdge: %v[%v] -> %v[%v]", v.Kind(), v.GetName(), vv.Kind(), vv.GetName())
+					log.Printf("Compile: Adding %v", txt)
+					g.AddEdge(v, vv, NewEdge(txt))
+				}
+				found = true
+				break
+			}
+		}
+		result = append(result, found)
+	}
+	return result
+}
+
+// AutoEdges adds the automatic edges to the graph.
+func (g *Graph) AutoEdges() {
+	log.Println("Compile: Adding AutoEdges...")
+	for _, v := range g.GetVertices() { // for each vertexes autoedges
+		if !v.Meta().AutoEdge { // is the metaparam true?
+			continue
+		}
+		autoEdgeObj := v.AutoEdges()
+		if autoEdgeObj == nil {
+			log.Printf("%v[%v]: Config: No auto edges were found!", v.Kind(), v.GetName())
+			continue // next vertex
+		}
+
+		for { // while the autoEdgeObj has more uids to add...
+			uids := autoEdgeObj.Next() // get some!
+			if uids == nil {
+				log.Printf("%v[%v]: Config: The auto edge list is empty!", v.Kind(), v.GetName())
+				break // inner loop
+			}
+			if g.Flags.Debug {
+				log.Println("Compile: AutoEdge: UIDS:")
+				for i, u := range uids {
+					log.Printf("Compile: AutoEdge: UID%d: %v", i, u)
+				}
+			}
+
+			// match and add edges
+			result := g.addEdgesByMatchingUIDS(v, uids)
+
+			// report back, and find out if we should continue
+			if !autoEdgeObj.Test(result) {
+				break
+			}
+		}
+	}
+}

pgraph/autogroup.go

@@ -0,0 +1,348 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgraph
+
+import (
+	"fmt"
+	"log"
+
+	errwrap "github.com/pkg/errors"
+)
+
+// AutoGrouper is the required interface to implement for an autogroup algorithm
+type AutoGrouper interface {
+	// listed in the order these are typically called in...
+	name() string                                  // friendly identifier
+	init(*Graph) error                             // only call once
+	vertexNext() (*Vertex, *Vertex, error)         // mostly algorithmic
+	vertexCmp(*Vertex, *Vertex) error              // can we merge these ?
+	vertexMerge(*Vertex, *Vertex) (*Vertex, error) // vertex merge fn to use
+	edgeMerge(*Edge, *Edge) *Edge                  // edge merge fn to use
+	vertexTest(bool) (bool, error)                 // call until false
+}
+
+// baseGrouper is the base type for implementing the AutoGrouper interface
+type baseGrouper struct {
+	graph    *Graph    // store a pointer to the graph
+	vertices []*Vertex // cached list of vertices
+	i        int
+	j        int
+	done     bool
+}
+
+// name provides a friendly name for the logs to see
+func (ag *baseGrouper) name() string {
+	return "baseGrouper"
+}
+
+// init is called only once and before using other AutoGrouper interface methods
+// the name method is the only exception: call it any time without side effects!
+func (ag *baseGrouper) init(g *Graph) error {
+	if ag.graph != nil {
+		return fmt.Errorf("The init method has already been called!")
+	}
+	ag.graph = g                               // pointer
+	ag.vertices = ag.graph.GetVerticesSorted() // cache in deterministic order!
+	ag.i = 0
+	ag.j = 0
+	if len(ag.vertices) == 0 { // empty graph
+		ag.done = true
+		return nil
+	}
+	return nil
+}
+
+// vertexNext is a simple iterator that loops through vertex (pair) combinations
+// an intelligent algorithm would selectively offer only valid pairs of vertices
+// these should satisfy logical grouping requirements for the autogroup designs!
+// the desired algorithms can override, but keep this method as a base iterator!
+func (ag *baseGrouper) vertexNext() (v1, v2 *Vertex, err error) {
+	// this does a for v... { for w... { return v, w }} but stepwise!
+	l := len(ag.vertices)
+	if ag.i < l {
+		v1 = ag.vertices[ag.i]
+	}
+	if ag.j < l {
+		v2 = ag.vertices[ag.j]
+	}
+
+	// in case the vertex was deleted
+	if !ag.graph.HasVertex(v1) {
+		v1 = nil
+	}
+	if !ag.graph.HasVertex(v2) {
+		v2 = nil
+	}
+
+	// two nested loops...
+	if ag.j < l {
+		ag.j++
+	}
+	if ag.j == l {
+		ag.j = 0
+		if ag.i < l {
+			ag.i++
+		}
+		if ag.i == l {
+			ag.done = true
+		}
+	}
+
+	return
+}
+
+func (ag *baseGrouper) vertexCmp(v1, v2 *Vertex) error {
+	if v1 == nil || v2 == nil {
+		return fmt.Errorf("Vertex is nil!")
+	}
+	if v1 == v2 { // skip yourself
+		return fmt.Errorf("Vertices are the same!")
+	}
+	if v1.Kind() != v2.Kind() { // we must group similar kinds
+		// TODO: maybe future resources won't need this limitation?
+		return fmt.Errorf("The two resources aren't the same kind!")
+	}
+	// someone doesn't want to group!
+	if !v1.Meta().AutoGroup || !v2.Meta().AutoGroup {
+		return fmt.Errorf("One of the autogroup flags is false!")
+	}
+	if v1.Res.IsGrouped() { // already grouped!
+		return fmt.Errorf("Already grouped!")
+	}
+	if len(v2.Res.GetGroup()) > 0 { // already has children grouped!
+		return fmt.Errorf("Already has groups!")
+	}
+	if !v1.Res.GroupCmp(v2.Res) { // resource groupcmp failed!
+		return fmt.Errorf("The GroupCmp failed!")
+	}
+	return nil // success
+}
+
+func (ag *baseGrouper) vertexMerge(v1, v2 *Vertex) (v *Vertex, err error) {
+	// NOTE: it's important to use w.Res instead of w, b/c
+	// the w by itself is the *Vertex obj, not the *Res obj
+	// which is contained within it! They both satisfy the
+	// Res interface, which is why both will compile! :(
+	err = v1.Res.GroupRes(v2.Res) // GroupRes skips stupid groupings
+	return                        // success or fail, and no need to merge the actual vertices!
+}
+
+func (ag *baseGrouper) edgeMerge(e1, e2 *Edge) *Edge {
+	return e1 // noop
+}
+
+// vertexTest processes the results of the grouping for the algorithm to know
+// return an error if something went horribly wrong, and bool false to stop
+func (ag *baseGrouper) vertexTest(b bool) (bool, error) {
+	// NOTE: this particular baseGrouper version doesn't track what happens
+	// because since we iterate over every pair, we don't care which merge!
+	if ag.done {
+		return false, nil
+	}
+	return true, nil
+}
+
+// TODO: this algorithm may not be correct in all cases. replace if needed!
+type nonReachabilityGrouper struct {
+	baseGrouper // "inherit" what we want, and reimplement the rest
+}
+
+func (ag *nonReachabilityGrouper) name() string {
+	return "nonReachabilityGrouper"
+}
+
+// this algorithm relies on the observation that if there's a path from a to b,
+// then they *can't* be merged (b/c of the existing dependency) so therefore we
+// merge anything that *doesn't* satisfy this condition or that of the reverse!
+func (ag *nonReachabilityGrouper) vertexNext() (v1, v2 *Vertex, err error) {
+	for {
+		v1, v2, err = ag.baseGrouper.vertexNext() // get all iterable pairs
+		if err != nil {
+			log.Fatalf("Error running autoGroup(vertexNext): %v", err)
+		}
+
+		if v1 != v2 { // ignore self cmp early (perf optimization)
+			// if NOT reachable, they're viable...
+			out1 := ag.graph.Reachability(v1, v2)
+			out2 := ag.graph.Reachability(v2, v1)
+			if len(out1) == 0 && len(out2) == 0 {
+				return // return v1 and v2, they're viable
+			}
+		}
+
+		// if we got here, it means we're skipping over this candidate!
+		if ok, err := ag.baseGrouper.vertexTest(false); err != nil {
+			log.Fatalf("Error running autoGroup(vertexTest): %v", err)
+		} else if !ok {
+			return nil, nil, nil // done!
+		}
+
+		// the vertexTest passed, so loop and try with a new pair...
+	}
+}
+
+// VertexMerge merges v2 into v1 by reattaching the edges where appropriate,
+// and then by deleting v2 from the graph. Since more than one edge between two
+// vertices is not allowed, duplicate edges are merged as well. an edge merge
+// function can be provided if you'd like to control how you merge the edges!
+func (g *Graph) VertexMerge(v1, v2 *Vertex, vertexMergeFn func(*Vertex, *Vertex) (*Vertex, error), edgeMergeFn func(*Edge, *Edge) *Edge) error {
+	// methodology
+	// 1) edges between v1 and v2 are removed
+	//Loop:
+	for k1 := range g.Adjacency {
+		for k2 := range g.Adjacency[k1] {
+			// v1 -> v2 || v2 -> v1
+			if (k1 == v1 && k2 == v2) || (k1 == v2 && k2 == v1) {
+				delete(g.Adjacency[k1], k2) // delete map & edge
+				// NOTE: if we assume this is a DAG, then we can
+				// assume only v1 -> v2 OR v2 -> v1 exists, and
+				// we can break out of these loops immediately!
+				//break Loop
+				break
+			}
+		}
+	}
+
+	// 2) edges that point towards v2 from X now point to v1 from X (no dupes)
+	for _, x := range g.IncomingGraphVertices(v2) { // all to vertex v (??? -> v)
+		e := g.Adjacency[x][v2] // previous edge
+		r := g.Reachability(x, v1)
+		// merge e with ex := g.Adjacency[x][v1] if it exists!
+		if ex, exists := g.Adjacency[x][v1]; exists && edgeMergeFn != nil && len(r) == 0 {
+			e = edgeMergeFn(e, ex)
+		}
+		if len(r) == 0 { // if not reachable, add it
+			g.AddEdge(x, v1, e) // overwrite edge
+		} else if edgeMergeFn != nil { // reachable, merge e through...
+			prev := x // initial condition
+			for i, next := range r {
+				if i == 0 {
+					// next == prev, therefore skip
+					continue
+				}
+				// this edge is from: prev, to: next
+				ex, _ := g.Adjacency[prev][next] // get
+				ex = edgeMergeFn(ex, e)
+				g.Adjacency[prev][next] = ex // set
+				prev = next
+			}
+		}
+		delete(g.Adjacency[x], v2) // delete old edge
+	}
+
+	// 3) edges that point from v2 to X now point from v1 to X (no dupes)
+	for _, x := range g.OutgoingGraphVertices(v2) { // all from vertex v (v -> ???)
+		e := g.Adjacency[v2][x] // previous edge
+		r := g.Reachability(v1, x)
+		// merge e with ex := g.Adjacency[v1][x] if it exists!
+		if ex, exists := g.Adjacency[v1][x]; exists && edgeMergeFn != nil && len(r) == 0 {
+			e = edgeMergeFn(e, ex)
+		}
+		if len(r) == 0 {
+			g.AddEdge(v1, x, e) // overwrite edge
+		} else if edgeMergeFn != nil { // reachable, merge e through...
+			prev := v1 // initial condition
+			for i, next := range r {
+				if i == 0 {
+					// next == prev, therefore skip
+					continue
+				}
+				// this edge is from: prev, to: next
+				ex, _ := g.Adjacency[prev][next]
+				ex = edgeMergeFn(ex, e)
+				g.Adjacency[prev][next] = ex
+				prev = next
+			}
+		}
+		delete(g.Adjacency[v2], x)
+	}
+
+	// 4) merge and then remove the (now merged/grouped) vertex
+	if vertexMergeFn != nil { // run vertex merge function
+		if v, err := vertexMergeFn(v1, v2); err != nil {
+			return err
+		} else if v != nil { // replace v1 with the "merged" version...
+			*v1 = *v // TODO: is this safe? (replacing mutexes is undefined!)
+		}
+	}
+	g.DeleteVertex(v2) // remove grouped vertex
+
+	// 5) creation of a cyclic graph should throw an error
+	if _, err := g.TopologicalSort(); err != nil { // am i a dag or not?
+		return errwrap.Wrapf(err, "TopologicalSort failed") // not a dag
+	}
+	return nil // success
+}
+
+// autoGroup is the mechanical auto group "runner" that runs the interface spec
+func (g *Graph) autoGroup(ag AutoGrouper) chan string {
+	strch := make(chan string) // output log messages here
+	go func(strch chan string) {
+		strch <- fmt.Sprintf("Compile: Grouping: Algorithm: %v...", ag.name())
+		if err := ag.init(g); err != nil {
+			log.Fatalf("Error running autoGroup(init): %v", err)
+		}
+
+		for {
+			var v, w *Vertex
+			v, w, err := ag.vertexNext() // get pair to compare
+			if err != nil {
+				log.Fatalf("Error running autoGroup(vertexNext): %v", err)
+			}
+			merged := false
+			// save names since they change during the runs
+			vStr := fmt.Sprintf("%s", v) // valid even if it is nil
+			wStr := fmt.Sprintf("%s", w)
+
+			if err := ag.vertexCmp(v, w); err != nil { // cmp ?
+				if g.Flags.Debug {
+					strch <- fmt.Sprintf("Compile: Grouping: !GroupCmp for: %s into %s", wStr, vStr)
+				}
+
+				// remove grouped vertex and merge edges (res is safe)
+			} else if err := g.VertexMerge(v, w, ag.vertexMerge, ag.edgeMerge); err != nil { // merge...
+				strch <- fmt.Sprintf("Compile: Grouping: !VertexMerge for: %s into %s", wStr, vStr)
+
+			} else { // success!
+				strch <- fmt.Sprintf("Compile: Grouping: Success for: %s into %s", wStr, vStr)
+				merged = true // woo
+			}
+
+			// did these get used?
+			if ok, err := ag.vertexTest(merged); err != nil {
+				log.Fatalf("Error running autoGroup(vertexTest): %v", err)
+			} else if !ok {
+				break // done!
+			}
+		}
+
+		close(strch)
+		return
+	}(strch) // call function
+	return strch
+}
+
+// AutoGroup runs the auto grouping on the graph and prints out log messages
+func (g *Graph) AutoGroup() {
+	// receive log messages from channel...
+	// this allows test cases to avoid printing them when they're unwanted!
+	// TODO: this algorithm may not be correct in all cases. replace if needed!
+	for str := range g.autoGroup(&nonReachabilityGrouper{}) {
+		log.Println(str)
+	}
+}

pgraph/graphviz.go

@@ -0,0 +1,110 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgraph
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"strconv"
+	"syscall"
+)
+
+// Graphviz outputs the graph in graphviz format.
+// https://en.wikipedia.org/wiki/DOT_%28graph_description_language%29
+func (g *Graph) Graphviz() (out string) {
+	//digraph g {
+	//	label="hello world";
+	//	node [shape=box];
+	//	A [label="A"];
+	//	B [label="B"];
+	//	C [label="C"];
+	//	D [label="D"];
+	//	E [label="E"];
+	//	A -> B [label=f];
+	//	B -> C [label=g];
+	//	D -> E [label=h];
+	//}
+	out += fmt.Sprintf("digraph %s {\n", g.GetName())
+	out += fmt.Sprintf("\tlabel=\"%s\";\n", g.GetName())
+	//out += "\tnode [shape=box];\n"
+	str := ""
+	for i := range g.Adjacency { // reverse paths
+		out += fmt.Sprintf("\t%s [label=\"%s[%s]\"];\n", i.GetName(), i.Kind(), i.GetName())
+		for j := range g.Adjacency[i] {
+			k := g.Adjacency[i][j]
+			// use str for clearer output ordering
+			str += fmt.Sprintf("\t%s -> %s [label=%s];\n", i.GetName(), j.GetName(), k.Name)
+		}
+	}
+	out += str
+	out += "}\n"
+	return
+}
+
+// ExecGraphviz writes out the graphviz data and runs the correct graphviz
+// filter command.
+func (g *Graph) ExecGraphviz(program, filename string) error {
+
+	switch program {
+	case "dot", "neato", "twopi", "circo", "fdp":
+	default:
+		return fmt.Errorf("Invalid graphviz program selected!")
+	}
+
+	if filename == "" {
+		return fmt.Errorf("No filename given!")
+	}
+
+	// run as a normal user if possible when run with sudo
+	uid, err1 := strconv.Atoi(os.Getenv("SUDO_UID"))
+	gid, err2 := strconv.Atoi(os.Getenv("SUDO_GID"))
+
+	err := ioutil.WriteFile(filename, []byte(g.Graphviz()), 0644)
+	if err != nil {
+		return fmt.Errorf("Error writing to filename!")
+	}
+
+	if err1 == nil && err2 == nil {
+		if err := os.Chown(filename, uid, gid); err != nil {
+			return fmt.Errorf("Error changing file owner!")
+		}
+	}
+
+	path, err := exec.LookPath(program)
+	if err != nil {
+		return fmt.Errorf("Graphviz is missing!")
+	}
+
+	out := fmt.Sprintf("%s.png", filename)
+	cmd := exec.Command(path, "-Tpng", fmt.Sprintf("-o%s", out), filename)
+
+	if err1 == nil && err2 == nil {
+		cmd.SysProcAttr = &syscall.SysProcAttr{}
+		cmd.SysProcAttr.Credential = &syscall.Credential{
+			Uid: uint32(uid),
+			Gid: uint32(gid),
+		}
+	}
+	_, err = cmd.Output()
+	if err != nil {
+		return fmt.Errorf("Error writing to image!")
+	}
+	return nil
+}

pgraph/pgraph.go

@@ -0,0 +1,658 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// Package pgraph represents the internal "pointer graph" that we use.
+package pgraph
+
+import (
+	"fmt"
+	"sort"
+	"sync"
+
+	"github.com/purpleidea/mgmt/event"
+	"github.com/purpleidea/mgmt/resources"
+
+	errwrap "github.com/pkg/errors"
+)
+
+//go:generate stringer -type=graphState -output=graphstate_stringer.go
+type graphState int
+
+const (
+	graphStateNil graphState = iota
+	graphStateStarting
+	graphStateStarted
+	graphStatePausing
+	graphStatePaused
+)
+
+// Flags contains specific constants used by the graph.
+type Flags struct {
+	Debug bool
+}
+
+// Graph is the graph structure in this library.
+// The graph abstract data type (ADT) is defined as follows:
+// * the directed graph arrows point from left to right ( -> )
+// * the arrows point away from their dependencies (eg: arrows mean "before")
+// * IOW, you might see package -> file -> service (where package runs first)
+// * This is also the direction that the notify should happen in...
+type Graph struct {
+	Name      string
+	Adjacency map[*Vertex]map[*Vertex]*Edge // *Vertex -> *Vertex (edge)
+	Flags     Flags
+	state     graphState
+	mutex     *sync.Mutex // used when modifying graph State variable
+	wg        *sync.WaitGroup
+}
+
+// Vertex is the primary vertex struct in this library.
+type Vertex struct {
+	resources.Res       // anonymous field
+	timestamp     int64 // last updated timestamp ?
+}
+
+// Edge is the primary edge struct in this library.
+type Edge struct {
+	Name   string
+	Notify bool // should we send a refresh notification along this edge?
+
+	refresh bool // is there a notify pending for the dest vertex ?
+}
+
+// NewGraph builds a new graph.
+func NewGraph(name string) *Graph {
+	return &Graph{
+		Name:      name,
+		Adjacency: make(map[*Vertex]map[*Vertex]*Edge),
+		state:     graphStateNil,
+		// ptr b/c: Mutex/WaitGroup must not be copied after first use
+		mutex: &sync.Mutex{},
+		wg:    &sync.WaitGroup{},
+	}
+}
+
+// NewVertex returns a new graph vertex struct with a contained resource.
+func NewVertex(r resources.Res) *Vertex {
+	return &Vertex{
+		Res: r,
+	}
+}
+
+// NewEdge returns a new graph edge struct.
+func NewEdge(name string) *Edge {
+	return &Edge{
+		Name: name,
+	}
+}
+
+// Refresh returns the pending refresh status of this edge.
+func (obj *Edge) Refresh() bool {
+	return obj.refresh
+}
+
+// SetRefresh sets the pending refresh status of this edge.
+func (obj *Edge) SetRefresh(b bool) {
+	obj.refresh = b
+}
+
+// Copy makes a copy of the graph struct
+func (g *Graph) Copy() *Graph {
+	newGraph := &Graph{
+		Name:      g.Name,
+		Adjacency: make(map[*Vertex]map[*Vertex]*Edge, len(g.Adjacency)),
+		Flags:     g.Flags,
+		state:     g.state,
+		mutex:     g.mutex,
+		wg:        g.wg,
+	}
+	for k, v := range g.Adjacency {
+		newGraph.Adjacency[k] = v // copy
+	}
+	return newGraph
+}
+
+// GetName returns the name of the graph.
+func (g *Graph) GetName() string {
+	return g.Name
+}
+
+// SetName sets the name of the graph.
+func (g *Graph) SetName(name string) {
+	g.Name = name
+}
+
+// getState returns the state of the graph. This state is used for optimizing
+// certain algorithms by knowing what part of processing the graph is currently
+// undergoing.
+func (g *Graph) getState() graphState {
+	//g.mutex.Lock()
+	//defer g.mutex.Unlock()
+	return g.state
+}
+
+// setState sets the graph state and returns the previous state.
+func (g *Graph) setState(state graphState) graphState {
+	g.mutex.Lock()
+	defer g.mutex.Unlock()
+	prev := g.getState()
+	g.state = state
+	return prev
+}
+
+// AddVertex uses variadic input to add all listed vertices to the graph
+func (g *Graph) AddVertex(xv ...*Vertex) {
+	for _, v := range xv {
+		if _, exists := g.Adjacency[v]; !exists {
+			g.Adjacency[v] = make(map[*Vertex]*Edge)
+		}
+	}
+}
+
+// DeleteVertex deletes a particular vertex from the graph.
+func (g *Graph) DeleteVertex(v *Vertex) {
+	delete(g.Adjacency, v)
+	for k := range g.Adjacency {
+		delete(g.Adjacency[k], v)
+	}
+}
+
+// AddEdge adds a directed edge to the graph from v1 to v2.
+func (g *Graph) AddEdge(v1, v2 *Vertex, e *Edge) {
+	// NOTE: this doesn't allow more than one edge between two vertexes...
+	g.AddVertex(v1, v2) // supports adding N vertices now
+	// TODO: check if an edge exists to avoid overwriting it!
+	// NOTE: VertexMerge() depends on overwriting it at the moment...
+	g.Adjacency[v1][v2] = e
+}
+
+// DeleteEdge deletes a particular edge from the graph.
+// FIXME: add test cases
+func (g *Graph) DeleteEdge(e *Edge) {
+	for v1 := range g.Adjacency {
+		for v2, edge := range g.Adjacency[v1] {
+			if e == edge {
+				delete(g.Adjacency[v1], v2)
+			}
+		}
+	}
+}
+
+// GetVertexMatch searches for an equivalent resource in the graph and returns
+// the vertex it is found in, or nil if not found.
+func (g *Graph) GetVertexMatch(obj resources.Res) *Vertex {
+	for k := range g.Adjacency {
+		if k.Res.Compare(obj) {
+			return k
+		}
+	}
+	return nil
+}
+
+// HasVertex returns if the input vertex exists in the graph.
+func (g *Graph) HasVertex(v *Vertex) bool {
+	if _, exists := g.Adjacency[v]; exists {
+		return true
+	}
+	return false
+}
+
+// NumVertices returns the number of vertices in the graph.
+func (g *Graph) NumVertices() int {
+	return len(g.Adjacency)
+}
+
+// NumEdges returns the number of edges in the graph.
+func (g *Graph) NumEdges() int {
+	count := 0
+	for k := range g.Adjacency {
+		count += len(g.Adjacency[k])
+	}
+	return count
+}
+
+// GetVertices returns a randomly sorted slice of all vertices in the graph
+// The order is random, because the map implementation is intentionally so!
+func (g *Graph) GetVertices() []*Vertex {
+	var vertices []*Vertex
+	for k := range g.Adjacency {
+		vertices = append(vertices, k)
+	}
+	return vertices
+}
+
+// GetVerticesChan returns a channel of all vertices in the graph.
+func (g *Graph) GetVerticesChan() chan *Vertex {
+	ch := make(chan *Vertex)
+	go func(ch chan *Vertex) {
+		for k := range g.Adjacency {
+			ch <- k
+		}
+		close(ch)
+	}(ch)
+	return ch
+}
+
+// VertexSlice is a linear list of vertices. It can be sorted.
+type VertexSlice []*Vertex
+
+func (vs VertexSlice) Len() int           { return len(vs) }
+func (vs VertexSlice) Swap(i, j int)      { vs[i], vs[j] = vs[j], vs[i] }
+func (vs VertexSlice) Less(i, j int) bool { return vs[i].String() < vs[j].String() }
+
+// GetVerticesSorted returns a sorted slice of all vertices in the graph
+// The order is sorted by String() to avoid the non-determinism in the map type
+func (g *Graph) GetVerticesSorted() []*Vertex {
+	var vertices []*Vertex
+	for k := range g.Adjacency {
+		vertices = append(vertices, k)
+	}
+	sort.Sort(VertexSlice(vertices)) // add determinism
+	return vertices
+}
+
+// String makes the graph pretty print.
+func (g *Graph) String() string {
+	return fmt.Sprintf("Vertices(%d), Edges(%d)", g.NumVertices(), g.NumEdges())
+}
+
+// String returns the canonical form for a vertex
+func (v *Vertex) String() string {
+	return fmt.Sprintf("%s[%s]", v.Res.Kind(), v.Res.GetName())
+}
+
+// IncomingGraphVertices returns an array (slice) of all directed vertices to
+// vertex v (??? -> v). OKTimestamp should probably use this.
+func (g *Graph) IncomingGraphVertices(v *Vertex) []*Vertex {
+	// TODO: we might be able to implement this differently by reversing
+	// the Adjacency graph and then looping through it again...
+	var s []*Vertex
+	for k := range g.Adjacency { // reverse paths
+		for w := range g.Adjacency[k] {
+			if w == v {
+				s = append(s, k)
+			}
+		}
+	}
+	return s
+}
+
+// OutgoingGraphVertices returns an array (slice) of all vertices that vertex v
+// points to (v -> ???). Poke should probably use this.
+func (g *Graph) OutgoingGraphVertices(v *Vertex) []*Vertex {
+	var s []*Vertex
+	for k := range g.Adjacency[v] { // forward paths
+		s = append(s, k)
+	}
+	return s
+}
+
+// GraphVertices returns an array (slice) of all vertices that connect to vertex v.
+// This is the union of IncomingGraphVertices and OutgoingGraphVertices.
+func (g *Graph) GraphVertices(v *Vertex) []*Vertex {
+	var s []*Vertex
+	s = append(s, g.IncomingGraphVertices(v)...)
+	s = append(s, g.OutgoingGraphVertices(v)...)
+	return s
+}
+
+// IncomingGraphEdges returns all of the edges that point to vertex v (??? -> v).
+func (g *Graph) IncomingGraphEdges(v *Vertex) []*Edge {
+	var edges []*Edge
+	for v1 := range g.Adjacency { // reverse paths
+		for v2, e := range g.Adjacency[v1] {
+			if v2 == v {
+				edges = append(edges, e)
+			}
+		}
+	}
+	return edges
+}
+
+// OutgoingGraphEdges returns all of the edges that point from vertex v (v -> ???).
+func (g *Graph) OutgoingGraphEdges(v *Vertex) []*Edge {
+	var edges []*Edge
+	for _, e := range g.Adjacency[v] { // forward paths
+		edges = append(edges, e)
+	}
+	return edges
+}
+
+// GraphEdges returns an array (slice) of all edges that connect to vertex v.
+// This is the union of IncomingGraphEdges and OutgoingGraphEdges.
+func (g *Graph) GraphEdges(v *Vertex) []*Edge {
+	var edges []*Edge
+	edges = append(edges, g.IncomingGraphEdges(v)...)
+	edges = append(edges, g.OutgoingGraphEdges(v)...)
+	return edges
+}
+
+// DFS returns a depth first search for the graph, starting at the input vertex.
+func (g *Graph) DFS(start *Vertex) []*Vertex {
+	var d []*Vertex // discovered
+	var s []*Vertex // stack
+	if _, exists := g.Adjacency[start]; !exists {
+		return nil // TODO: error
+	}
+	v := start
+	s = append(s, v)
+	for len(s) > 0 {
+		v, s = s[len(s)-1], s[:len(s)-1] // s.pop()
+
+		if !VertexContains(v, d) { // if not discovered
+			d = append(d, v) // label as discovered
+
+			for _, w := range g.GraphVertices(v) {
+				s = append(s, w)
+			}
+		}
+	}
+	return d
+}
+
+// FilterGraph builds a new graph containing only vertices from the list.
+func (g *Graph) FilterGraph(name string, vertices []*Vertex) *Graph {
+	newgraph := NewGraph(name)
+	for k1, x := range g.Adjacency {
+		for k2, e := range x {
+			//log.Printf("Filter: %s -> %s # %s", k1.Name, k2.Name, e.Name)
+			if VertexContains(k1, vertices) || VertexContains(k2, vertices) {
+				newgraph.AddEdge(k1, k2, e)
+			}
+		}
+	}
+	return newgraph
+}
+
+// GetDisconnectedGraphs returns a channel containing the N disconnected graphs
+// in our main graph. We can then process each of these in parallel.
+func (g *Graph) GetDisconnectedGraphs() chan *Graph {
+	ch := make(chan *Graph)
+	go func() {
+		var start *Vertex
+		var d []*Vertex // discovered
+		c := g.NumVertices()
+		for len(d) < c {
+
+			// get an undiscovered vertex to start from
+			for _, s := range g.GetVertices() {
+				if !VertexContains(s, d) {
+					start = s
+				}
+			}
+
+			// dfs through the graph
+			dfs := g.DFS(start)
+			// filter all the collected elements into a new graph
+			newgraph := g.FilterGraph(g.Name, dfs)
+
+			// add number of elements found to found variable
+			d = append(d, dfs...) // extend
+
+			// return this new graph to the channel
+			ch <- newgraph
+
+			// if we've found all the elements, then we're done
+			// otherwise loop through to continue...
+		}
+		close(ch)
+	}()
+	return ch
+}
+
+// InDegree returns the count of vertices that point to me in one big lookup map.
+func (g *Graph) InDegree() map[*Vertex]int {
+	result := make(map[*Vertex]int)
+	for k := range g.Adjacency {
+		result[k] = 0 // initialize
+	}
+
+	for k := range g.Adjacency {
+		for z := range g.Adjacency[k] {
+			result[z]++
+		}
+	}
+	return result
+}
+
+// OutDegree returns the count of vertices that point away in one big lookup map.
+func (g *Graph) OutDegree() map[*Vertex]int {
+	result := make(map[*Vertex]int)
+
+	for k := range g.Adjacency {
+		result[k] = 0 // initialize
+		for range g.Adjacency[k] {
+			result[k]++
+		}
+	}
+	return result
+}
+
+// TopologicalSort returns the sort of graph vertices in that order.
+// based on descriptions and code from wikipedia and rosetta code
+// TODO: add memoization, and cache invalidation to speed this up :)
+func (g *Graph) TopologicalSort() ([]*Vertex, error) { // kahn's algorithm
+	var L []*Vertex                    // empty list that will contain the sorted elements
+	var S []*Vertex                    // set of all nodes with no incoming edges
+	remaining := make(map[*Vertex]int) // amount of edges remaining
+
+	for v, d := range g.InDegree() {
+		if d == 0 {
+			// accumulate set of all nodes with no incoming edges
+			S = append(S, v)
+		} else {
+			// initialize remaining edge count from indegree
+			remaining[v] = d
+		}
+	}
+
+	for len(S) > 0 {
+		last := len(S) - 1 // remove a node v from S
+		v := S[last]
+		S = S[:last]
+		L = append(L, v) // add v to tail of L
+		for n := range g.Adjacency[v] {
+			// for each node n remaining in the graph, consume from
+			// remaining, so for remaining[n] > 0
+			if remaining[n] > 0 {
+				remaining[n]--         // remove edge from the graph
+				if remaining[n] == 0 { // if n has no other incoming edges
+					S = append(S, n) // insert n into S
+				}
+			}
+		}
+	}
+
+	// if graph has edges, eg if any value in rem is > 0
+	for c, in := range remaining {
+		if in > 0 {
+			for n := range g.Adjacency[c] {
+				if remaining[n] > 0 {
+					return nil, fmt.Errorf("Not a dag!")
+				}
+			}
+		}
+	}
+
+	return L, nil
+}
+
+// Reachability finds the shortest path in a DAG from a to b, and returns the
+// slice of vertices that matched this particular path including both a and b.
+// It returns nil if a or b is nil, and returns empty list if no path is found.
+// Since there could be more than one possible result for this operation, we
+// arbitrarily choose one of the shortest possible. As a result, this should
+// actually return a tree if we cared about correctness.
+// This operates by a recursive algorithm; a more efficient version is likely.
+// If you don't give this function a DAG, you might cause infinite recursion!
+func (g *Graph) Reachability(a, b *Vertex) []*Vertex {
+	if a == nil || b == nil {
+		return nil
+	}
+	vertices := g.OutgoingGraphVertices(a) // what points away from a ?
+	if len(vertices) == 0 {
+		return []*Vertex{} // nope
+	}
+	if VertexContains(b, vertices) {
+		return []*Vertex{a, b} // found
+	}
+	// TODO: parallelize this with go routines?
+	var collected = make([][]*Vertex, len(vertices))
+	pick := -1
+	for i, v := range vertices {
+		collected[i] = g.Reachability(v, b) // find b by recursion
+		if l := len(collected[i]); l > 0 {
+			// pick shortest path
+			// TODO: technically i should return a tree
+			if pick < 0 || l < len(collected[pick]) {
+				pick = i
+			}
+		}
+	}
+	if pick < 0 {
+		return []*Vertex{} // nope
+	}
+	result := []*Vertex{a} // tack on a
+	result = append(result, collected[pick]...)
+	return result
+}
+
+// GraphSync updates the oldGraph so that it matches the newGraph receiver. It
+// leaves identical elements alone so that they don't need to be refreshed.
+// FIXME: add test cases
+func (g *Graph) GraphSync(oldGraph *Graph) (*Graph, error) {
+
+	if oldGraph == nil {
+		oldGraph = NewGraph(g.GetName()) // copy over the name
+	}
+	oldGraph.SetName(g.GetName()) // overwrite the name
+
+	var lookup = make(map[*Vertex]*Vertex)
+	var vertexKeep []*Vertex // list of vertices which are the same in new graph
+	var edgeKeep []*Edge     // list of vertices which are the same in new graph
+
+	for v := range g.Adjacency { // loop through the vertices (resources)
+		res := v.Res // resource
+
+		vertex := oldGraph.GetVertexMatch(res)
+		if vertex == nil { // no match found
+			if err := res.Init(); err != nil {
+				return nil, errwrap.Wrapf(err, "could not Init() resource")
+			}
+			vertex = NewVertex(res)
+			oldGraph.AddVertex(vertex) // call standalone in case not part of an edge
+		}
+		lookup[v] = vertex                      // used for constructing edges
+		vertexKeep = append(vertexKeep, vertex) // append
+	}
+
+	// get rid of any vertices we shouldn't keep (that aren't in new graph)
+	for v := range oldGraph.Adjacency {
+		if !VertexContains(v, vertexKeep) {
+			// wait for exit before starting new graph!
+			v.SendEvent(event.EventExit, true, false)
+			oldGraph.DeleteVertex(v)
+		}
+	}
+
+	// compare edges
+	for v1 := range g.Adjacency { // loop through the vertices (resources)
+		for v2, e := range g.Adjacency[v1] {
+			// we have an edge!
+
+			// lookup vertices (these should exist now)
+			//res1 := v1.Res // resource
+			//res2 := v2.Res
+			//vertex1 := oldGraph.GetVertexMatch(res1)
+			//vertex2 := oldGraph.GetVertexMatch(res2)
+			vertex1, exists1 := lookup[v1]
+			vertex2, exists2 := lookup[v2]
+			if !exists1 || !exists2 { // no match found, bug?
+				//if vertex1 == nil || vertex2 == nil { // no match found
+				return nil, fmt.Errorf("New vertices weren't found!") // programming error
+			}
+
+			edge, exists := oldGraph.Adjacency[vertex1][vertex2]
+			if !exists || edge.Name != e.Name { // TODO: edgeCmp
+				edge = e // use or overwrite edge
+			}
+			oldGraph.Adjacency[vertex1][vertex2] = edge // store it (AddEdge)
+			edgeKeep = append(edgeKeep, edge)           // mark as saved
+		}
+	}
+
+	// delete unused edges
+	for v1 := range oldGraph.Adjacency {
+		for _, e := range oldGraph.Adjacency[v1] {
+			// we have an edge!
+			if !EdgeContains(e, edgeKeep) {
+				oldGraph.DeleteEdge(e)
+			}
+		}
+	}
+
+	return oldGraph, nil
+}
+
+// GraphMetas returns a list of pointers to each of the resource MetaParams.
+func (g *Graph) GraphMetas() []*resources.MetaParams {
+	metas := []*resources.MetaParams{}
+	for v := range g.Adjacency { // loop through the vertices (resources))
+		res := v.Res // resource
+		meta := res.Meta()
+		metas = append(metas, meta)
+	}
+	return metas
+}
+
+// AssociateData associates some data with the object in the graph in question.
+func (g *Graph) AssociateData(data *resources.Data) {
+	for k := range g.Adjacency {
+		k.Res.AssociateData(data)
+	}
+}
+
+// VertexContains is an "in array" function to test for a vertex in a slice of vertices.
+func VertexContains(needle *Vertex, haystack []*Vertex) bool {
+	for _, v := range haystack {
+		if needle == v {
+			return true
+		}
+	}
+	return false
+}
+
+// EdgeContains is an "in array" function to test for an edge in a slice of edges.
+func EdgeContains(needle *Edge, haystack []*Edge) bool {
+	for _, v := range haystack {
+		if needle == v {
+			return true
+		}
+	}
+	return false
+}
+
+// Reverse reverses a list of vertices.
+func Reverse(vs []*Vertex) []*Vertex {
+	//var out []*Vertex       // XXX: golint suggests, but it fails testing
+	out := make([]*Vertex, 0) // empty list
+	l := len(vs)
+	for i := range vs {
+		out = append(out, vs[l-i-1])
+	}
+	return out
+}

pgraph/pgraph_test.go

@@ -15,9 +15,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-// NOTE: this is pgraph, a pointer graph
-
-package main
+package pgraph
 
 import (
 	"fmt"
@@ -25,8 +23,18 @@ import (
 	"sort"
 	"strings"
 	"testing"
+	"time"
 )
 
+// NV is a helper function to make testing easier. It creates a new noop vertex.
+func NV(s string) *Vertex {
+	obj, err := NewNoopRes(s)
+	if err != nil {
+		panic(err) // unlikely test failure!
+	}
+	return NewVertex(obj)
+}
+
 func TestPgraphT1(t *testing.T) {
 
 	G := NewGraph("g1")
@@ -39,8 +47,8 @@ func TestPgraphT1(t *testing.T) {
 		t.Errorf("Should have 0 edges instead of: %d.", i)
 	}
 
-	v1 := NewVertex(NewNoopRes("v1"))
-	v2 := NewVertex(NewNoopRes("v2"))
+	v1 := NV("v1")
+	v2 := NV("v2")
 	e1 := NewEdge("e1")
 	G.AddEdge(v1, v2, e1)
 
@@ -56,12 +64,12 @@ func TestPgraphT1(t *testing.T) {
 func TestPgraphT2(t *testing.T) {
 
 	G := NewGraph("g2")
-	v1 := NewVertex(NewNoopRes("v1"))
-	v2 := NewVertex(NewNoopRes("v2"))
-	v3 := NewVertex(NewNoopRes("v3"))
-	v4 := NewVertex(NewNoopRes("v4"))
-	v5 := NewVertex(NewNoopRes("v5"))
-	v6 := NewVertex(NewNoopRes("v6"))
+	v1 := NV("v1")
+	v2 := NV("v2")
+	v3 := NV("v3")
+	v4 := NV("v4")
+	v5 := NV("v5")
+	v6 := NV("v6")
 	e1 := NewEdge("e1")
 	e2 := NewEdge("e2")
 	e3 := NewEdge("e3")
@@ -83,12 +91,12 @@ func TestPgraphT2(t *testing.T) {
 func TestPgraphT3(t *testing.T) {
 
 	G := NewGraph("g3")
-	v1 := NewVertex(NewNoopRes("v1"))
-	v2 := NewVertex(NewNoopRes("v2"))
-	v3 := NewVertex(NewNoopRes("v3"))
-	v4 := NewVertex(NewNoopRes("v4"))
-	v5 := NewVertex(NewNoopRes("v5"))
-	v6 := NewVertex(NewNoopRes("v6"))
+	v1 := NV("v1")
+	v2 := NV("v2")
+	v3 := NV("v3")
+	v4 := NV("v4")
+	v5 := NV("v5")
+	v6 := NV("v6")
 	e1 := NewEdge("e1")
 	e2 := NewEdge("e2")
 	e3 := NewEdge("e3")
@@ -124,9 +132,9 @@ func TestPgraphT3(t *testing.T) {
 func TestPgraphT4(t *testing.T) {
 
 	G := NewGraph("g4")
-	v1 := NewVertex(NewNoopRes("v1"))
-	v2 := NewVertex(NewNoopRes("v2"))
-	v3 := NewVertex(NewNoopRes("v3"))
+	v1 := NV("v1")
+	v2 := NV("v2")
+	v3 := NV("v3")
 	e1 := NewEdge("e1")
 	e2 := NewEdge("e2")
 	e3 := NewEdge("e3")
@@ -146,12 +154,12 @@ func TestPgraphT4(t *testing.T) {
 
 func TestPgraphT5(t *testing.T) {
 	G := NewGraph("g5")
-	v1 := NewVertex(NewNoopRes("v1"))
-	v2 := NewVertex(NewNoopRes("v2"))
-	v3 := NewVertex(NewNoopRes("v3"))
-	v4 := NewVertex(NewNoopRes("v4"))
-	v5 := NewVertex(NewNoopRes("v5"))
-	v6 := NewVertex(NewNoopRes("v6"))
+	v1 := NV("v1")
+	v2 := NV("v2")
+	v3 := NV("v3")
+	v4 := NV("v4")
+	v5 := NV("v5")
+	v6 := NV("v6")
 	e1 := NewEdge("e1")
 	e2 := NewEdge("e2")
 	e3 := NewEdge("e3")
@@ -175,12 +183,12 @@ func TestPgraphT5(t *testing.T) {
 
 func TestPgraphT6(t *testing.T) {
 	G := NewGraph("g6")
-	v1 := NewVertex(NewNoopRes("v1"))
-	v2 := NewVertex(NewNoopRes("v2"))
-	v3 := NewVertex(NewNoopRes("v3"))
-	v4 := NewVertex(NewNoopRes("v4"))
-	v5 := NewVertex(NewNoopRes("v5"))
-	v6 := NewVertex(NewNoopRes("v6"))
+	v1 := NV("v1")
+	v2 := NV("v2")
+	v3 := NV("v3")
+	v4 := NV("v4")
+	v5 := NV("v5")
+	v6 := NV("v6")
 	e1 := NewEdge("e1")
 	e2 := NewEdge("e2")
 	e3 := NewEdge("e3")
@@ -213,9 +221,9 @@ func TestPgraphT6(t *testing.T) {
 func TestPgraphT7(t *testing.T) {
 
 	G := NewGraph("g7")
-	v1 := NewVertex(NewNoopRes("v1"))
-	v2 := NewVertex(NewNoopRes("v2"))
-	v3 := NewVertex(NewNoopRes("v3"))
+	v1 := NV("v1")
+	v2 := NV("v2")
+	v3 := NV("v3")
 	e1 := NewEdge("e1")
 	e2 := NewEdge("e2")
 	e3 := NewEdge("e3")
@@ -254,28 +262,28 @@ func TestPgraphT7(t *testing.T) {
 
 func TestPgraphT8(t *testing.T) {
 
-	v1 := NewVertex(NewNoopRes("v1"))
-	v2 := NewVertex(NewNoopRes("v2"))
-	v3 := NewVertex(NewNoopRes("v3"))
+	v1 := NV("v1")
+	v2 := NV("v2")
+	v3 := NV("v3")
 	if VertexContains(v1, []*Vertex{v1, v2, v3}) != true {
 		t.Errorf("Should be true instead of false.")
 	}
 
-	v4 := NewVertex(NewNoopRes("v4"))
-	v5 := NewVertex(NewNoopRes("v5"))
-	v6 := NewVertex(NewNoopRes("v6"))
+	v4 := NV("v4")
+	v5 := NV("v5")
+	v6 := NV("v6")
 	if VertexContains(v4, []*Vertex{v5, v6}) != false {
 		t.Errorf("Should be false instead of true.")
 	}
 
-	v7 := NewVertex(NewNoopRes("v7"))
-	v8 := NewVertex(NewNoopRes("v8"))
-	v9 := NewVertex(NewNoopRes("v9"))
+	v7 := NV("v7")
+	v8 := NV("v8")
+	v9 := NV("v9")
 	if VertexContains(v8, []*Vertex{v7, v8, v9}) != true {
 		t.Errorf("Should be true instead of false.")
 	}
 
-	v1b := NewVertex(NewNoopRes("v1")) // same value, different objects
+	v1b := NV("v1") // same value, different objects
 	if VertexContains(v1b, []*Vertex{v1, v2, v3}) != false {
 		t.Errorf("Should be false instead of true.")
 	}
@@ -284,12 +292,12 @@ func TestPgraphT8(t *testing.T) {
 func TestPgraphT9(t *testing.T) {
 
 	G := NewGraph("g9")
-	v1 := NewVertex(NewNoopRes("v1"))
-	v2 := NewVertex(NewNoopRes("v2"))
-	v3 := NewVertex(NewNoopRes("v3"))
-	v4 := NewVertex(NewNoopRes("v4"))
-	v5 := NewVertex(NewNoopRes("v5"))
-	v6 := NewVertex(NewNoopRes("v6"))
+	v1 := NV("v1")
+	v2 := NV("v2")
+	v3 := NV("v3")
+	v4 := NV("v4")
+	v5 := NV("v5")
+	v6 := NV("v6")
 	e1 := NewEdge("e1")
 	e2 := NewEdge("e2")
 	e3 := NewEdge("e3")
@@ -344,11 +352,11 @@ func TestPgraphT9(t *testing.T) {
 		t.Errorf("Outdegree of v6 should be 0 instead of: %d.", i)
 	}
 
-	s, ok := G.TopologicalSort()
+	s, err := G.TopologicalSort()
 	// either possibility is a valid toposort
 	match := reflect.DeepEqual(s, []*Vertex{v1, v2, v3, v4, v5, v6}) || reflect.DeepEqual(s, []*Vertex{v1, v3, v2, v4, v5, v6})
-	if !ok || !match {
-		t.Errorf("Topological sort failed, status: %v.", ok)
+	if err != nil || !match {
+		t.Errorf("Topological sort failed, error: %v.", err)
 		str := "Found:"
 		for _, v := range s {
 			str += " " + v.Res.GetName()
@@ -360,12 +368,12 @@ func TestPgraphT9(t *testing.T) {
 func TestPgraphT10(t *testing.T) {
 
 	G := NewGraph("g10")
-	v1 := NewVertex(NewNoopRes("v1"))
-	v2 := NewVertex(NewNoopRes("v2"))
-	v3 := NewVertex(NewNoopRes("v3"))
-	v4 := NewVertex(NewNoopRes("v4"))
-	v5 := NewVertex(NewNoopRes("v5"))
-	v6 := NewVertex(NewNoopRes("v6"))
+	v1 := NV("v1")
+	v2 := NV("v2")
+	v3 := NV("v3")
+	v4 := NV("v4")
+	v5 := NV("v5")
+	v6 := NV("v6")
 	e1 := NewEdge("e1")
 	e2 := NewEdge("e2")
 	e3 := NewEdge("e3")
@@ -379,8 +387,8 @@ func TestPgraphT10(t *testing.T) {
 	G.AddEdge(v5, v6, e5)
 	G.AddEdge(v4, v2, e6) // cycle
 
-	if _, ok := G.TopologicalSort(); ok {
-		t.Errorf("Topological sort passed, but graph is cyclic.")
+	if _, err := G.TopologicalSort(); err == nil {
+		t.Errorf("Topological sort passed, but graph is cyclic!")
 	}
 }
 
@@ -400,8 +408,8 @@ func TestPgraphReachability0(t *testing.T) {
 	}
 	{
 		G := NewGraph("g")
-		v1 := NewVertex(NewNoopRes("v1"))
-		v6 := NewVertex(NewNoopRes("v6"))
+		v1 := NV("v1")
+		v6 := NV("v6")
 
 		result := G.Reachability(v1, v6)
 		expected := []*Vertex{}
@@ -417,12 +425,12 @@ func TestPgraphReachability0(t *testing.T) {
 	}
 	{
 		G := NewGraph("g")
-		v1 := NewVertex(NewNoopRes("v1"))
-		v2 := NewVertex(NewNoopRes("v2"))
-		v3 := NewVertex(NewNoopRes("v3"))
-		v4 := NewVertex(NewNoopRes("v4"))
-		v5 := NewVertex(NewNoopRes("v5"))
-		v6 := NewVertex(NewNoopRes("v6"))
+		v1 := NV("v1")
+		v2 := NV("v2")
+		v3 := NV("v3")
+		v4 := NV("v4")
+		v5 := NV("v5")
+		v6 := NV("v6")
 		e1 := NewEdge("e1")
 		e2 := NewEdge("e2")
 		e3 := NewEdge("e3")
@@ -451,12 +459,12 @@ func TestPgraphReachability0(t *testing.T) {
 // simple linear path
 func TestPgraphReachability1(t *testing.T) {
 	G := NewGraph("g")
-	v1 := NewVertex(NewNoopRes("v1"))
-	v2 := NewVertex(NewNoopRes("v2"))
-	v3 := NewVertex(NewNoopRes("v3"))
-	v4 := NewVertex(NewNoopRes("v4"))
-	v5 := NewVertex(NewNoopRes("v5"))
-	v6 := NewVertex(NewNoopRes("v6"))
+	v1 := NV("v1")
+	v2 := NV("v2")
+	v3 := NV("v3")
+	v4 := NV("v4")
+	v5 := NV("v5")
+	v6 := NV("v6")
 	e1 := NewEdge("e1")
 	e2 := NewEdge("e2")
 	e3 := NewEdge("e3")
@@ -485,12 +493,12 @@ func TestPgraphReachability1(t *testing.T) {
 // pick one of two correct paths
 func TestPgraphReachability2(t *testing.T) {
 	G := NewGraph("g")
-	v1 := NewVertex(NewNoopRes("v1"))
-	v2 := NewVertex(NewNoopRes("v2"))
-	v3 := NewVertex(NewNoopRes("v3"))
-	v4 := NewVertex(NewNoopRes("v4"))
-	v5 := NewVertex(NewNoopRes("v5"))
-	v6 := NewVertex(NewNoopRes("v6"))
+	v1 := NV("v1")
+	v2 := NV("v2")
+	v3 := NV("v3")
+	v4 := NV("v4")
+	v5 := NV("v5")
+	v6 := NV("v6")
 	e1 := NewEdge("e1")
 	e2 := NewEdge("e2")
 	e3 := NewEdge("e3")
@@ -522,12 +530,12 @@ func TestPgraphReachability2(t *testing.T) {
 // pick shortest path
 func TestPgraphReachability3(t *testing.T) {
 	G := NewGraph("g")
-	v1 := NewVertex(NewNoopRes("v1"))
-	v2 := NewVertex(NewNoopRes("v2"))
-	v3 := NewVertex(NewNoopRes("v3"))
-	v4 := NewVertex(NewNoopRes("v4"))
-	v5 := NewVertex(NewNoopRes("v5"))
-	v6 := NewVertex(NewNoopRes("v6"))
+	v1 := NV("v1")
+	v2 := NV("v2")
+	v3 := NV("v3")
+	v4 := NV("v4")
+	v5 := NV("v5")
+	v6 := NV("v6")
 	e1 := NewEdge("e1")
 	e2 := NewEdge("e2")
 	e3 := NewEdge("e3")
@@ -557,12 +565,12 @@ func TestPgraphReachability3(t *testing.T) {
 // direct path
 func TestPgraphReachability4(t *testing.T) {
 	G := NewGraph("g")
-	v1 := NewVertex(NewNoopRes("v1"))
-	v2 := NewVertex(NewNoopRes("v2"))
-	v3 := NewVertex(NewNoopRes("v3"))
-	v4 := NewVertex(NewNoopRes("v4"))
-	v5 := NewVertex(NewNoopRes("v5"))
-	v6 := NewVertex(NewNoopRes("v6"))
+	v1 := NV("v1")
+	v2 := NV("v2")
+	v3 := NV("v3")
+	v4 := NV("v4")
+	v5 := NV("v5")
+	v6 := NV("v6")
 	e1 := NewEdge("e1")
 	e2 := NewEdge("e2")
 	e3 := NewEdge("e3")
@@ -590,12 +598,12 @@ func TestPgraphReachability4(t *testing.T) {
 }
 
 func TestPgraphT11(t *testing.T) {
-	v1 := NewVertex(NewNoopRes("v1"))
-	v2 := NewVertex(NewNoopRes("v2"))
-	v3 := NewVertex(NewNoopRes("v3"))
-	v4 := NewVertex(NewNoopRes("v4"))
-	v5 := NewVertex(NewNoopRes("v5"))
-	v6 := NewVertex(NewNoopRes("v6"))
+	v1 := NV("v1")
+	v2 := NV("v2")
+	v3 := NV("v3")
+	v4 := NV("v4")
+	v5 := NV("v5")
+	v6 := NV("v6")
 
 	if rev := Reverse([]*Vertex{}); !reflect.DeepEqual(rev, []*Vertex{}) {
 		t.Errorf("Reverse of vertex slice failed.")
@@ -638,7 +646,7 @@ func NewNoopResTest(name string) *NoopResTest {
 		NoopRes: NoopRes{
 			BaseRes: BaseRes{
 				Name: name,
-				Meta: MetaParams{
+				MetaParams: MetaParams{
 					AutoGroup: true, // always autogroup
 				},
 			},
@@ -807,7 +815,7 @@ func (g *Graph) fullPrint() (str string) {
 // helper function
 func runGraphCmp(t *testing.T, g1, g2 *Graph) {
 	ch := g1.autoGroup(&testGrouper{}) // edits the graph
-	for _ = range ch {                 // bleed the channel or it won't run :(
+	for range ch {                     // bleed the channel or it won't run :(
 		// pass
 	}
 	err := GraphCmp(g1, g2)
@@ -819,7 +827,7 @@ func runGraphCmp(t *testing.T, g1, g2 *Graph) {
 	}
 }
 
-// all of the following test cases are layed out with the following semantics:
+// all of the following test cases are laid out with the following semantics:
 // * vertices which start with the same single letter are considered "like"
 // * "like" elements should be merged
 // * vertices can have any integer after their single letter "family" type
@@ -1282,3 +1290,13 @@ func TestPgraphGroupingConnected1(t *testing.T) {
 	}
 	runGraphCmp(t, g1, g2)
 }
+
+func TestDurationAssumptions(t *testing.T) {
+	var d time.Duration
+	if (d == 0) != true {
+		t.Errorf("Empty time.Duration is no longer equal to zero!")
+	}
+	if (d > 0) != false {
+		t.Errorf("Empty time.Duration is now greater than zero!")
+	}
+}

puppet/gapi.go

@@ -0,0 +1,121 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package puppet
+
+import (
+	"fmt"
+	"log"
+	"sync"
+	"time"
+
+	"github.com/purpleidea/mgmt/gapi"
+	"github.com/purpleidea/mgmt/pgraph"
+)
+
+// GAPI implements the main puppet GAPI interface.
+type GAPI struct {
+	PuppetParam *string // puppet mode to run; nil if undefined
+	PuppetConf  string  // the path to an alternate puppet.conf file
+
+	data        gapi.Data
+	initialized bool
+	closeChan   chan struct{}
+	wg          sync.WaitGroup // sync group for tunnel go routines
+}
+
+// NewGAPI creates a new puppet GAPI struct and calls Init().
+func NewGAPI(data gapi.Data, puppetParam *string, puppetConf string) (*GAPI, error) {
+	obj := &GAPI{
+		PuppetParam: puppetParam,
+		PuppetConf:  puppetConf,
+	}
+	return obj, obj.Init(data)
+}
+
+// Init initializes the puppet GAPI struct.
+func (obj *GAPI) Init(data gapi.Data) error {
+	if obj.initialized {
+		return fmt.Errorf("Already initialized!")
+	}
+	if obj.PuppetParam == nil {
+		return fmt.Errorf("The PuppetParam param must be specified!")
+	}
+	obj.data = data // store for later
+	obj.closeChan = make(chan struct{})
+	obj.initialized = true
+	return nil
+}
+
+// Graph returns a current Graph.
+func (obj *GAPI) Graph() (*pgraph.Graph, error) {
+	if !obj.initialized {
+		return nil, fmt.Errorf("Puppet: GAPI is not initialized!")
+	}
+	config := ParseConfigFromPuppet(*obj.PuppetParam, obj.PuppetConf)
+	if config == nil {
+		return nil, fmt.Errorf("Puppet: ParseConfigFromPuppet returned nil!")
+	}
+	g, err := config.NewGraphFromConfig(obj.data.Hostname, obj.data.World, obj.data.Noop)
+	return g, err
+}
+
+// Next returns nil errors every time there could be a new graph.
+func (obj *GAPI) Next() chan error {
+	if obj.data.NoWatch {
+		return nil
+	}
+	puppetChan := func() <-chan time.Time { // helper function
+		return time.Tick(time.Duration(PuppetInterval(obj.PuppetConf)) * time.Second)
+	}
+	ch := make(chan error)
+	obj.wg.Add(1)
+	go func() {
+		defer obj.wg.Done()
+		defer close(ch) // this will run before the obj.wg.Done()
+		if !obj.initialized {
+			ch <- fmt.Errorf("Puppet: GAPI is not initialized!")
+			return
+		}
+		pChan := puppetChan()
+		for {
+			select {
+			case _, ok := <-pChan:
+				if !ok { // the channel closed!
+					return
+				}
+				log.Printf("Puppet: Generating new graph...")
+				pChan = puppetChan() // TODO: okay to update interval in case it changed?
+				ch <- nil            // trigger a run
+			case <-obj.closeChan:
+				return
+			}
+		}
+	}()
+	return ch
+}
+
+// Close shuts down the Puppet GAPI.
+func (obj *GAPI) Close() error {
+	if !obj.initialized {
+		return fmt.Errorf("Puppet: GAPI is not initialized!")
+	}
+	close(obj.closeChan)
+	obj.wg.Wait()
+	obj.initialized = false // closed = true
+	return nil
+}

puppet/puppet.go

@@ -0,0 +1,146 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// Package puppet provides the integration entrypoint for the puppet language.
+package puppet
+
+import (
+	"bufio"
+	"io"
+	"log"
+	"os/exec"
+	"strconv"
+	"strings"
+
+	"github.com/purpleidea/mgmt/yamlgraph"
+)
+
+const (
+	// PuppetYAMLBufferSize is the maximum buffer size for the yaml input data
+	PuppetYAMLBufferSize = 65535
+	// Debug is a local debug constant used in this module
+	Debug = false // FIXME: integrate with global debug flag
+)
+
+func runPuppetCommand(cmd *exec.Cmd) ([]byte, error) {
+	if Debug {
+		log.Printf("Puppet: running command: %v", cmd)
+	}
+
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		log.Printf("Puppet: Error opening pipe to puppet command: %v", err)
+		return nil, err
+	}
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		log.Printf("Puppet: Error opening error pipe to puppet command: %v", err)
+		return nil, err
+	}
+
+	if err := cmd.Start(); err != nil {
+		log.Printf("Puppet: Error starting puppet command: %v", err)
+		return nil, err
+	}
+
+	// XXX: the current implementation is likely prone to fail
+	// as soon as the YAML data overflows the buffer.
+	data := make([]byte, PuppetYAMLBufferSize)
+	var result []byte
+	for err == nil {
+		var count int
+		count, err = stdout.Read(data)
+		if err != nil && err != io.EOF {
+			log.Printf("Puppet: Error reading YAML data from puppet: %v", err)
+			return nil, err
+		}
+		// Slicing down to the number of actual bytes is important, the YAML parser
+		// will choke on an oversized slice. http://stackoverflow.com/a/33726617/3356612
+		result = append(result, data[0:count]...)
+	}
+	if Debug {
+		log.Printf("Puppet: read %v bytes of data from puppet", len(result))
+	}
+	for scanner := bufio.NewScanner(stderr); scanner.Scan(); {
+		log.Printf("Puppet: (output) %v", scanner.Text())
+	}
+	if err := cmd.Wait(); err != nil {
+		log.Printf("Puppet: Error: puppet command did not complete: %v", err)
+		return nil, err
+	}
+
+	return result, nil
+}
+
+// ParseConfigFromPuppet takes a special puppet param string and config and
+// returns the graph configuration structure.
+func ParseConfigFromPuppet(puppetParam, puppetConf string) *yamlgraph.GraphConfig {
+	var puppetConfArg string
+	if puppetConf != "" {
+		puppetConfArg = "--config=" + puppetConf
+	}
+
+	var cmd *exec.Cmd
+	if puppetParam == "agent" {
+		cmd = exec.Command("puppet", "mgmtgraph", "print", puppetConfArg)
+	} else if strings.HasSuffix(puppetParam, ".pp") {
+		cmd = exec.Command("puppet", "mgmtgraph", "print", puppetConfArg, "--manifest", puppetParam)
+	} else {
+		cmd = exec.Command("puppet", "mgmtgraph", "print", puppetConfArg, "--code", puppetParam)
+	}
+
+	log.Println("Puppet: launching translator")
+
+	var config yamlgraph.GraphConfig
+	if data, err := runPuppetCommand(cmd); err != nil {
+		return nil
+	} else if err := config.Parse(data); err != nil {
+		log.Printf("Puppet: Error: Could not parse YAML output with Parse: %v", err)
+		return nil
+	}
+
+	return &config
+}
+
+// PuppetInterval returns the graph refresh interval from the puppet configuration.
+func PuppetInterval(puppetConf string) int {
+	if Debug {
+		log.Printf("Puppet: determining graph refresh interval")
+	}
+	var cmd *exec.Cmd
+	if puppetConf != "" {
+		cmd = exec.Command("puppet", "config", "print", "runinterval", "--config", puppetConf)
+	} else {
+		cmd = exec.Command("puppet", "config", "print", "runinterval")
+	}
+
+	log.Println("Puppet: inspecting runinterval configuration")
+
+	interval := 1800
+	data, err := runPuppetCommand(cmd)
+	if err != nil {
+		log.Printf("Puppet: could not determine configured run interval (%v), using default of %v", err, interval)
+		return interval
+	}
+	result, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 0)
+	if err != nil {
+		log.Printf("Puppet: error reading numeric runinterval value (%v), using default of %v", err, interval)
+		return interval
+	}
+
+	return int(result)
+}

recwatch/configwatch.go

@@ -0,0 +1,135 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package recwatch
+
+import (
+	"log"
+	"sync"
+)
+
+// ConfigWatcher returns events on a channel anytime one of its files events.
+type ConfigWatcher struct {
+	Flags Flags
+
+	ch        chan string
+	wg        sync.WaitGroup
+	closechan chan struct{}
+	errorchan chan error
+}
+
+// NewConfigWatcher creates a new ConfigWatcher struct.
+func NewConfigWatcher() *ConfigWatcher {
+	return &ConfigWatcher{
+		ch:        make(chan string),
+		closechan: make(chan struct{}),
+		errorchan: make(chan error),
+	}
+}
+
+// Add new file paths to watch for events on.
+func (obj *ConfigWatcher) Add(file ...string) {
+	if len(file) == 0 {
+		return
+	}
+	if len(file) > 1 {
+		for _, f := range file { // add all the files...
+			obj.Add(f) // recurse
+		}
+		return
+	}
+	// otherwise, add the one file passed in...
+	obj.wg.Add(1)
+	go func() {
+		defer obj.wg.Done()
+		ch := obj.ConfigWatch(file[0])
+		for {
+			select {
+			case e := <-ch:
+				if e != nil {
+					obj.errorchan <- e
+					return
+				}
+				obj.ch <- file[0]
+				continue
+			case <-obj.closechan:
+				return
+			}
+		}
+	}()
+}
+
+// Error returns a channel of errors that notifies us of permanent issues.
+func (obj *ConfigWatcher) Error() <-chan error {
+	return obj.errorchan
+}
+
+// Events returns a channel to listen on for file events. It closes when it is
+// emptied after the Close() method is called. You can test for closure with the
+// f, more := <-obj.Events() pattern.
+func (obj *ConfigWatcher) Events() chan string {
+	return obj.ch
+}
+
+// Close shuts down the ConfigWatcher object. It closes the Events channel after
+// all the currently pending events have been emptied.
+func (obj *ConfigWatcher) Close() {
+	if obj.ch == nil {
+		return
+	}
+	close(obj.closechan)
+	obj.wg.Wait() // wait until everyone is done sending on obj.ch
+	//obj.ch <- "" // send finished message
+	close(obj.ch)
+	obj.ch = nil
+	close(obj.errorchan)
+}
+
+// ConfigWatch writes on the channel every time an event is seen for the path.
+func (obj *ConfigWatcher) ConfigWatch(file string) chan error {
+	ch := make(chan error)
+	go func() {
+		recWatcher, err := NewRecWatcher(file, false)
+		if err != nil {
+			ch <- err
+			close(ch)
+			return
+		}
+		recWatcher.Flags = obj.Flags
+		defer recWatcher.Close()
+		for {
+			if obj.Flags.Debug {
+				log.Printf("Watching: %v", file)
+			}
+			select {
+			case event, ok := <-recWatcher.Events():
+				if !ok { // channel is closed
+					close(ch)
+					return
+				}
+				if err := event.Error; err != nil {
+					ch <- err
+					close(ch)
+					return
+				}
+				ch <- nil // send event!
+			}
+		}
+		//close(ch)
+	}()
+	return ch
+}

recwatch/flags.go

@@ -0,0 +1,23 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package recwatch
+
+// Flags contains all the constant flags that recwatch needs.
+type Flags struct {
+	Debug bool
+}

recwatch/recwatch.go

@@ -0,0 +1,319 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// Package recwatch provides recursive file watching events via fsnotify.
+package recwatch
+
+import (
+	"fmt"
+	"log"
+	"math"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"sync"
+	"syscall"
+
+	"github.com/purpleidea/mgmt/util"
+
+	"gopkg.in/fsnotify.v1"
+	//"github.com/go-fsnotify/fsnotify" // git master of "gopkg.in/fsnotify.v1"
+)
+
+// Event represents a watcher event. These can include errors.
+type Event struct {
+	Error error
+	Body  *fsnotify.Event
+}
+
+// RecWatcher is the struct for the recursive watcher. Run Init() on it.
+type RecWatcher struct {
+	Path     string // computed path
+	Recurse  bool   // should we watch recursively?
+	Flags    Flags
+	isDir    bool   // computed isDir
+	safename string // safe path
+	watcher  *fsnotify.Watcher
+	watches  map[string]struct{}
+	events   chan Event // one channel for events and err...
+	once     sync.Once
+	wg       sync.WaitGroup
+	exit     chan struct{}
+	closeErr error
+}
+
+// NewRecWatcher creates an initializes a new recursive watcher.
+func NewRecWatcher(path string, recurse bool) (*RecWatcher, error) {
+	obj := &RecWatcher{
+		Path:    path,
+		Recurse: recurse,
+	}
+	return obj, obj.Init()
+}
+
+// Init starts the recursive file watcher.
+func (obj *RecWatcher) Init() error {
+	obj.watcher = nil
+	obj.watches = make(map[string]struct{})
+	obj.events = make(chan Event)
+	obj.exit = make(chan struct{})
+	obj.isDir = strings.HasSuffix(obj.Path, "/") // dirs have trailing slashes
+	obj.safename = path.Clean(obj.Path)          // no trailing slash
+
+	var err error
+	obj.watcher, err = fsnotify.NewWatcher()
+	if err != nil {
+		return err
+	}
+
+	if obj.isDir {
+		if err := obj.addSubFolders(obj.safename); err != nil {
+			return err
+		}
+	}
+
+	go func() {
+		if err := obj.Watch(); err != nil {
+			obj.events <- Event{Error: err}
+		}
+		obj.Close()
+	}()
+	return nil
+}
+
+//func (obj *RecWatcher) Add(path string) error { // XXX: implement me or not?
+//
+//}
+//
+//func (obj *RecWatcher) Remove(path string) error { // XXX: implement me or not?
+//
+//}
+
+// Close shuts down the watcher.
+func (obj *RecWatcher) Close() error {
+	obj.once.Do(obj.close) // don't cause the channel to close twice
+	return obj.closeErr
+}
+
+// This close function is the function that actually does the close work. Don't
+// call it more than once!
+func (obj *RecWatcher) close() {
+	var err error
+	close(obj.exit) // send exit signal
+	obj.wg.Wait()
+	if obj.watcher != nil {
+		err = obj.watcher.Close()
+		obj.watcher = nil
+		// TODO: should we send the close error?
+		//if err != nil {
+		//	obj.events <- Event{Error: err}
+		//}
+	}
+	close(obj.events)
+	obj.closeErr = err // set the error
+}
+
+// Events returns a channel of events. These include events for errors.
+func (obj *RecWatcher) Events() chan Event { return obj.events }
+
+// Watch is the primary listener for this resource and it outputs events.
+func (obj *RecWatcher) Watch() error {
+	if obj.watcher == nil {
+		return fmt.Errorf("Watcher is not initialized!")
+	}
+	obj.wg.Add(1)
+	defer obj.wg.Done()
+
+	patharray := util.PathSplit(obj.safename) // tokenize the path
+	var index = len(patharray)                // starting index
+	var current string                        // current "watcher" location
+	var deltaDepth int                        // depth delta between watcher and event
+	var send = false                          // send event?
+
+	for {
+		current = strings.Join(patharray[0:index], "/")
+		if current == "" { // the empty string top is the root dir ("/")
+			current = "/"
+		}
+		if obj.Flags.Debug {
+			log.Printf("Watching: %s", current) // attempting to watch...
+		}
+		// initialize in the loop so that we can reset on rm-ed handles
+		if err := obj.watcher.Add(current); err != nil {
+			if obj.Flags.Debug {
+				log.Printf("watcher.Add(%s): Error: %v", current, err)
+			}
+
+			if err == syscall.ENOENT {
+				index-- // usually not found, move up one dir
+				index = int(math.Max(1, float64(index)))
+				continue
+			}
+
+			if err == syscall.ENOSPC {
+				// no space left on device, out of inotify watches
+				// TODO: consider letting the user fall back to
+				// polling if they hit this error very often...
+				return fmt.Errorf("Out of inotify watches: %v", err)
+			} else if os.IsPermission(err) {
+				return fmt.Errorf("Permission denied adding a watch: %v", err)
+			}
+			return fmt.Errorf("Unknown error: %v", err)
+		}
+
+		select {
+		case event := <-obj.watcher.Events:
+			if obj.Flags.Debug {
+				log.Printf("Watch(%s), Event(%s): %v", current, event.Name, event.Op)
+			}
+			// the deeper you go, the bigger the deltaDepth is...
+			// this is the difference between what we're watching,
+			// and the event... doesn't mean we can't watch deeper
+			if current == event.Name {
+				deltaDepth = 0 // i was watching what i was looking for
+
+			} else if util.HasPathPrefix(event.Name, current) {
+				deltaDepth = len(util.PathSplit(current)) - len(util.PathSplit(event.Name)) // -1 or less
+
+			} else if util.HasPathPrefix(current, event.Name) {
+				deltaDepth = len(util.PathSplit(event.Name)) - len(util.PathSplit(current)) // +1 or more
+				// if below me...
+				if _, exists := obj.watches[event.Name]; exists {
+					send = true
+					if event.Op&fsnotify.Remove == fsnotify.Remove {
+						obj.watcher.Remove(event.Name)
+						delete(obj.watches, event.Name)
+					}
+					if (event.Op&fsnotify.Create == fsnotify.Create) && isDir(event.Name) {
+						obj.watcher.Add(event.Name)
+						obj.watches[event.Name] = struct{}{}
+						if err := obj.addSubFolders(event.Name); err != nil {
+							return err
+						}
+					}
+				}
+
+			} else {
+				// TODO: different watchers get each others events!
+				// https://github.com/go-fsnotify/fsnotify/issues/95
+				// this happened with two values such as:
+				// event.Name: /tmp/mgmt/f3 and current: /tmp/mgmt/f2
+				continue
+			}
+			//log.Printf("The delta depth is: %v", deltaDepth)
+
+			// if we have what we wanted, awesome, send an event...
+			if event.Name == obj.safename {
+				//log.Println("Event!")
+				// FIXME: should all these below cases trigger?
+				send = true
+
+				if obj.isDir {
+					if err := obj.addSubFolders(obj.safename); err != nil {
+						return err
+					}
+				}
+
+				// file removed, move the watch upwards
+				if deltaDepth >= 0 && (event.Op&fsnotify.Remove == fsnotify.Remove) {
+					//log.Println("Removal!")
+					obj.watcher.Remove(current)
+					index--
+				}
+
+				// we must be a parent watcher, so descend in
+				if deltaDepth < 0 {
+					// XXX: we can block here due to: https://github.com/fsnotify/fsnotify/issues/123
+					obj.watcher.Remove(current)
+					index++
+				}
+
+				// if safename starts with event.Name, we're above, and no event should be sent
+			} else if util.HasPathPrefix(obj.safename, event.Name) {
+				//log.Println("Above!")
+
+				if deltaDepth >= 0 && (event.Op&fsnotify.Remove == fsnotify.Remove) {
+					log.Println("Removal!")
+					obj.watcher.Remove(current)
+					index--
+				}
+
+				if deltaDepth < 0 {
+					log.Println("Parent!")
+					if util.PathPrefixDelta(obj.safename, event.Name) == 1 { // we're the parent dir
+						send = true
+					}
+					obj.watcher.Remove(current)
+					index++
+				}
+
+				// if event.Name startswith safename, send event, we're already deeper
+			} else if util.HasPathPrefix(event.Name, obj.safename) {
+				//log.Println("Event2!")
+				send = true
+			}
+
+			// do all our event sending all together to avoid duplicate msgs
+			if send {
+				send = false
+				// only invalid state on certain types of events
+				obj.events <- Event{Error: nil, Body: &event}
+			}
+
+		case err := <-obj.watcher.Errors:
+			return fmt.Errorf("Unknown watcher error: %v", err)
+
+		case <-obj.exit:
+			return nil
+		}
+	}
+}
+
+// addSubFolders is a helper that is used to add recursive dirs to the watches.
+func (obj *RecWatcher) addSubFolders(p string) error {
+	if !obj.Recurse {
+		return nil // if we're not watching recursively, just exit early
+	}
+	// look at all subfolders...
+	walkFn := func(path string, info os.FileInfo, err error) error {
+		if obj.Flags.Debug {
+			log.Printf("Walk: %s (%v): %v", path, info, err)
+		}
+		if err != nil {
+			return nil
+		}
+		if info.IsDir() {
+			obj.watches[path] = struct{}{} // add key
+			err := obj.watcher.Add(path)
+			if err != nil {
+				return err // TODO: will this bubble up?
+			}
+		}
+		return nil
+	}
+	err := filepath.Walk(p, walkFn)
+	return err
+}
+
+func isDir(path string) bool {
+	finfo, err := os.Stat(path)
+	if err != nil {
+		return false
+	}
+	return finfo.IsDir()
+}

resources.go

@@ -1,353 +0,0 @@
-// Mgmt
-// Copyright (C) 2013-2016+ James Shubin and the project contributors
-// Written by James Shubin <james@shubin.ca> and the project contributors
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package main
-
-import (
-	"bytes"
-	"encoding/base64"
-	"encoding/gob"
-	"fmt"
-	"log"
-)
-
-//go:generate stringer -type=resState -output=resstate_stringer.go
-type resState int
-
-const (
-	resStateNil resState = iota
-	resStateWatching
-	resStateEvent // an event has happened, but we haven't poked yet
-	resStateCheckApply
-	resStatePoking
-)
-
-//go:generate stringer -type=resConvergedState -output=resconvergedstate_stringer.go
-type resConvergedState int
-
-const (
-	resConvergedNil resConvergedState = iota
-	//resConverged
-	resConvergedTimeout
-)
-
-// a unique identifier for a resource, namely it's name, and the kind ("type")
-type ResUUID interface {
-	GetName() string
-	Kind() string
-	IFF(ResUUID) bool
-
-	Reversed() bool // true means this resource happens before the generator
-}
-
-type BaseUUID struct {
-	name string // name and kind are the values of where this is coming from
-	kind string
-
-	reversed *bool // piggyback edge information here
-}
-
-type AutoEdge interface {
-	Next() []ResUUID  // call to get list of edges to add
-	Test([]bool) bool // call until false
-}
-
-type MetaParams struct {
-	AutoEdge  bool `yaml:"autoedge"`  // metaparam, should we generate auto edges? // XXX should default to true
-	AutoGroup bool `yaml:"autogroup"` // metaparam, should we auto group? // XXX should default to true
-}
-
-// this interface is everything that is common to all resources
-// everything here only needs to be implemented once, in the BaseRes
-type Base interface {
-	GetName() string // can't be named "Name()" because of struct field
-	SetName(string)
-	Kind() string
-	GetMeta() MetaParams
-	SetVertex(*Vertex)
-	SetConvergedCallback(ctimeout int, converged chan bool)
-	IsWatching() bool
-	SetWatching(bool)
-	GetConvergedState() resConvergedState
-	SetConvergedState(resConvergedState)
-	GetState() resState
-	SetState(resState)
-	SendEvent(eventName, bool, bool) bool
-	ReadEvent(*Event) (bool, bool) // TODO: optional here?
-	GroupCmp(Res) bool             // TODO: is there a better name for this?
-	GroupRes(Res) error            // group resource (arg) into self
-	IsGrouped() bool               // am I grouped?
-	SetGrouped(bool)               // set grouped bool
-	GetGroup() []Res               // return everyone grouped inside me
-	SetGroup([]Res)
-}
-
-// this is the minimum interface you need to implement to make a new resource
-type Res interface {
-	Base // include everything from the Base interface
-	Init()
-	//Validate() bool    // TODO: this might one day be added
-	GetUUIDs() []ResUUID // most resources only return one
-	Watch(chan Event)    // send on channel to signal process() events
-	CheckApply(bool) (bool, error)
-	AutoEdges() AutoEdge
-	Compare(Res) bool
-	CollectPattern(string) // XXX: temporary until Res collection is more advanced
-}
-
-type BaseRes struct {
-	Name           string     `yaml:"name"`
-	Meta           MetaParams `yaml:"meta"` // struct of all the metaparams
-	kind           string
-	events         chan Event
-	vertex         *Vertex
-	state          resState
-	convergedState resConvergedState
-	watching       bool // is Watch() loop running ?
-	ctimeout       int  // converged timeout
-	converged      chan bool
-	isStateOK      bool  // whether the state is okay based on events or not
-	isGrouped      bool  // am i contained within a group?
-	grouped        []Res // list of any grouped resources
-}
-
-// wraps the IFF method when used with a list of UUID's
-func UUIDExistsInUUIDs(uuid ResUUID, uuids []ResUUID) bool {
-	for _, u := range uuids {
-		if uuid.IFF(u) {
-			return true
-		}
-	}
-	return false
-}
-
-func (obj *BaseUUID) GetName() string {
-	return obj.name
-}
-
-func (obj *BaseUUID) Kind() string {
-	return obj.kind
-}
-
-// if and only if they are equivalent, return true
-// if they are not equivalent, return false
-// most resource will want to override this method, since it does the important
-// work of actually discerning if two resources are identical in function
-func (obj *BaseUUID) IFF(uuid ResUUID) bool {
-	res, ok := uuid.(*BaseUUID)
-	if !ok {
-		return false
-	}
-	return obj.name == res.name
-}
-
-func (obj *BaseUUID) Reversed() bool {
-	if obj.reversed == nil {
-		log.Fatal("Programming error!")
-	}
-	return *obj.reversed
-}
-
-// initialize structures like channels if created without New constructor
-func (obj *BaseRes) Init() {
-	obj.events = make(chan Event) // unbuffered chan size to avoid stale events
-}
-
-// this method gets used by all the resources
-func (obj *BaseRes) GetName() string {
-	return obj.Name
-}
-
-func (obj *BaseRes) SetName(name string) {
-	obj.Name = name
-}
-
-// return the kind of resource this is
-func (obj *BaseRes) Kind() string {
-	return obj.kind
-}
-
-func (obj *BaseRes) GetMeta() MetaParams {
-	return obj.Meta
-}
-
-func (obj *BaseRes) GetVertex() *Vertex {
-	return obj.vertex
-}
-
-func (obj *BaseRes) SetVertex(v *Vertex) {
-	obj.vertex = v
-}
-
-func (obj *BaseRes) SetConvergedCallback(ctimeout int, converged chan bool) {
-	obj.ctimeout = ctimeout
-	obj.converged = converged
-}
-
-// is the Watch() function running?
-func (obj *BaseRes) IsWatching() bool {
-	return obj.watching
-}
-
-// store status of if the Watch() function is running
-func (obj *BaseRes) SetWatching(b bool) {
-	obj.watching = b
-}
-
-func (obj *BaseRes) GetConvergedState() resConvergedState {
-	return obj.convergedState
-}
-
-func (obj *BaseRes) SetConvergedState(state resConvergedState) {
-	obj.convergedState = state
-}
-
-func (obj *BaseRes) GetState() resState {
-	return obj.state
-}
-
-func (obj *BaseRes) SetState(state resState) {
-	if DEBUG {
-		log.Printf("%v[%v]: State: %v -> %v", obj.Kind(), obj.GetName(), obj.GetState(), state)
-	}
-	obj.state = state
-}
-
-// push an event into the message queue for a particular vertex
-func (obj *BaseRes) SendEvent(event eventName, sync bool, activity bool) bool {
-	// TODO: isn't this race-y ?
-	if !obj.IsWatching() { // element has already exited
-		return false // if we don't return, we'll block on the send
-	}
-	if !sync {
-		obj.events <- Event{event, nil, "", activity}
-		return true
-	}
-
-	resp := make(chan bool)
-	obj.events <- Event{event, resp, "", activity}
-	for {
-		value := <-resp
-		// wait until true value
-		if value {
-			return true
-		}
-	}
-}
-
-// process events when a select gets one, this handles the pause code too!
-// the return values specify if we should exit and poke respectively
-func (obj *BaseRes) ReadEvent(event *Event) (exit, poke bool) {
-	event.ACK()
-	switch event.Name {
-	case eventStart:
-		return false, true
-
-	case eventPoke:
-		return false, true
-
-	case eventBackPoke:
-		return false, true // forward poking in response to a back poke!
-
-	case eventExit:
-		return true, false
-
-	case eventPause:
-		// wait for next event to continue
-		select {
-		case e := <-obj.events:
-			e.ACK()
-			if e.Name == eventExit {
-				return true, false
-			} else if e.Name == eventStart { // eventContinue
-				return false, false // don't poke on unpause!
-			} else {
-				// if we get a poke event here, it's a bug!
-				log.Fatalf("%v[%v]: Unknown event: %v, while paused!", obj.Kind(), obj.GetName(), e)
-			}
-		}
-
-	default:
-		log.Fatal("Unknown event: ", event)
-	}
-	return true, false // required to keep the stupid go compiler happy
-}
-
-func (obj *BaseRes) GroupRes(res Res) error {
-	if l := len(res.GetGroup()); l > 0 {
-		return fmt.Errorf("Res: %v already contains %d grouped resources!", res, l)
-	}
-	if res.IsGrouped() {
-		return fmt.Errorf("Res: %v is already grouped!", res)
-	}
-
-	obj.grouped = append(obj.grouped, res)
-	res.SetGrouped(true) // i am contained _in_ a group
-	return nil
-}
-
-func (obj *BaseRes) IsGrouped() bool { // am I grouped?
-	return obj.isGrouped
-}
-
-func (obj *BaseRes) SetGrouped(b bool) {
-	obj.isGrouped = b
-}
-
-func (obj *BaseRes) GetGroup() []Res { // return everyone grouped inside me
-	return obj.grouped
-}
-
-func (obj *BaseRes) SetGroup(g []Res) {
-	obj.grouped = g
-}
-
-func (obj *BaseRes) CollectPattern(pattern string) {
-	// XXX: default method is empty
-}
-
-// ResToB64 encodes a resource to a base64 encoded string (after serialization)
-func ResToB64(res Res) (string, error) {
-	b := bytes.Buffer{}
-	e := gob.NewEncoder(&b)
-	err := e.Encode(&res) // pass with &
-	if err != nil {
-		return "", fmt.Errorf("Gob failed to encode: %v", err)
-	}
-	return base64.StdEncoding.EncodeToString(b.Bytes()), nil
-}
-
-// B64ToRes decodes a resource from a base64 encoded string (after deserialization)
-func B64ToRes(str string) (Res, error) {
-	var output interface{}
-	bb, err := base64.StdEncoding.DecodeString(str)
-	if err != nil {
-		return nil, fmt.Errorf("Base64 failed to decode: %v", err)
-	}
-	b := bytes.NewBuffer(bb)
-	d := gob.NewDecoder(b)
-	err = d.Decode(&output) // pass with &
-	if err != nil {
-		return nil, fmt.Errorf("Gob failed to decode: %v", err)
-	}
-	res, ok := output.(Res)
-	if !ok {
-		return nil, fmt.Errorf("Output %v is not a Res", res)
-
-	}
-	return res, nil
-}

resources/exec.go

@@ -15,22 +15,28 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package main
+package resources
 
 import (
 	"bufio"
 	"bytes"
 	"encoding/gob"
-	"errors"
+	"fmt"
 	"log"
 	"os/exec"
 	"strings"
+
+	"github.com/purpleidea/mgmt/event"
+	"github.com/purpleidea/mgmt/util"
+
+	errwrap "github.com/pkg/errors"
 )
 
 func init() {
 	gob.Register(&ExecRes{})
 }
 
+// ExecRes is an exec resource for running commands.
 type ExecRes struct {
 	BaseRes    `yaml:",inline"`
 	State      string `yaml:"state"`      // state: exists/present?, absent, (undefined?)
@@ -44,7 +50,8 @@ type ExecRes struct {
 	PollInt    int    `yaml:"pollint"`    // the poll interval for the ifcmd
 }
 
-func NewExecRes(name, cmd, shell string, timeout int, watchcmd, watchshell, ifcmd, ifshell string, pollint int, state string) *ExecRes {
+// NewExecRes is a constructor for this resource. It also calls Init() for you.
+func NewExecRes(name, cmd, shell string, timeout int, watchcmd, watchshell, ifcmd, ifshell string, pollint int, state string) (*ExecRes, error) {
 	obj := &ExecRes{
 		BaseRes: BaseRes{
 			Name: name,
@@ -59,31 +66,31 @@ func NewExecRes(name, cmd, shell string, timeout int, watchcmd, watchshell, ifcm
 		PollInt:    pollint,
 		State:      state,
 	}
-	obj.Init()
-	return obj
+	return obj, obj.Init()
 }
 
-func (obj *ExecRes) Init() {
+// Init runs some startup code for this resource.
+func (obj *ExecRes) Init() error {
 	obj.BaseRes.kind = "Exec"
-	obj.BaseRes.Init() // call base init, b/c we're overriding
+	return obj.BaseRes.Init() // call base init, b/c we're overriding
 }
 
-// validate if the params passed in are valid data
+// Validate if the params passed in are valid data.
 // FIXME: where should this get called ?
-func (obj *ExecRes) Validate() bool {
+func (obj *ExecRes) Validate() error {
 	if obj.Cmd == "" { // this is the only thing that is really required
-		return false
+		return fmt.Errorf("Command can't be empty!")
 	}
 
 	// if we have a watch command, then we don't poll with the if command!
 	if obj.WatchCmd != "" && obj.PollInt > 0 {
-		return false
+		return fmt.Errorf("Don't poll when we have a watch command.")
 	}
 
-	return true
+	return nil
 }
 
-// wraps the scanner output in a channel
+// BufioChanScanner wraps the scanner output in a channel.
 func (obj *ExecRes) BufioChanScanner(scanner *bufio.Scanner) (chan string, chan error) {
 	ch, errch := make(chan string), make(chan error)
 	go func() {
@@ -91,7 +98,7 @@ func (obj *ExecRes) BufioChanScanner(scanner *bufio.Scanner) (chan string, chan
 			ch <- scanner.Text() // blocks here ?
 			if e := scanner.Err(); e != nil {
 				errch <- e // send any misc errors we encounter
-				//break // TODO ?
+				//break // TODO: ?
 			}
 		}
 		close(ch)
@@ -101,18 +108,13 @@ func (obj *ExecRes) BufioChanScanner(scanner *bufio.Scanner) (chan string, chan
 	return ch, errch
 }
 
-// Exec watcher
-func (obj *ExecRes) Watch(processChan chan Event) {
-	if obj.IsWatching() {
-		return
-	}
-	obj.SetWatching(true)
-	defer obj.SetWatching(false)
+// Watch is the primary listener for this resource and it outputs events.
+func (obj *ExecRes) Watch(processChan chan event.Event) error {
+	cuid := obj.Converger() // get the converger uid used to report status
 
 	var send = false // send event?
 	var exit = false
 	bufioch, errch := make(chan string), make(chan error)
-	//vertex := obj.GetVertex()         // stored with SetVertex
 
 	if obj.WatchCmd != "" {
 		var cmdName string
@@ -124,7 +126,7 @@ func (obj *ExecRes) Watch(processChan chan Event) {
 			cmdName = split[0]
 			//d, _ := os.Getwd() // TODO: how does this ever error ?
 			//cmdName = path.Join(d, cmdName)
-			cmdArgs = split[1:len(split)]
+			cmdArgs = split[1:]
 		} else {
 			cmdName = obj.Shell // usually bash, or sh
 			cmdArgs = []string{"-c", obj.WatchCmd}
@@ -134,8 +136,7 @@ func (obj *ExecRes) Watch(processChan chan Event) {
 
 		cmdReader, err := cmd.StdoutPipe()
 		if err != nil {
-			log.Printf("%v[%v]: Error creating StdoutPipe for Cmd: %v", obj.Kind(), obj.GetName(), err)
-			log.Fatal(err) // XXX: how should we handle errors?
+			return errwrap.Wrapf(err, "Error creating StdoutPipe for Cmd")
 		}
 		scanner := bufio.NewScanner(cmdReader)
 
@@ -146,45 +147,46 @@ func (obj *ExecRes) Watch(processChan chan Event) {
 			cmd.Process.Kill() // TODO: is this necessary?
 		}()
 		if err := cmd.Start(); err != nil {
-			log.Printf("%v[%v]: Error starting Cmd: %v", obj.Kind(), obj.GetName(), err)
-			log.Fatal(err) // XXX: how should we handle errors?
+			return errwrap.Wrapf(err, "Error starting Cmd")
 		}
 
 		bufioch, errch = obj.BufioChanScanner(scanner)
 	}
 
+	// notify engine that we're running
+	if err := obj.Running(processChan); err != nil {
+		return err // bubble up a NACK...
+	}
+
 	for {
-		obj.SetState(resStateWatching) // reset
+		obj.SetState(ResStateWatching) // reset
 		select {
 		case text := <-bufioch:
-			obj.SetConvergedState(resConvergedNil)
+			cuid.SetConverged(false)
 			// each time we get a line of output, we loop!
-			log.Printf("%v[%v]: Watch output: %s", obj.Kind(), obj.GetName(), text)
+			log.Printf("%s[%s]: Watch output: %s", obj.Kind(), obj.GetName(), text)
 			if text != "" {
 				send = true
 			}
 
 		case err := <-errch:
-			obj.SetConvergedState(resConvergedNil) // XXX ?
+			cuid.SetConverged(false)
 			if err == nil { // EOF
 				// FIXME: add an "if watch command ends/crashes"
 				// restart or generate error option
-				log.Printf("%v[%v]: Reached EOF", obj.Kind(), obj.GetName())
-				return
+				return fmt.Errorf("Reached EOF")
 			}
-			log.Printf("%v[%v]: Error reading input?: %v", obj.Kind(), obj.GetName(), err)
-			log.Fatal(err)
-			// XXX: how should we handle errors?
+			// error reading input?
+			return errwrap.Wrapf(err, "Unknown error")
 
-		case event := <-obj.events:
-			obj.SetConvergedState(resConvergedNil)
+		case event := <-obj.Events():
+			cuid.SetConverged(false)
 			if exit, send = obj.ReadEvent(&event); exit {
-				return // exit
+				return nil // exit
 			}
 
-		case _ = <-TimeAfterOrBlock(obj.ctimeout):
-			obj.SetConvergedState(resConvergedTimeout)
-			obj.converged <- true
+		case <-cuid.ConvergedTimer():
+			cuid.SetConverged(true) // converged!
 			continue
 		}
 
@@ -192,21 +194,22 @@ func (obj *ExecRes) Watch(processChan chan Event) {
 		if send {
 			send = false
 			// it is okay to invalidate the clean state on poke too
-			obj.isStateOK = false // something made state dirty
-			resp := NewResp()
-			processChan <- Event{eventNil, resp, "", true} // trigger process
-			resp.ACKWait()                                 // wait for the ACK()
+			obj.StateOK(false) // something made state dirty
+			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
+				return err // we exit or bubble up a NACK...
+			}
 		}
 	}
 }
 
+// CheckApply checks the resource state and applies the resource if the bool
+// input is true. It returns error info and if the state check passed or not.
 // TODO: expand the IfCmd to be a list of commands
-func (obj *ExecRes) CheckApply(apply bool) (stateok bool, err error) {
-	log.Printf("%v[%v]: CheckApply(%t)", obj.Kind(), obj.GetName(), apply)
+func (obj *ExecRes) CheckApply(apply bool) (checkOK bool, err error) {
 
 	// if there is a watch command, but no if command, run based on state
 	if obj.WatchCmd != "" && obj.IfCmd == "" {
-		if obj.isStateOK {
+		if obj.IsStateOK() { // FIXME: this is done by engine now...
 			return true, nil
 		}
 
@@ -216,8 +219,8 @@ func (obj *ExecRes) CheckApply(apply bool) (stateok bool, err error) {
 		//} else if obj.IfCmd != "" && obj.WatchCmd != "" {
 
 		if obj.PollInt > 0 { // && obj.WatchCmd == ""
-			// XXX have the Watch() command output onlyif poll events...
-			// XXX we can optimize by saving those results for returning here
+			// XXX: have the Watch() command output onlyif poll events...
+			// XXX: we can optimize by saving those results for returning here
 			// return XXX
 		}
 
@@ -230,7 +233,7 @@ func (obj *ExecRes) CheckApply(apply bool) (stateok bool, err error) {
 			cmdName = split[0]
 			//d, _ := os.Getwd() // TODO: how does this ever error ?
 			//cmdName = path.Join(d, cmdName)
-			cmdArgs = split[1:len(split)]
+			cmdArgs = split[1:]
 		} else {
 			cmdName = obj.IfShell // usually bash, or sh
 			cmdArgs = []string{"-c", obj.IfCmd}
@@ -244,7 +247,7 @@ func (obj *ExecRes) CheckApply(apply bool) (stateok bool, err error) {
 		// if there is no watcher and no onlyif check, assume we should run
 	} else { // if obj.WatchCmd == "" && obj.IfCmd == "" {
 		// just run if state is dirty
-		if obj.isStateOK {
+		if obj.IsStateOK() { // FIXME: this is done by engine now...
 			return true, nil
 		}
 	}
@@ -255,7 +258,7 @@ func (obj *ExecRes) CheckApply(apply bool) (stateok bool, err error) {
 	}
 
 	// apply portion
-	log.Printf("%v[%v]: Apply", obj.Kind(), obj.GetName())
+	log.Printf("%s[%s]: Apply", obj.Kind(), obj.GetName())
 	var cmdName string
 	var cmdArgs []string
 	if obj.Shell == "" {
@@ -266,7 +269,7 @@ func (obj *ExecRes) CheckApply(apply bool) (stateok bool, err error) {
 		cmdName = split[0]
 		//d, _ := os.Getwd() // TODO: how does this ever error ?
 		//cmdName = path.Join(d, cmdName)
-		cmdArgs = split[1:len(split)]
+		cmdArgs = split[1:]
 	} else {
 		cmdName = obj.Shell // usually bash, or sh
 		cmdArgs = []string{"-c", obj.Cmd}
@@ -276,9 +279,8 @@ func (obj *ExecRes) CheckApply(apply bool) (stateok bool, err error) {
 	var out bytes.Buffer
 	cmd.Stdout = &out
 
-	if err = cmd.Start(); err != nil {
-		log.Printf("%v[%v]: Error starting Cmd: %v", obj.Kind(), obj.GetName(), err)
-		return false, err
+	if err := cmd.Start(); err != nil {
+		return false, errwrap.Wrapf(err, "Error starting Cmd")
 	}
 
 	timeout := obj.Timeout
@@ -289,48 +291,48 @@ func (obj *ExecRes) CheckApply(apply bool) (stateok bool, err error) {
 	go func() { done <- cmd.Wait() }()
 
 	select {
-	case err = <-done:
+	case err := <-done:
 		if err != nil {
-			log.Printf("%v[%v]: Error waiting for Cmd: %v", obj.Kind(), obj.GetName(), err)
-			return false, err
+			e := errwrap.Wrapf(err, "Error waiting for Cmd")
+			return false, e
 		}
 
-	case <-TimeAfterOrBlock(timeout):
-		log.Printf("%v[%v]: Timeout waiting for Cmd", obj.Kind(), obj.GetName())
+	case <-util.TimeAfterOrBlock(timeout):
 		//cmd.Process.Kill() // TODO: is this necessary?
-		return false, errors.New("Timeout waiting for Cmd!")
+		return false, fmt.Errorf("Timeout waiting for Cmd!")
 	}
 
 	// TODO: if we printed the stdout while the command is running, this
 	// would be nice, but it would require terminal log output that doesn't
 	// interleave all the parallel parts which would mix it all up...
 	if s := out.String(); s == "" {
-		log.Printf("Exec[%v]: Command output is empty!", obj.Name)
+		log.Printf("%s[%s]: Command output is empty!", obj.Kind(), obj.GetName())
+
 	} else {
-		log.Printf("Exec[%v]: Command output is:", obj.Name)
+		log.Printf("%s[%s]: Command output is:", obj.Kind(), obj.GetName())
 		log.Printf(out.String())
 	}
 	// XXX: return based on exit value!!
 
-	// the state tracking is for exec resources that can't "detect" their
+	// The state tracking is for exec resources that can't "detect" their
 	// state, and assume it's invalid when the Watch() function triggers.
-	// if we apply state successfully, we should reset it here so that we
+	// If we apply state successfully, we should reset it here so that we
 	// know that we have applied since the state was set not ok by event!
-	obj.isStateOK = true // reset
+	// This now happens automatically after the engine runs CheckApply().
 	return false, nil // success
 }
 
-type ExecUUID struct {
-	BaseUUID
+// ExecUID is the UID struct for ExecRes.
+type ExecUID struct {
+	BaseUID
 	Cmd   string
 	IfCmd string
 	// TODO: add more elements here
 }
 
-// if and only if they are equivalent, return true
-// if they are not equivalent, return false
-func (obj *ExecUUID) IFF(uuid ResUUID) bool {
-	res, ok := uuid.(*ExecUUID)
+// IFF aka if and only if they are equivalent, return true. If not, false.
+func (obj *ExecUID) IFF(uid ResUID) bool {
+	res, ok := uid.(*ExecUID)
 	if !ok {
 		return false
 	}
@@ -362,35 +364,43 @@ func (obj *ExecUUID) IFF(uuid ResUUID) bool {
 	return true
 }
 
+// AutoEdges returns the AutoEdge interface. In this case no autoedges are used.
 func (obj *ExecRes) AutoEdges() AutoEdge {
 	// TODO: parse as many exec params to look for auto edges, for example
 	// the path of the binary in the Cmd variable might be from in a pkg
 	return nil
 }
 
-// include all params to make a unique identification of this object
-func (obj *ExecRes) GetUUIDs() []ResUUID {
-	x := &ExecUUID{
-		BaseUUID: BaseUUID{name: obj.GetName(), kind: obj.Kind()},
+// GetUIDs includes all params to make a unique identification of this object.
+// Most resources only return one, although some resources can return multiple.
+func (obj *ExecRes) GetUIDs() []ResUID {
+	x := &ExecUID{
+		BaseUID: BaseUID{name: obj.GetName(), kind: obj.Kind()},
 		Cmd:     obj.Cmd,
 		IfCmd:   obj.IfCmd,
 		// TODO: add more params here
 	}
-	return []ResUUID{x}
+	return []ResUID{x}
 }
 
+// GroupCmp returns whether two resources can be grouped together or not.
 func (obj *ExecRes) GroupCmp(r Res) bool {
-	_, ok := r.(*SvcRes)
+	_, ok := r.(*ExecRes)
 	if !ok {
 		return false
 	}
 	return false // not possible atm
 }
 
+// Compare two resources and return if they are equivalent.
 func (obj *ExecRes) Compare(res Res) bool {
 	switch res.(type) {
 	case *ExecRes:
 		res := res.(*ExecRes)
+		if !obj.BaseRes.Compare(res) { // call base Compare
+			return false
+		}
+
 		if obj.Name != res.Name {
 			return false
 		}

resources/file.go

@@ -0,0 +1,806 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/gob"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/purpleidea/mgmt/event"
+	"github.com/purpleidea/mgmt/recwatch"
+	"github.com/purpleidea/mgmt/util"
+
+	errwrap "github.com/pkg/errors"
+)
+
+func init() {
+	gob.Register(&FileRes{})
+}
+
+// FileRes is a file and directory resource.
+type FileRes struct {
+	BaseRes    `yaml:",inline"`
+	Path       string  `yaml:"path"` // path variable (should default to name)
+	Dirname    string  `yaml:"dirname"`
+	Basename   string  `yaml:"basename"`
+	Content    *string `yaml:"content"` // nil to mark as undefined
+	Source     string  `yaml:"source"`  // file path for source content
+	State      string  `yaml:"state"`   // state: exists/present?, absent, (undefined?)
+	Recurse    bool    `yaml:"recurse"`
+	Force      bool    `yaml:"force"`
+	path       string  // computed path
+	isDir      bool    // computed isDir
+	sha256sum  string
+	recWatcher *recwatch.RecWatcher
+}
+
+// NewFileRes is a constructor for this resource. It also calls Init() for you.
+func NewFileRes(name, path, dirname, basename string, content *string, source, state string, recurse, force bool) (*FileRes, error) {
+	obj := &FileRes{
+		BaseRes: BaseRes{
+			Name: name,
+		},
+		Path:     path,
+		Dirname:  dirname,
+		Basename: basename,
+		Content:  content,
+		Source:   source,
+		State:    state,
+		Recurse:  recurse,
+		Force:    force,
+	}
+	return obj, obj.Init()
+}
+
+// Init runs some startup code for this resource.
+func (obj *FileRes) Init() error {
+	obj.sha256sum = ""
+	if obj.Path == "" { // use the name as the path default if missing
+		obj.Path = obj.BaseRes.Name
+	}
+	obj.path = obj.GetPath()                     // compute once
+	obj.isDir = strings.HasSuffix(obj.path, "/") // dirs have trailing slashes
+
+	obj.BaseRes.kind = "File"
+	return obj.BaseRes.Init() // call base init, b/c we're overriding
+}
+
+// GetPath returns the actual path to use for this resource. It computes this
+// after analysis of the Path, Dirname and Basename values. Dirs end with slash.
+func (obj *FileRes) GetPath() string {
+	d := util.Dirname(obj.Path)
+	b := util.Basename(obj.Path)
+	if obj.Dirname == "" && obj.Basename == "" {
+		return obj.Path
+	}
+	if obj.Dirname == "" {
+		return d + obj.Basename
+	}
+	if obj.Basename == "" {
+		return obj.Dirname + b
+	}
+	// if obj.dirname != "" && obj.basename != ""
+	return obj.Dirname + obj.Basename
+}
+
+// Validate reports any problems with the struct definition.
+func (obj *FileRes) Validate() error {
+	if obj.Dirname != "" && !strings.HasSuffix(obj.Dirname, "/") {
+		return fmt.Errorf("Dirname must end with a slash.")
+	}
+
+	if strings.HasPrefix(obj.Basename, "/") {
+		return fmt.Errorf("Basename must not start with a slash.")
+	}
+
+	if obj.Content != nil && obj.Source != "" {
+		return fmt.Errorf("Can't specify both Content and Source.")
+	}
+
+	if obj.isDir && obj.Content != nil { // makes no sense
+		return fmt.Errorf("Can't specify Content when creating a Dir.")
+	}
+
+	// XXX: should this specify that we create an empty directory instead?
+	//if obj.Source == "" && obj.isDir {
+	//	return fmt.Errorf("Can't specify an empty source when creating a Dir.")
+	//}
+
+	return nil
+}
+
+// Watch is the primary listener for this resource and it outputs events.
+// This one is a file watcher for files and directories.
+// Modify with caution, it is probably important to write some test cases first!
+// If the Watch returns an error, it means that something has gone wrong, and it
+// must be restarted. On a clean exit it returns nil.
+// FIXME: Also watch the source directory when using obj.Source !!!
+func (obj *FileRes) Watch(processChan chan event.Event) error {
+	cuid := obj.Converger() // get the converger uid used to report status
+
+	var err error
+	obj.recWatcher, err = recwatch.NewRecWatcher(obj.Path, obj.Recurse)
+	if err != nil {
+		return err
+	}
+	defer obj.recWatcher.Close()
+
+	// notify engine that we're running
+	if err := obj.Running(processChan); err != nil {
+		return err // bubble up a NACK...
+	}
+
+	var send = false // send event?
+	var exit = false
+
+	for {
+		if obj.debug {
+			log.Printf("%s[%s]: Watching: %s", obj.Kind(), obj.GetName(), obj.Path) // attempting to watch...
+		}
+
+		obj.SetState(ResStateWatching) // reset
+		select {
+		case event, ok := <-obj.recWatcher.Events():
+			if !ok { // channel shutdown
+				return nil
+			}
+			cuid.SetConverged(false)
+			if err := event.Error; err != nil {
+				return errwrap.Wrapf(err, "Unknown %s[%s] watcher error", obj.Kind(), obj.GetName())
+			}
+			if obj.debug { // don't access event.Body if event.Error isn't nil
+				log.Printf("%s[%s]: Event(%s): %v", obj.Kind(), obj.GetName(), event.Body.Name, event.Body.Op)
+			}
+			send = true
+			obj.StateOK(false) // dirty
+
+		case event := <-obj.Events():
+			cuid.SetConverged(false)
+			if exit, send = obj.ReadEvent(&event); exit {
+				return nil // exit
+			}
+			//obj.StateOK(false) // dirty // these events don't invalidate state
+
+		case <-cuid.ConvergedTimer():
+			cuid.SetConverged(true) // converged!
+			continue
+		}
+
+		// do all our event sending all together to avoid duplicate msgs
+		if send {
+			send = false
+			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
+				return err // we exit or bubble up a NACK...
+			}
+		}
+	}
+}
+
+// smartPath adds a trailing slash to the path if it is a directory.
+func smartPath(fileInfo os.FileInfo) string {
+	smartPath := fileInfo.Name() // absolute path
+	if fileInfo.IsDir() {
+		smartPath += "/" // add a trailing slash for dirs
+	}
+	return smartPath
+}
+
+// FileInfo is an enhanced variant of the traditional os.FileInfo struct. It can
+// store both the absolute and the relative paths (when built from our ReadDir),
+// and those two paths contain a trailing slash when they refer to a directory.
+type FileInfo struct {
+	os.FileInfo        // embed
+	AbsPath     string // smart variant
+	RelPath     string // smart variant
+}
+
+// ReadDir reads a directory path, and returns a list of enhanced FileInfo's.
+func ReadDir(path string) ([]FileInfo, error) {
+	if !strings.HasSuffix(path, "/") { // dirs have trailing slashes
+		return nil, fmt.Errorf("Path must be a directory.")
+	}
+	output := []FileInfo{} // my file info
+	fileInfos, err := ioutil.ReadDir(path)
+	if os.IsNotExist(err) {
+		return output, err // return empty list
+	}
+	if err != nil {
+		return nil, err
+	}
+	for _, fi := range fileInfos {
+		abs := path + smartPath(fi)
+		rel, err := filepath.Rel(path, abs) // NOTE: calls Clean()
+		if err != nil {                     // shouldn't happen
+			return nil, errwrap.Wrapf(err, "ReadDir: Unhandled error")
+		}
+		if fi.IsDir() {
+			rel += "/" // add a trailing slash for dirs
+		}
+		x := FileInfo{
+			FileInfo: fi,
+			AbsPath:  abs,
+			RelPath:  rel,
+		}
+		output = append(output, x)
+	}
+	return output, nil
+}
+
+// smartMapPaths adds a trailing slash to every path that is a directory. It
+// returns the data as a map where the keys are the smart paths and where the
+// values are the original os.FileInfo entries.
+func mapPaths(fileInfos []FileInfo) map[string]FileInfo {
+	paths := make(map[string]FileInfo)
+	for _, fileInfo := range fileInfos {
+		paths[fileInfo.RelPath] = fileInfo
+	}
+	return paths
+}
+
+// fileCheckApply is the CheckApply operation for a source and destination file.
+// It can accept an io.Reader as the source, which can be a regular file, or it
+// can be a bytes Buffer struct. It can take an input sha256 hash to use instead
+// of computing the source data hash, and it returns the computed value if this
+// function reaches that stage. As usual, it respects the apply action variable,
+// and it symmetry with the main CheckApply function returns checkOK and error.
+func (obj *FileRes) fileCheckApply(apply bool, src io.ReadSeeker, dst string, sha256sum string) (string, bool, error) {
+	// TODO: does it make sense to switch dst to an io.Writer ?
+	// TODO: use obj.Force when dealing with symlinks and other file types!
+	if obj.debug {
+		log.Printf("fileCheckApply: %s -> %s", src, dst)
+	}
+
+	srcFile, isFile := src.(*os.File)
+	_, isBytes := src.(*bytes.Reader) // supports seeking!
+	if !isFile && !isBytes {
+		return "", false, fmt.Errorf("Can't open src as either file or buffer!")
+	}
+
+	var srcStat os.FileInfo
+	if isFile {
+		var err error
+		srcStat, err = srcFile.Stat()
+		if err != nil {
+			return "", false, err
+		}
+		// TODO: deal with symlinks
+		if !srcStat.Mode().IsRegular() { // can't copy non-regular files or dirs
+			return "", false, fmt.Errorf("Non-regular src file: %s (%q)", srcStat.Name(), srcStat.Mode())
+		}
+	}
+
+	dstFile, err := os.Open(dst)
+	if err != nil && !os.IsNotExist(err) { // ignore ErrNotExist errors
+		return "", false, err
+	}
+	dstClose := func() error {
+		return dstFile.Close() // calling this twice is safe :)
+	}
+	defer dstClose()
+	dstExists := !os.IsNotExist(err)
+
+	dstStat, err := dstFile.Stat()
+	if err != nil && dstExists {
+		return "", false, err
+	}
+
+	if dstExists && dstStat.IsDir() { // oops, dst is a dir, and we want a file...
+		if !apply {
+			return "", false, nil
+		}
+		if !obj.Force {
+			return "", false, fmt.Errorf("Can't force dir into file: %s", dst)
+		}
+
+		cleanDst := path.Clean(dst)
+		if cleanDst == "" || cleanDst == "/" {
+			return "", false, fmt.Errorf("Don't want to remove root!") // safety
+		}
+		// FIXME: respect obj.Recurse here...
+		// there is a dir here, where we want a file...
+		log.Printf("fileCheckApply: Removing (force): %s", cleanDst)
+		if err := os.RemoveAll(cleanDst); err != nil { // dangerous ;)
+			return "", false, err
+		}
+		dstExists = false // now it's gone!
+
+	} else if err == nil {
+		if !dstStat.Mode().IsRegular() {
+			return "", false, fmt.Errorf("Non-regular dst file: %s (%q)", dstStat.Name(), dstStat.Mode())
+		}
+		if isFile && os.SameFile(srcStat, dstStat) { // same inode, we're done!
+			return "", true, nil
+		}
+	}
+
+	if dstExists { // if dst doesn't exist, no need to compare hashes
+		// hash comparison (efficient because we can cache hash of content str)
+		if sha256sum == "" { // cache is invalid
+			hash := sha256.New()
+			// TODO: file existence test?
+			if _, err := io.Copy(hash, src); err != nil {
+				return "", false, err
+			}
+			sha256sum = hex.EncodeToString(hash.Sum(nil))
+			// since we re-use this src handler below, it is
+			// *critical* to seek to 0, or we'll copy nothing!
+			if n, err := src.Seek(0, 0); err != nil || n != 0 {
+				return sha256sum, false, err
+			}
+		}
+
+		// dst hash
+		hash := sha256.New()
+		if _, err := io.Copy(hash, dstFile); err != nil {
+			return "", false, err
+		}
+		if h := hex.EncodeToString(hash.Sum(nil)); h == sha256sum {
+			return sha256sum, true, nil // same!
+		}
+	}
+
+	// state is not okay, no work done, exit, but without error
+	if !apply {
+		return sha256sum, false, nil
+	}
+	if obj.debug {
+		log.Printf("fileCheckApply: Apply: %s -> %s", src, dst)
+	}
+
+	dstClose() // unlock file usage so we can write to it
+	dstFile, err = os.Create(dst)
+	if err != nil {
+		return sha256sum, false, err
+	}
+	defer dstFile.Close() // TODO: is this redundant because of the earlier defered Close() ?
+
+	if isFile { // set mode because it's a new file
+		if err := dstFile.Chmod(srcStat.Mode()); err != nil {
+			return sha256sum, false, err
+		}
+	}
+
+	// TODO: attempt to reflink with Splice() and int(file.Fd()) as input...
+	// syscall.Splice(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int64, err error)
+
+	// TODO: should we offer a way to cancel the copy on ^C ?
+	if obj.debug {
+		log.Printf("fileCheckApply: Copy: %s -> %s", src, dst)
+	}
+	if n, err := io.Copy(dstFile, src); err != nil {
+		return sha256sum, false, err
+	} else if obj.debug {
+		log.Printf("fileCheckApply: Copied: %v", n)
+	}
+	return sha256sum, false, dstFile.Sync()
+}
+
+// syncCheckApply is the CheckApply operation for a source and destination dir.
+// It is recursive and can create directories directly, and files via the usual
+// fileCheckApply method. It returns checkOK and error as is normally expected.
+func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
+	if obj.debug {
+		log.Printf("syncCheckApply: %s -> %s", src, dst)
+	}
+	if src == "" || dst == "" {
+		return false, fmt.Errorf("The src and dst must not be empty!")
+	}
+
+	var checkOK = true
+	// TODO: handle ./ cases or ../ cases that need cleaning ?
+
+	srcIsDir := strings.HasSuffix(src, "/")
+	dstIsDir := strings.HasSuffix(dst, "/")
+
+	if srcIsDir != dstIsDir {
+		return false, fmt.Errorf("The src and dst must be both either files or directories.")
+	}
+
+	if !srcIsDir && !dstIsDir {
+		if obj.debug {
+			log.Printf("syncCheckApply: %s -> %s", src, dst)
+		}
+		fin, err := os.Open(src)
+		if err != nil {
+			if obj.debug && os.IsNotExist(err) { // if we get passed an empty src
+				log.Printf("syncCheckApply: Missing src: %s", src)
+			}
+			return false, err
+		}
+
+		_, checkOK, err := obj.fileCheckApply(apply, fin, dst, "")
+		if err != nil {
+			fin.Close()
+			return false, err
+		}
+		return checkOK, fin.Close()
+	}
+
+	// else: if srcIsDir && dstIsDir
+	srcFiles, err := ReadDir(src)          // if src does not exist...
+	if err != nil && !os.IsNotExist(err) { // an empty map comes out below!
+		return false, err
+	}
+	dstFiles, err := ReadDir(dst)
+	if err != nil && !os.IsNotExist(err) {
+		return false, err
+	}
+	//log.Printf("syncCheckApply: srcFiles: %v", srcFiles)
+	//log.Printf("syncCheckApply: dstFiles: %v", dstFiles)
+	smartSrc := mapPaths(srcFiles)
+	smartDst := mapPaths(dstFiles)
+
+	for relPath, fileInfo := range smartSrc {
+		absSrc := fileInfo.AbsPath // absolute path
+		absDst := dst + relPath    // absolute dest
+
+		if _, exists := smartDst[relPath]; !exists {
+			if fileInfo.IsDir() {
+				if !apply { // only checking and not identical!
+					return false, nil
+				}
+
+				// file exists, but we want a dir: we need force
+				// we check for the file w/o the smart dir slash
+				relPathFile := strings.TrimSuffix(relPath, "/")
+				if _, ok := smartDst[relPathFile]; ok {
+					absCleanDst := path.Clean(absDst)
+					if !obj.Force {
+						return false, fmt.Errorf("Can't force file into dir: %s", absCleanDst)
+					}
+					if absCleanDst == "" || absCleanDst == "/" {
+						return false, fmt.Errorf("Don't want to remove root!") // safety
+					}
+					log.Printf("syncCheckApply: Removing (force): %s", absCleanDst)
+					if err := os.Remove(absCleanDst); err != nil {
+						return false, err
+					}
+					delete(smartDst, relPathFile) // rm from purge list
+				}
+
+				if obj.debug {
+					log.Printf("syncCheckApply: mkdir -m %s '%s'", fileInfo.Mode(), absDst)
+				}
+				if err := os.Mkdir(absDst, fileInfo.Mode()); err != nil {
+					return false, err
+				}
+				checkOK = false // we did some work
+			}
+			// if we're a regular file, the recurse will create it
+		}
+
+		if obj.debug {
+			log.Printf("syncCheckApply: Recurse: %s -> %s", absSrc, absDst)
+		}
+		if obj.Recurse {
+			if c, err := obj.syncCheckApply(apply, absSrc, absDst); err != nil { // recurse
+				return false, errwrap.Wrapf(err, "syncCheckApply: Recurse failed")
+			} else if !c { // don't let subsequent passes make this true
+				checkOK = false
+			}
+		}
+		if !apply && !checkOK { // check failed, and no apply to do, so exit!
+			return false, nil
+		}
+		delete(smartDst, relPath) // rm from purge list
+	}
+
+	if !apply && len(smartDst) > 0 { // we know there are files to remove!
+		return false, nil // so just exit now
+	}
+	// any files that now remain in smartDst need to be removed...
+	for relPath, fileInfo := range smartDst {
+		absSrc := src + relPath    // absolute dest (should not exist!)
+		absDst := fileInfo.AbsPath // absolute path (should get removed)
+		absCleanDst := path.Clean(absDst)
+		if absCleanDst == "" || absCleanDst == "/" {
+			return false, fmt.Errorf("Don't want to remove root!") // safety
+		}
+
+		// FIXME: respect obj.Recurse here...
+
+		// NOTE: we could use os.RemoveAll instead of recursing, but I
+		// think the symmetry is more elegant and correct here for now
+		// Avoiding this is also useful if we had a recurse limit arg!
+		if true { // switch
+			log.Printf("syncCheckApply: Removing: %s", absCleanDst)
+			if apply {
+				if err := os.RemoveAll(absCleanDst); err != nil { // dangerous ;)
+					return false, err
+				}
+				checkOK = false
+			}
+			continue
+		}
+		_ = absSrc
+		//log.Printf("syncCheckApply: Recurse rm: %s -> %s", absSrc, absDst)
+		//if c, err := obj.syncCheckApply(apply, absSrc, absDst); err != nil {
+		//	return false, errwrap.Wrapf(err, "syncCheckApply: Recurse rm failed")
+		//} else if !c { // don't let subsequent passes make this true
+		//	checkOK = false
+		//}
+		//log.Printf("syncCheckApply: Removing: %s", absCleanDst)
+		//if apply { // safety
+		//	if err := os.Remove(absCleanDst); err != nil {
+		//		return false, err
+		//	}
+		//	checkOK = false
+		//}
+	}
+
+	return checkOK, nil
+}
+
+// contentCheckApply performs a CheckApply for the file existence and content.
+func (obj *FileRes) contentCheckApply(apply bool) (checkOK bool, _ error) {
+	log.Printf("%s[%s]: contentCheckApply(%t)", obj.Kind(), obj.GetName(), apply)
+
+	if obj.State == "absent" {
+		if _, err := os.Stat(obj.path); os.IsNotExist(err) {
+			// no such file or directory, but
+			// file should be missing, phew :)
+			return true, nil
+
+		} else if err != nil { // what could this error be?
+			return false, err
+		}
+
+		// state is not okay, no work done, exit, but without error
+		if !apply {
+			return false, nil
+		}
+
+		// apply portion
+		if obj.path == "" || obj.path == "/" {
+			return false, fmt.Errorf("Don't want to remove root!") // safety
+		}
+		log.Printf("contentCheckApply: Removing: %s", obj.path)
+		// FIXME: respect obj.Recurse here...
+		// TODO: add recurse limit here
+		err := os.RemoveAll(obj.path) // dangerous ;)
+		return false, err             // either nil or not
+	}
+
+	// content is not defined, leave it alone...
+	if obj.Content == nil {
+		return true, nil
+	}
+
+	if obj.Source == "" { // do the obj.Content checks first...
+		if obj.isDir { // TODO: should we create an empty dir this way?
+			log.Fatal("XXX: Not implemented!") // XXX
+		}
+
+		bufferSrc := bytes.NewReader([]byte(*obj.Content))
+		sha256sum, checkOK, err := obj.fileCheckApply(apply, bufferSrc, obj.path, obj.sha256sum)
+		if sha256sum != "" { // empty values mean errored or didn't hash
+			// this can be valid even when the whole function errors
+			obj.sha256sum = sha256sum // cache value
+		}
+		if err != nil {
+			return false, err
+		}
+		// if no err, but !ok, then...
+		return checkOK, nil // success
+	}
+
+	checkOK, err := obj.syncCheckApply(apply, obj.Source, obj.path)
+	if err != nil {
+		log.Printf("syncCheckApply: Error: %v", err)
+		return false, err
+	}
+
+	return checkOK, nil
+}
+
+// CheckApply checks the resource state and applies the resource if the bool
+// input is true. It returns error info and if the state check passed or not.
+func (obj *FileRes) CheckApply(apply bool) (checkOK bool, _ error) {
+
+	// NOTE: all send/recv change notifications *must* be processed before
+	// there is a possibility of failure in CheckApply. This is because if
+	// we fail (and possibly run again) the subsequent send->recv transfer
+	// might not have a new value to copy, and therefore we won't see this
+	// notification of change. Therefore, it is important to process these
+	// promptly, if they must not be lost, such as for cache invalidation.
+	if val, exists := obj.Recv["Content"]; exists && val.Changed {
+		// if we received on Content, and it changed, invalidate the cache!
+		log.Printf("contentCheckApply: Invalidating sha256sum of `Content`")
+		obj.sha256sum = "" // invalidate!!
+	}
+
+	checkOK = true
+
+	if c, err := obj.contentCheckApply(apply); err != nil {
+		return false, err
+	} else if !c {
+		checkOK = false
+	}
+
+	// TODO
+	//if c, err := obj.chmodCheckApply(apply); err != nil {
+	//	return false, err
+	//} else if !c {
+	//	checkOK = false
+	//}
+
+	// TODO
+	//if c, err := obj.chownCheckApply(apply); err != nil {
+	//	return false, err
+	//} else if !c {
+	//	checkOK = false
+	//}
+
+	return checkOK, nil // w00t
+}
+
+// FileUID is the UID struct for FileRes.
+type FileUID struct {
+	BaseUID
+	path string
+}
+
+// IFF aka if and only if they are equivalent, return true. If not, false.
+func (obj *FileUID) IFF(uid ResUID) bool {
+	res, ok := uid.(*FileUID)
+	if !ok {
+		return false
+	}
+	return obj.path == res.path
+}
+
+// FileResAutoEdges holds the state of the auto edge generator.
+type FileResAutoEdges struct {
+	data    []ResUID
+	pointer int
+	found   bool
+}
+
+// Next returns the next automatic edge.
+func (obj *FileResAutoEdges) Next() []ResUID {
+	if obj.found {
+		log.Fatal("Shouldn't be called anymore!")
+	}
+	if len(obj.data) == 0 { // check length for rare scenarios
+		return nil
+	}
+	value := obj.data[obj.pointer]
+	obj.pointer++
+	return []ResUID{value} // we return one, even though api supports N
+}
+
+// Test gets results of the earlier Next() call, & returns if we should continue!
+func (obj *FileResAutoEdges) Test(input []bool) bool {
+	// if there aren't any more remaining
+	if len(obj.data) <= obj.pointer {
+		return false
+	}
+	if obj.found { // already found, done!
+		return false
+	}
+	if len(input) != 1 { // in case we get given bad data
+		log.Fatal("Expecting a single value!")
+	}
+	if input[0] { // if a match is found, we're done!
+		obj.found = true // no more to find!
+		return false
+	}
+	return true // keep going
+}
+
+// AutoEdges generates a simple linear sequence of each parent directory from
+// the bottom up!
+func (obj *FileRes) AutoEdges() AutoEdge {
+	var data []ResUID                              // store linear result chain here...
+	values := util.PathSplitFullReversed(obj.path) // build it
+	_, values = values[0], values[1:]              // get rid of first value which is me!
+	for _, x := range values {
+		var reversed = true // cheat by passing a pointer
+		data = append(data, &FileUID{
+			BaseUID: BaseUID{
+				name:     obj.GetName(),
+				kind:     obj.Kind(),
+				reversed: &reversed,
+			},
+			path: x, // what matters
+		}) // build list
+	}
+	return &FileResAutoEdges{
+		data:    data,
+		pointer: 0,
+		found:   false,
+	}
+}
+
+// GetUIDs includes all params to make a unique identification of this object.
+// Most resources only return one, although some resources can return multiple.
+func (obj *FileRes) GetUIDs() []ResUID {
+	x := &FileUID{
+		BaseUID: BaseUID{name: obj.GetName(), kind: obj.Kind()},
+		path:    obj.path,
+	}
+	return []ResUID{x}
+}
+
+// GroupCmp returns whether two resources can be grouped together or not.
+func (obj *FileRes) GroupCmp(r Res) bool {
+	_, ok := r.(*FileRes)
+	if !ok {
+		return false
+	}
+	// TODO: we might be able to group directory children into a single
+	// recursive watcher in the future, thus saving fanotify watches
+	return false // not possible atm
+}
+
+// Compare two resources and return if they are equivalent.
+func (obj *FileRes) Compare(res Res) bool {
+	switch res.(type) {
+	case *FileRes:
+		res := res.(*FileRes)
+		if !obj.BaseRes.Compare(res) { // call base Compare
+			return false
+		}
+
+		if obj.Name != res.Name {
+			return false
+		}
+		if obj.path != res.path {
+			return false
+		}
+		if (obj.Content == nil) != (res.Content == nil) { // xor
+			return false
+		}
+		if obj.Content != nil && res.Content != nil {
+			if *obj.Content != *res.Content { // compare the strings
+				return false
+			}
+		}
+		if obj.Source != res.Source {
+			return false
+		}
+		if obj.State != res.State {
+			return false
+		}
+		if obj.Recurse != res.Recurse {
+			return false
+		}
+		if obj.Force != res.Force {
+			return false
+		}
+	default:
+		return false
+	}
+	return true
+}
+
+// CollectPattern applies the pattern for collection resources.
+func (obj *FileRes) CollectPattern(pattern string) {
+	// XXX: currently the pattern for files can only override the Dirname variable :P
+	obj.Dirname = pattern // XXX: simplistic for now
+}

resources/hostname.go

@@ -0,0 +1,295 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"encoding/gob"
+	"errors"
+	"fmt"
+	"log"
+
+	"github.com/purpleidea/mgmt/event"
+	"github.com/purpleidea/mgmt/util"
+
+	"github.com/godbus/dbus"
+	errwrap "github.com/pkg/errors"
+)
+
+// ErrResourceInsufficientParameters is returned when the configuration of the resource
+// is insufficient for the resource to do any useful work.
+var ErrResourceInsufficientParameters = errors.New(
+	"Insufficient parameters for this resource")
+
+func init() {
+	gob.Register(&HostnameRes{})
+}
+
+const (
+	hostname1Path  = "/org/freedesktop/hostname1"
+	hostname1Iface = "org.freedesktop.hostname1"
+	dbusAddMatch   = "org.freedesktop.DBus.AddMatch"
+)
+
+// HostnameRes is a resource that allows setting and watching the hostname.
+//
+// StaticHostname is the one configured in /etc/hostname or a similar file.
+// It is chosen by the local user. It is not always in sync with the current
+// host name as returned by the gethostname() system call.
+//
+// TransientHostname is the one configured via the kernel's sethostbyname().
+// It can be different from the static hostname in case DHCP or mDNS have been
+// configured to change the name based on network information.
+//
+// PrettyHostname is a free-form UTF8 host name for presentation to the user.
+//
+// Hostname is the fallback value for all 3 fields above, if only Hostname is
+// specified, it will set all 3 fields to this value.
+type HostnameRes struct {
+	BaseRes           `yaml:",inline"`
+	Hostname          string `yaml:"hostname"`
+	PrettyHostname    string `yaml:"pretty_hostname"`
+	StaticHostname    string `yaml:"static_hostname"`
+	TransientHostname string `yaml:"transient_hostname"`
+
+	conn *dbus.Conn
+}
+
+// NewHostnameRes is a constructor for this resource. It also calls Init() for you.
+func NewHostnameRes(name, staticHostname, transientHostname, prettyHostname string) (*HostnameRes, error) {
+	obj := &HostnameRes{
+		BaseRes: BaseRes{
+			Name: name,
+		},
+		PrettyHostname:    prettyHostname,
+		StaticHostname:    staticHostname,
+		TransientHostname: transientHostname,
+	}
+	return obj, obj.Init()
+}
+
+// Init runs some startup code for this resource.
+func (obj *HostnameRes) Init() error {
+	obj.BaseRes.kind = "Hostname"
+	if obj.PrettyHostname == "" {
+		obj.PrettyHostname = obj.Hostname
+	}
+	if obj.StaticHostname == "" {
+		obj.StaticHostname = obj.Hostname
+	}
+	if obj.TransientHostname == "" {
+		obj.TransientHostname = obj.Hostname
+	}
+	return obj.BaseRes.Init() // call base init, b/c we're overriding
+}
+
+// Validate if the params passed in are valid data.
+// FIXME: where should this get called ?
+func (obj *HostnameRes) Validate() error {
+	if obj.PrettyHostname == "" && obj.StaticHostname == "" && obj.TransientHostname == "" {
+		return ErrResourceInsufficientParameters
+	}
+	return nil
+}
+
+// Watch is the primary listener for this resource and it outputs events.
+func (obj *HostnameRes) Watch(processChan chan event.Event) error {
+	cuid := obj.Converger() // get the converger uid used to report status
+
+	// if we share the bus with others, we will get each others messages!!
+	bus, err := util.SystemBusPrivateUsable() // don't share the bus connection!
+	if err != nil {
+		return errwrap.Wrap(err, "Failed to connect to bus")
+	}
+	defer bus.Close()
+	callResult := bus.BusObject().Call(
+		"org.freedesktop.DBus.AddMatch", 0,
+		fmt.Sprintf("type='signal',path='%s',interface='org.freedesktop.DBus.Properties',member='PropertiesChanged'", hostname1Path))
+	if callResult.Err != nil {
+		return errwrap.Wrap(callResult.Err, "Failed to subscribe to DBus events for hostname1")
+	}
+
+	signals := make(chan *dbus.Signal, 10) // closed by dbus package
+	bus.Signal(signals)
+
+	// notify engine that we're running
+	if err := obj.Running(processChan); err != nil {
+		return err // bubble up a NACK...
+	}
+
+	var send = false // send event?
+
+	for {
+		obj.SetState(ResStateWatching) // reset
+		select {
+		case <-signals:
+			cuid.SetConverged(false)
+			send = true
+			obj.StateOK(false) // dirty
+
+		case event := <-obj.Events():
+			cuid.SetConverged(false)
+			// we avoid sending events on unpause
+			if exit, _ := obj.ReadEvent(&event); exit {
+				return nil // exit
+			}
+			send = true
+			obj.StateOK(false) // dirty
+
+		case <-cuid.ConvergedTimer():
+			cuid.SetConverged(true) // converged!
+			continue
+		}
+
+		// do all our event sending all together to avoid duplicate msgs
+		if send {
+			send = false
+
+			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
+				return err // we exit or bubble up a NACK...
+			}
+		}
+	}
+}
+
+func updateHostnameProperty(object dbus.BusObject, expectedValue, property, setterName string, apply bool) (checkOK bool, err error) {
+	propertyObject, err := object.GetProperty("org.freedesktop.hostname1." + property)
+	if err != nil {
+		return false, errwrap.Wrapf(err, "failed to get org.freedesktop.hostname1.%s", property)
+	}
+	if propertyObject.Value() == nil {
+		return false, errwrap.Errorf("Unexpected nil value received when reading property %s", property)
+	}
+
+	propertyValue, ok := propertyObject.Value().(string)
+	if !ok {
+		return false, fmt.Errorf("Received unexpected type as %s value, expected string got '%T'", property, propertyValue)
+	}
+
+	// expected value and actual value match => checkOk
+	if propertyValue == expectedValue {
+		return true, nil
+	}
+
+	// nothing to do anymore
+	if !apply {
+		return false, nil
+	}
+
+	// attempting to apply the changes
+	log.Printf("Changing %s: %s => %s", property, propertyValue, expectedValue)
+	if err := object.Call("org.freedesktop.hostname1."+setterName, 0, expectedValue, false).Err; err != nil {
+		return false, errwrap.Wrapf(err, "failed to call org.freedesktop.hostname1.%s", setterName)
+	}
+
+	// all good changes should now be applied again
+	return false, nil
+}
+
+// CheckApply method for Hostname resource.
+func (obj *HostnameRes) CheckApply(apply bool) (checkOK bool, err error) {
+	conn, err := util.SystemBusPrivateUsable()
+	if err != nil {
+		return false, errwrap.Wrap(err, "Failed to connect to the private system bus")
+	}
+	defer conn.Close()
+
+	hostnameObject := conn.Object(hostname1Iface, hostname1Path)
+
+	checkOK = true
+	if obj.PrettyHostname != "" {
+		propertyCheckOK, err := updateHostnameProperty(hostnameObject, obj.PrettyHostname, "PrettyHostname", "SetPrettyHostname", apply)
+		if err != nil {
+			return false, err
+		}
+		checkOK = checkOK && propertyCheckOK
+	}
+	if obj.StaticHostname != "" {
+		propertyCheckOK, err := updateHostnameProperty(hostnameObject, obj.StaticHostname, "StaticHostname", "SetStaticHostname", apply)
+		if err != nil {
+			return false, err
+		}
+		checkOK = checkOK && propertyCheckOK
+	}
+	if obj.TransientHostname != "" {
+		propertyCheckOK, err := updateHostnameProperty(hostnameObject, obj.TransientHostname, "Hostname", "SetHostname", apply)
+		if err != nil {
+			return false, err
+		}
+		checkOK = checkOK && propertyCheckOK
+	}
+
+	return checkOK, nil
+}
+
+// HostnameUID is the UID struct for HostnameRes.
+type HostnameUID struct {
+	BaseUID
+	name              string
+	prettyHostname    string
+	staticHostname    string
+	transientHostname string
+}
+
+// AutoEdges returns the AutoEdge interface. In this case no autoedges are used.
+func (obj *HostnameRes) AutoEdges() AutoEdge {
+	return nil
+}
+
+// GetUIDs includes all params to make a unique identification of this object.
+// Most resources only return one, although some resources can return multiple.
+func (obj *HostnameRes) GetUIDs() []ResUID {
+	x := &HostnameUID{
+		BaseUID:           BaseUID{name: obj.GetName(), kind: obj.Kind()},
+		name:              obj.Name,
+		prettyHostname:    obj.PrettyHostname,
+		staticHostname:    obj.StaticHostname,
+		transientHostname: obj.TransientHostname,
+	}
+	return []ResUID{x}
+}
+
+// GroupCmp returns whether two resources can be grouped together or not.
+func (obj *HostnameRes) GroupCmp(r Res) bool {
+	return false
+}
+
+// Compare two resources and return if they are equivalent.
+func (obj *HostnameRes) Compare(res Res) bool {
+	switch res := res.(type) {
+	// we can only compare HostnameRes to others of the same resource
+	case *HostnameRes:
+		if !obj.BaseRes.Compare(res) { // call base Compare
+			return false
+		}
+		if obj.Name != res.Name {
+			return false
+		}
+		if obj.PrettyHostname != res.PrettyHostname {
+			return false
+		}
+		if obj.StaticHostname != res.StaticHostname {
+			return false
+		}
+		if obj.TransientHostname != res.TransientHostname {
+			return false
+		}
+	default:
+		return false
+	}
+	return true
+}

resources/msg.go

@@ -0,0 +1,261 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"encoding/gob"
+	"fmt"
+	"log"
+	"regexp"
+	"strings"
+
+	"github.com/purpleidea/mgmt/event"
+
+	"github.com/coreos/go-systemd/journal"
+)
+
+func init() {
+	gob.Register(&MsgRes{})
+}
+
+// MsgRes is a resource that writes messages to logs.
+type MsgRes struct {
+	BaseRes        `yaml:",inline"`
+	Body           string            `yaml:"body"`
+	Priority       string            `yaml:"priority"`
+	Fields         map[string]string `yaml:"fields"`
+	Journal        bool              `yaml:"journal"` // enable systemd journal output
+	Syslog         bool              `yaml:"syslog"`  // enable syslog output
+	logStateOK     bool
+	journalStateOK bool
+	syslogStateOK  bool
+}
+
+// MsgUID is a unique representation for a MsgRes object.
+type MsgUID struct {
+	BaseUID
+	body string
+}
+
+// NewMsgRes is a constructor for this resource.
+func NewMsgRes(name, body, priority string, journal, syslog bool, fields map[string]string) (*MsgRes, error) {
+	message := name
+	if body != "" {
+		message = body
+	}
+
+	obj := &MsgRes{
+		BaseRes: BaseRes{
+			Name: name,
+		},
+		Body:     message,
+		Priority: priority,
+		Fields:   fields,
+		Journal:  journal,
+		Syslog:   syslog,
+	}
+
+	return obj, obj.Init()
+}
+
+// Init runs some startup code for this resource.
+func (obj *MsgRes) Init() error {
+	obj.BaseRes.kind = "Msg"
+	return obj.BaseRes.Init() // call base init, b/c we're overrriding
+}
+
+// Validate the params that are passed to MsgRes
+func (obj *MsgRes) Validate() error {
+	invalidCharacters := regexp.MustCompile("[^a-zA-Z0-9_]")
+	for field := range obj.Fields {
+		if invalidCharacters.FindString(field) != "" {
+			return fmt.Errorf("Invalid character in field %s.", field)
+		}
+		if strings.HasPrefix(field, "_") {
+			return fmt.Errorf("Fields cannot begin with _.")
+		}
+	}
+	return nil
+}
+
+// isAllStateOK derives a compound state from all internal cache flags that apply to this resource.
+func (obj *MsgRes) isAllStateOK() bool {
+	if obj.Journal && !obj.journalStateOK {
+		return false
+	}
+	if obj.Syslog && !obj.syslogStateOK {
+		return false
+	}
+	return obj.logStateOK
+}
+
+// updateStateOK sets the global state so it can be read by the engine.
+func (obj *MsgRes) updateStateOK() {
+	obj.StateOK(obj.isAllStateOK())
+}
+
+// JournalPriority converts a string description to a numeric priority.
+// XXX: Have Validate() make sure it actually is one of these.
+func (obj *MsgRes) journalPriority() journal.Priority {
+	switch obj.Priority {
+	case "Emerg":
+		return journal.PriEmerg
+	case "Alert":
+		return journal.PriAlert
+	case "Crit":
+		return journal.PriCrit
+	case "Err":
+		return journal.PriErr
+	case "Warning":
+		return journal.PriWarning
+	case "Notice":
+		return journal.PriNotice
+	case "Info":
+		return journal.PriInfo
+	case "Debug":
+		return journal.PriDebug
+	}
+	return journal.PriNotice
+}
+
+// Watch is the primary listener for this resource and it outputs events.
+func (obj *MsgRes) Watch(processChan chan event.Event) error {
+	cuid := obj.Converger() // get the converger uid used to report status
+
+	// notify engine that we're running
+	if err := obj.Running(processChan); err != nil {
+		return err // bubble up a NACK...
+	}
+
+	var send = false // send event?
+	var exit = false
+	for {
+		obj.SetState(ResStateWatching) // reset
+		select {
+		case event := <-obj.Events():
+			cuid.SetConverged(false)
+			// we avoid sending events on unpause
+			if exit, send = obj.ReadEvent(&event); exit {
+				return nil // exit
+			}
+
+		case <-cuid.ConvergedTimer():
+			cuid.SetConverged(true) // converged!
+			continue
+		}
+
+		// do all our event sending all together to avoid duplicate msgs
+		if send {
+			send = false
+			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
+				return err // we exit or bubble up a NACK...
+			}
+		}
+	}
+}
+
+// CheckApply method for Msg resource.
+// Every check leads to an apply, meaning that the message is flushed to the journal.
+func (obj *MsgRes) CheckApply(apply bool) (bool, error) {
+
+	// isStateOK() done by engine, so we updateStateOK() to pass in value
+	//if obj.isAllStateOK() {
+	//	return true, nil
+	//}
+
+	if obj.Refresh() { // if we were notified...
+		// invalidate cached state...
+		obj.logStateOK = false
+		if obj.Journal {
+			obj.journalStateOK = false
+		}
+		if obj.Syslog {
+			obj.syslogStateOK = false
+		}
+		obj.updateStateOK()
+	}
+
+	if !obj.logStateOK {
+		log.Printf("%s[%s]: Body: %s", obj.Kind(), obj.GetName(), obj.Body)
+		obj.logStateOK = true
+		obj.updateStateOK()
+	}
+
+	if !apply {
+		return false, nil
+	}
+	if obj.Journal && !obj.journalStateOK {
+		if err := journal.Send(obj.Body, obj.journalPriority(), obj.Fields); err != nil {
+			return false, err
+		}
+		obj.journalStateOK = true
+		obj.updateStateOK()
+	}
+	if obj.Syslog && !obj.syslogStateOK {
+		// TODO: implement syslog client
+		obj.syslogStateOK = true
+		obj.updateStateOK()
+	}
+	return false, nil
+}
+
+// GetUIDs includes all params to make a unique identification of this object.
+// Most resources only return one, although some resources can return multiple.
+func (obj *MsgRes) GetUIDs() []ResUID {
+	x := &MsgUID{
+		BaseUID: BaseUID{
+			name: obj.GetName(),
+			kind: obj.Kind(),
+		},
+		body: obj.Body,
+	}
+	return []ResUID{x}
+}
+
+// AutoEdges returns the AutoEdges. In this case none are used.
+func (obj *MsgRes) AutoEdges() AutoEdge {
+	return nil
+}
+
+// Compare two resources and return if they are equivalent.
+func (obj *MsgRes) Compare(res Res) bool {
+	switch res.(type) {
+	case *MsgRes:
+		res := res.(*MsgRes)
+		if !obj.BaseRes.Compare(res) {
+			return false
+		}
+		if obj.Body != res.Body {
+			return false
+		}
+		if obj.Priority != res.Priority {
+			return false
+		}
+		if len(obj.Fields) != len(res.Fields) {
+			return false
+		}
+		for field, value := range obj.Fields {
+			if res.Fields[field] != value {
+				return false
+			}
+		}
+	default:
+		return false
+	}
+	return true
+}

resources/noop.go

@@ -15,107 +15,114 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package main
+package resources
 
 import (
 	"encoding/gob"
 	"log"
+
+	"github.com/purpleidea/mgmt/event"
 )
 
 func init() {
 	gob.Register(&NoopRes{})
 }
 
+// NoopRes is a no-op resource that does nothing.
 type NoopRes struct {
 	BaseRes `yaml:",inline"`
 	Comment string `yaml:"comment"` // extra field for example purposes
 }
 
-func NewNoopRes(name string) *NoopRes {
+// NewNoopRes is a constructor for this resource. It also calls Init() for you.
+func NewNoopRes(name string) (*NoopRes, error) {
 	obj := &NoopRes{
 		BaseRes: BaseRes{
 			Name: name,
 		},
 		Comment: "",
 	}
-	obj.Init()
-	return obj
+	return obj, obj.Init()
 }
 
-func (obj *NoopRes) Init() {
+// Init runs some startup code for this resource.
+func (obj *NoopRes) Init() error {
 	obj.BaseRes.kind = "Noop"
-	obj.BaseRes.Init() // call base init, b/c we're overriding
+	return obj.BaseRes.Init() // call base init, b/c we're overriding
 }
 
-// validate if the params passed in are valid data
+// Validate if the params passed in are valid data.
 // FIXME: where should this get called ?
-func (obj *NoopRes) Validate() bool {
-	return true
+func (obj *NoopRes) Validate() error {
+	return nil
 }
 
-func (obj *NoopRes) Watch(processChan chan Event) {
-	if obj.IsWatching() {
-		return
-	}
-	obj.SetWatching(true)
-	defer obj.SetWatching(false)
+// Watch is the primary listener for this resource and it outputs events.
+func (obj *NoopRes) Watch(processChan chan event.Event) error {
+	cuid := obj.Converger() // get the converger uid used to report status
+
+	// notify engine that we're running
+	if err := obj.Running(processChan); err != nil {
+		return err // bubble up a NACK...
+	}
 
-	//vertex := obj.vertex // stored with SetVertex
 	var send = false // send event?
 	var exit = false
 	for {
-		obj.SetState(resStateWatching) // reset
+		obj.SetState(ResStateWatching) // reset
 		select {
-		case event := <-obj.events:
-			obj.SetConvergedState(resConvergedNil)
+		case event := <-obj.Events():
+			cuid.SetConverged(false)
 			// we avoid sending events on unpause
 			if exit, send = obj.ReadEvent(&event); exit {
-				return // exit
+				return nil // exit
 			}
 
-		case _ = <-TimeAfterOrBlock(obj.ctimeout):
-			obj.SetConvergedState(resConvergedTimeout)
-			obj.converged <- true
+		case <-cuid.ConvergedTimer():
+			cuid.SetConverged(true) // converged!
 			continue
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
 			send = false
-			// only do this on certain types of events
-			//obj.isStateOK = false // something made state dirty
-			resp := NewResp()
-			processChan <- Event{eventNil, resp, "", true} // trigger process
-			resp.ACKWait()                                 // wait for the ACK()
+			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
+				return err // we exit or bubble up a NACK...
+			}
 		}
 	}
 }
 
 // CheckApply method for Noop resource. Does nothing, returns happy!
-func (obj *NoopRes) CheckApply(apply bool) (stateok bool, err error) {
-	log.Printf("%v[%v]: CheckApply(%t)", obj.Kind(), obj.GetName(), apply)
+func (obj *NoopRes) CheckApply(apply bool) (checkOK bool, err error) {
+	if obj.Refresh() {
+		log.Printf("%s[%s]: Received a notification!", obj.Kind(), obj.GetName())
+	}
 	return true, nil // state is always okay
 }
 
-type NoopUUID struct {
-	BaseUUID
+// NoopUID is the UID struct for NoopRes.
+type NoopUID struct {
+	BaseUID
 	name string
 }
 
+// AutoEdges returns the AutoEdge interface. In this case no autoedges are used.
 func (obj *NoopRes) AutoEdges() AutoEdge {
 	return nil
 }
 
-// include all params to make a unique identification of this object
-// most resources only return one, although some resources return multiple
-func (obj *NoopRes) GetUUIDs() []ResUUID {
-	x := &NoopUUID{
-		BaseUUID: BaseUUID{name: obj.GetName(), kind: obj.Kind()},
+// GetUIDs includes all params to make a unique identification of this object.
+// Most resources only return one, although some resources can return multiple.
+func (obj *NoopRes) GetUIDs() []ResUID {
+	x := &NoopUID{
+		BaseUID: BaseUID{name: obj.GetName(), kind: obj.Kind()},
 		name:    obj.Name,
 	}
-	return []ResUUID{x}
+	return []ResUID{x}
 }
 
+// GroupCmp returns whether two resources can be grouped together or not.
 func (obj *NoopRes) GroupCmp(r Res) bool {
 	_, ok := r.(*NoopRes)
 	if !ok {
@@ -128,11 +135,16 @@ func (obj *NoopRes) GroupCmp(r Res) bool {
 	return true // noop resources can always be grouped together!
 }
 
+// Compare two resources and return if they are equivalent.
 func (obj *NoopRes) Compare(res Res) bool {
 	switch res.(type) {
 	// we can only compare NoopRes to others of the same resource
 	case *NoopRes:
 		res := res.(*NoopRes)
+		// calling base Compare is unneeded for the noop res
+		//if !obj.BaseRes.Compare(res) { // call base Compare
+		//	return false
+		//}
 		if obj.Name != res.Name {
 			return false
 		}

resources/nspawn.go

@@ -0,0 +1,306 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"encoding/gob"
+	"errors"
+	"fmt"
+	"log"
+
+	"github.com/purpleidea/mgmt/event"
+	"github.com/purpleidea/mgmt/util"
+
+	systemdUtil "github.com/coreos/go-systemd/util"
+	"github.com/godbus/dbus"
+	errwrap "github.com/pkg/errors"
+	machined "github.com/purpleidea/go-systemd/machine1"
+)
+
+const (
+	running           = "running"
+	stopped           = "stopped"
+	dbusInterface     = "org.freedesktop.machine1.Manager"
+	machineNew        = "org.freedesktop.machine1.Manager.MachineNew"
+	machineRemoved    = "org.freedesktop.machine1.Manager.MachineRemoved"
+	nspawnServiceTmpl = "systemd-nspawn@%s"
+)
+
+func init() {
+	gob.Register(&NspawnRes{})
+}
+
+// NspawnRes is an nspawn container resource
+type NspawnRes struct {
+	BaseRes `yaml:",inline"`
+	State   string `yaml:"state"`
+	// we're using the svc resource to start the machine because that's
+	// what machinectl does. We're not using svc.Watch because then we
+	// would have two watches potentially racing each other and producing
+	// potentially unexpected results. We get everything we need to
+	// monitor the machine state changes from the org.freedesktop.machine1 object.
+	svc *SvcRes
+}
+
+// Init runs some startup code for this resource
+func (obj *NspawnRes) Init() error {
+	var serviceName = fmt.Sprintf(nspawnServiceTmpl, obj.GetName())
+	obj.svc = &SvcRes{}
+	obj.svc.Name = serviceName
+	obj.svc.State = obj.State
+	if err := obj.svc.Init(); err != nil {
+		return err
+	}
+	obj.BaseRes.kind = "Nspawn"
+	return obj.BaseRes.Init()
+}
+
+// NewNspawnRes is the constructor for this resource
+func NewNspawnRes(name string, state string) (*NspawnRes, error) {
+	obj := &NspawnRes{
+		BaseRes: BaseRes{
+			Name: name,
+		},
+		State: state,
+	}
+	return obj, obj.Init()
+}
+
+// Validate params
+func (obj *NspawnRes) Validate() error {
+	validStates := map[string]struct{}{
+		stopped: {},
+		running: {},
+	}
+	if _, exists := validStates[obj.State]; !exists {
+		return fmt.Errorf("Invalid State: %s", obj.State)
+	}
+	return obj.svc.Validate()
+}
+
+// Watch for state changes and sends a message to the bus if there is a change
+func (obj *NspawnRes) Watch(processChan chan event.Event) error {
+	cuid := obj.Converger() // get the converger uid used to report status
+
+	// this resource depends on systemd ensure that it's running
+	if !systemdUtil.IsRunningSystemd() {
+		return fmt.Errorf("Systemd is not running.")
+	}
+
+	// create a private message bus
+	bus, err := util.SystemBusPrivateUsable()
+	if err != nil {
+		return errwrap.Wrapf(err, "Failed to connect to bus")
+	}
+
+	// add a match rule to match messages going through the message bus
+	call := bus.BusObject().Call("org.freedesktop.DBus.AddMatch", 0,
+		fmt.Sprintf("type='signal',interface='%s',eavesdrop='true'",
+			dbusInterface))
+	// <-call.Done
+	if err := call.Err; err != nil {
+		return err
+	}
+	buschan := make(chan *dbus.Signal, 10)
+	bus.Signal(buschan)
+
+	// notify engine that we're running
+	if err := obj.Running(processChan); err != nil {
+		return err // bubble up a NACK...
+	}
+
+	var send = false
+	var exit = false
+
+	for {
+		obj.SetState(ResStateWatching)
+		select {
+		case event := <-buschan:
+			// process org.freedesktop.machine1 events for this resource's name
+			if event.Body[0] == obj.GetName() {
+				log.Printf("%s[%s]: Event received: %v", obj.Kind(), obj.GetName(), event.Name)
+				if event.Name == machineNew {
+					log.Printf("%s[%s]: Machine started", obj.Kind(), obj.GetName())
+				} else if event.Name == machineRemoved {
+					log.Printf("%s[%s]: Machine stopped", obj.Kind(), obj.GetName())
+				} else {
+					return fmt.Errorf("Unknown event: %s", event.Name)
+				}
+				send = true
+				obj.StateOK(false) // dirty
+			}
+
+		case event := <-obj.Events():
+			cuid.SetConverged(false)
+			if exit, send = obj.ReadEvent(&event); exit {
+				return nil // exit
+			}
+
+		case <-cuid.ConvergedTimer():
+			cuid.SetConverged(true) // converged!
+			continue
+		}
+
+		// do all our event sending all together to avoid duplicate msgs
+		if send {
+			send = false
+			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
+				return err // we exit or bubble up a NACK...
+			}
+		}
+	}
+}
+
+// CheckApply is run to check the state and, if apply is true, to apply the
+// necessary changes to reach the desired state. this is run before Watch and
+// again if watch finds a change occurring to the state
+func (obj *NspawnRes) CheckApply(apply bool) (checkOK bool, err error) {
+	// this resource depends on systemd ensure that it's running
+	if !systemdUtil.IsRunningSystemd() {
+		return false, errors.New("Systemd is not running.")
+	}
+
+	// connect to org.freedesktop.machine1.Manager
+	conn, err := machined.New()
+	if err != nil {
+		return false, errwrap.Wrapf(err, "Failed to connect to dbus")
+	}
+
+	// compare the current state with the desired state and perform the
+	// appropriate action
+	var exists = true
+	properties, err := conn.GetProperties(obj.GetName())
+	if err != nil {
+		if err, ok := err.(dbus.Error); ok && err.Name !=
+			"org.freedesktop.machine1.NoSuchMachine" {
+			return false, err
+		}
+		exists = false
+		// if we could not successfully get the properties because
+		// there's no such machine the machine is stopped
+		// error if we need the image ignore if we don't
+		if _, err = conn.GetImage(obj.GetName()); err != nil && obj.State != stopped {
+			return false, fmt.Errorf(
+				"No machine nor image named '%s'",
+				obj.GetName())
+		}
+	}
+	if obj.debug {
+		log.Printf("%s[%s]: properties: %v", obj.Kind(), obj.GetName(), properties)
+	}
+	// if the machine doesn't exist and is supposed to
+	// be stopped or the state matches we're done
+	if !exists && obj.State == stopped || properties["State"] == obj.State {
+		if obj.debug {
+			log.Printf("%s[%s]: CheckApply() in valid state", obj.Kind(), obj.GetName())
+		}
+		return true, nil
+	}
+
+	// end of state checking. if we're here, checkOK is false
+	if !apply {
+		return false, nil
+	}
+
+	if obj.debug {
+		log.Printf("%s[%s]: CheckApply() applying '%s' state", obj.Kind(), obj.GetName(), obj.State)
+	}
+
+	if obj.State == running {
+		// start the machine using svc resource
+		log.Printf("%s[%s]: Starting machine", obj.Kind(), obj.GetName())
+		// assume state had to be changed at this point, ignore checkOK
+		if _, err := obj.svc.CheckApply(apply); err != nil {
+			return false, errwrap.Wrapf(err, "Nested svc failed")
+		}
+	}
+	if obj.State == stopped {
+		// terminate the machine with
+		// org.freedesktop.machine1.Manager.KillMachine
+		log.Printf("%s[%s]: Stopping machine", obj.Kind(), obj.GetName())
+		if err := conn.TerminateMachine(obj.GetName()); err != nil {
+			return false, errwrap.Wrapf(err, "Failed to stop machine")
+		}
+	}
+
+	return false, nil
+}
+
+// NspawnUID is a unique resource identifier
+type NspawnUID struct {
+	// NOTE: there is also a name variable in the BaseUID struct, this is
+	// information about where this UID came from, and is unrelated to the
+	// information about the resource we're matching. That data which is
+	// used in the IFF function, is what you see in the struct fields here
+	BaseUID
+	name string // the machine name
+}
+
+// IFF aka if and only if they are equivalent, return true. If not, false
+func (obj *NspawnUID) IFF(uid ResUID) bool {
+	res, ok := uid.(*NspawnUID)
+	if !ok {
+		return false
+	}
+	return obj.name == res.name
+}
+
+// GetUIDs includes all params to make a unique identification of this object
+// most resources only return one although some resources can return multiple
+func (obj *NspawnRes) GetUIDs() []ResUID {
+	x := &NspawnUID{
+		BaseUID: BaseUID{name: obj.GetName(), kind: obj.Kind()},
+		name:    obj.Name, // svc name
+	}
+	return append([]ResUID{x}, obj.svc.GetUIDs()...)
+}
+
+// GroupCmp returns whether two resources can be grouped together or not
+func (obj *NspawnRes) GroupCmp(r Res) bool {
+	_, ok := r.(*NspawnRes)
+	if !ok {
+		return false
+	}
+	// TODO: this would be quite useful for this resource!
+	return false
+}
+
+// Compare two resources and return if they are equivalent
+func (obj *NspawnRes) Compare(res Res) bool {
+	switch res.(type) {
+	case *NspawnRes:
+		res := res.(*NspawnRes)
+		if !obj.BaseRes.Compare(res) {
+			return false
+		}
+		if obj.Name != res.Name {
+			return false
+		}
+		if !obj.svc.Compare(res.svc) {
+			return false
+		}
+	default:
+		return false
+	}
+	return true
+}
+
+// AutoEdges returns the AutoEdge interface in this case no autoedges are used
+func (obj *NspawnRes) AutoEdges() AutoEdge {
+	return nil
+}

resources/packagekit/packagekit.go

@@ -15,24 +15,29 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-// DOCS: https://www.freedesktop.org/software/PackageKit/gtk-doc/index.html
-
-//package packagekit // TODO
-package main
+// Package packagekit provides an interface to interact with packagekit.
+// See: https://www.freedesktop.org/software/PackageKit/gtk-doc/index.html for
+// more information.
+package packagekit
 
 import (
 	"fmt"
-	"github.com/godbus/dbus"
 	"log"
 	"runtime"
 	"strings"
+
+	"github.com/purpleidea/mgmt/util"
+
+	"github.com/godbus/dbus"
 )
 
+// global tweaks of verbosity and code path
 const (
 	PK_DEBUG = false
 	PARANOID = false // enable if you see any ghosts
 )
 
+// constants which might need to be tweaked or which contain special dbus strings.
 const (
 	// FIXME: if PkBufferSize is too low, install seems to drop signals
 	PkBufferSize = 1000
@@ -46,11 +51,13 @@ const (
 )
 
 var (
+	// PkArchMap contains the mapping from PackageKit arch to GOARCH.
 	// GOARCH's: 386, amd64, arm, arm64, mips64, mips64le, ppc64, ppc64le
 	PkArchMap = map[string]string{ // map of PackageKit arch to GOARCH
 		// TODO: add more values
 		// noarch
-		"noarch": "ANY", // special value "ANY"
+		"noarch": "ANY", // special value "ANY" (noarch as seen in Fedora)
+		"all":    "ANY", // special value "ANY" ('all' as seen in Debian)
 		// fedora
 		"x86_64":  "amd64",
 		"aarch64": "arm64",
@@ -97,6 +104,7 @@ const ( //static const PkEnumMatch enum_filter[]
 	PK_FILTER_ENUM_NOT_DOWNLOADED                     // "~downloaded"
 )
 
+// constants from packagekit c library.
 const ( //static const PkEnumMatch enum_transaction_flag[]
 	PK_TRANSACTION_FLAG_ENUM_NONE            uint64 = 1 << iota // "none"
 	PK_TRANSACTION_FLAG_ENUM_ONLY_TRUSTED                       // "only-trusted"
@@ -107,6 +115,7 @@ const ( //static const PkEnumMatch enum_transaction_flag[]
 	PK_TRANSACTION_FLAG_ENUM_ALLOW_DOWNGRADE                    // "allow-downgrade"
 )
 
+// constants from packagekit c library.
 const ( //typedef enum
 	PK_INFO_ENUM_UNKNOWN uint64 = 1 << iota
 	PK_INFO_ENUM_INSTALLED
@@ -137,12 +146,12 @@ const ( //typedef enum
 	PK_INFO_ENUM_LAST
 )
 
-// wrapper struct so we can pass bus connection around in the struct
+// Conn is a wrapper struct so we can pass bus connection around in the struct.
 type Conn struct {
 	conn *dbus.Conn
 }
 
-// struct that is returned by PackagesToPackageIDs in the map values
+// PkPackageIDActionData is a struct that is returned by PackagesToPackageIDs in the map values.
 type PkPackageIDActionData struct {
 	Found     bool
 	Installed bool
@@ -151,10 +160,10 @@ type PkPackageIDActionData struct {
 	Newest    bool
 }
 
-// get a new bus connection
+// NewBus returns a new bus connection.
 func NewBus() *Conn {
 	// if we share the bus with others, we will get each others messages!!
-	bus, err := SystemBusPrivateUsable() // don't share the bus connection!
+	bus, err := util.SystemBusPrivateUsable() // don't share the bus connection!
 	if err != nil {
 		return nil
 	}
@@ -163,12 +172,12 @@ func NewBus() *Conn {
 	}
 }
 
-// get the dbus connection object
+// GetBus gets the dbus connection object.
 func (bus *Conn) GetBus() *dbus.Conn {
 	return bus.conn
 }
 
-// close the dbus connection object
+// Close closes the dbus connection object.
 func (bus *Conn) Close() error {
 	return bus.conn.Close()
 }
@@ -204,7 +213,7 @@ func (bus *Conn) matchSignal(ch chan *dbus.Signal, path dbus.ObjectPath, iface s
 	return nil
 }
 
-// get a signal anytime an event happens
+// WatchChanges gets a signal anytime an event happens.
 func (bus *Conn) WatchChanges() (chan *dbus.Signal, error) {
 	ch := make(chan *dbus.Signal, PkBufferSize)
 	// NOTE: the TransactionListChanged signal fires much more frequently,
@@ -246,7 +255,7 @@ func (bus *Conn) WatchChanges() (chan *dbus.Signal, error) {
 	return ch, nil
 }
 
-// create and return a transaction path
+// CreateTransaction creates and returns a transaction path.
 func (bus *Conn) CreateTransaction() (dbus.ObjectPath, error) {
 	if PK_DEBUG {
 		log.Println("PackageKit: CreateTransaction()")
@@ -263,6 +272,7 @@ func (bus *Conn) CreateTransaction() (dbus.ObjectPath, error) {
 	return interfacePath, nil
 }
 
+// ResolvePackages runs the PackageKit Resolve method and returns the result.
 func (bus *Conn) ResolvePackages(packages []string, filter uint64) ([]string, error) {
 	packageIDs := []string{}
 	ch := make(chan *dbus.Signal, PkBufferSize)   // we need to buffer :(
@@ -326,6 +336,7 @@ loop:
 	return packageIDs, nil
 }
 
+// IsInstalledList queries a list of packages to see if they are installed.
 func (bus *Conn) IsInstalledList(packages []string) ([]bool, error) {
 	var filter uint64             // initializes at the "zero" value of 0
 	filter += PK_FILTER_ENUM_ARCH // always search in our arch
@@ -362,7 +373,7 @@ func (bus *Conn) IsInstalledList(packages []string) ([]bool, error) {
 	return r, nil
 }
 
-// is package installed ?
+// IsInstalled returns if a package is installed.
 // TODO: this could be optimized by making the resolve call directly
 func (bus *Conn) IsInstalled(pkg string) (bool, error) {
 	p, e := bus.IsInstalledList([]string{pkg})
@@ -372,7 +383,7 @@ func (bus *Conn) IsInstalled(pkg string) (bool, error) {
 	return p[0], nil
 }
 
-// install list of packages by packageID
+// InstallPackages installs a list of packages by packageID.
 func (bus *Conn) InstallPackages(packageIDs []string, transactionFlags uint64) error {
 
 	ch := make(chan *dbus.Signal, PkBufferSize)   // we need to buffer :(
@@ -414,7 +425,7 @@ loop:
 			} else {
 				return fmt.Errorf("PackageKit: Error: %v", signal.Body)
 			}
-		case _ = <-TimeAfterOrBlock(timeout):
+		case <-util.TimeAfterOrBlock(timeout):
 			if finished {
 				log.Println("PackageKit: Timeout: InstallPackages: Waiting for 'Destroy'")
 				return nil // got tired of waiting for Destroy
@@ -424,7 +435,7 @@ loop:
 	}
 }
 
-// remove list of packages
+// RemovePackages removes a list of packages by packageID.
 func (bus *Conn) RemovePackages(packageIDs []string, transactionFlags uint64) error {
 
 	var allowDeps = true                          // TODO: configurable
@@ -472,7 +483,7 @@ loop:
 	return nil
 }
 
-// update list of packages to versions that are specified
+// UpdatePackages updates a list of packages to versions that are specified.
 func (bus *Conn) UpdatePackages(packageIDs []string, transactionFlags uint64) error {
 	ch := make(chan *dbus.Signal, PkBufferSize) // we need to buffer :(
 	interfacePath, err := bus.CreateTransaction()
@@ -515,7 +526,7 @@ loop:
 	return nil
 }
 
-// get the list of files that are contained inside a list of packageids
+// GetFilesByPackageID gets the list of files that are contained inside a list of packageIDs.
 func (bus *Conn) GetFilesByPackageID(packageIDs []string) (files map[string][]string, err error) {
 	// NOTE: the maximum number of files in an RPM is 52116 in Fedora 23
 	// https://gist.github.com/purpleidea/b98e60dcd449e1ac3b8a
@@ -580,7 +591,7 @@ loop:
 	return
 }
 
-// get list of packages that are installed and which can be updated, mod filter
+// GetUpdates gets a list of packages that are installed and which can be updated, mod filter.
 func (bus *Conn) GetUpdates(filter uint64) ([]string, error) {
 	if PK_DEBUG {
 		log.Println("PackageKit: GetUpdates()")
@@ -641,9 +652,10 @@ loop:
 	return packageIDs, nil
 }
 
-// this is a helper function that *might* be generally useful outside mgmtconfig
-// packageMap input has the package names as keys and requested states as values
-// these states can be installed, uninstalled, newest or a requested version str
+// PackagesToPackageIDs is a helper function that *might* be generally useful
+// outside mgmt. The packageMap input has the package names as keys and
+// requested states as values. These states can be: installed, uninstalled,
+// newest or a requested version str.
 func (bus *Conn) PackagesToPackageIDs(packageMap map[string]string, filter uint64) (map[string]*PkPackageIDActionData, error) {
 	count := 0
 	packages := make([]string, len(packageMap))
@@ -814,7 +826,7 @@ func (bus *Conn) PackagesToPackageIDs(packageMap map[string]string, filter uint6
 	return result, nil
 }
 
-// returns a list of packageIDs which match the set of package names in packages
+// FilterPackageIDs returns a list of packageIDs which match the set of package names in packages.
 func FilterPackageIDs(m map[string]*PkPackageIDActionData, packages []string) ([]string, error) {
 	result := []string{}
 	for _, k := range packages {
@@ -828,6 +840,7 @@ func FilterPackageIDs(m map[string]*PkPackageIDActionData, packages []string) ([
 	return result, nil
 }
 
+// FilterState returns a map of whether each package queried matches the particular state.
 func FilterState(m map[string]*PkPackageIDActionData, packages []string, state string) (result map[string]bool, err error) {
 	result = make(map[string]bool)
 	pkgs := []string{} // bad pkgs that don't have a bool state
@@ -857,7 +870,7 @@ func FilterState(m map[string]*PkPackageIDActionData, packages []string, state s
 	return result, err
 }
 
-// return all packages that are in package and match the specific state
+// FilterPackageState returns all packages that are in package and match the specific state.
 func FilterPackageState(m map[string]*PkPackageIDActionData, packages []string, state string) (result []string, err error) {
 	result = []string{}
 	for _, k := range packages {
@@ -883,7 +896,7 @@ func FilterPackageState(m map[string]*PkPackageIDActionData, packages []string,
 	return result, err
 }
 
-// does flag exist inside data portion of packageID field?
+// FlagInData asks whether a flag exists inside the data portion of a packageID field?
 func FlagInData(flag, data string) bool {
 	flags := strings.Split(data, ":")
 	for _, f := range flags {
@@ -894,11 +907,12 @@ func FlagInData(flag, data string) bool {
 	return false
 }
 
-// builds the transaction method string
+// FmtTransactionMethod builds the transaction method string properly.
 func FmtTransactionMethod(method string) string {
 	return fmt.Sprintf("%s.%s", PkIfaceTransaction, method)
 }
 
+// IsMyArch determines if a PackageKit architecture matches the current os arch.
 func IsMyArch(arch string) bool {
 	goarch, ok := PkArchMap[arch]
 	if !ok {

resources/password.go

@@ -0,0 +1,365 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"crypto/rand"
+	"encoding/gob"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"math/big"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/purpleidea/mgmt/event"
+	"github.com/purpleidea/mgmt/recwatch"
+
+	errwrap "github.com/pkg/errors"
+)
+
+func init() {
+	gob.Register(&PasswordRes{})
+}
+
+const (
+	alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+	newline  = "\n" // something not in alphabet that TrimSpace can trim
+)
+
+// PasswordRes is a no-op resource that returns a random password string.
+type PasswordRes struct {
+	BaseRes `yaml:",inline"`
+	// FIXME: is uint16 too big?
+	Length        uint16  `yaml:"length"` // number of characters to return
+	Saved         bool    // this caches the password in the clear locally
+	CheckRecovery bool    // recovery from integrity checks by re-generating
+	Password      *string // the generated password, read only, do not set!
+
+	path       string // the path to local storage
+	recWatcher *recwatch.RecWatcher
+}
+
+// NewPasswordRes is a constructor for this resource. It also calls Init() for you.
+func NewPasswordRes(name string, length uint16) (*PasswordRes, error) {
+	obj := &PasswordRes{
+		BaseRes: BaseRes{
+			Name: name,
+		},
+		Length: length,
+	}
+	return obj, obj.Init()
+}
+
+// Init generates a new password for this resource if one was not provided. It
+// will save this into a local file. It will load it back in from previous runs.
+func (obj *PasswordRes) Init() error {
+	obj.BaseRes.kind = "Password" // must be set before using VarDir
+
+	dir, err := obj.VarDir("")
+	if err != nil {
+		return errwrap.Wrapf(err, "could not get VarDir in Init()")
+	}
+	obj.path = path.Join(dir, "password") // return a unique file
+
+	return obj.BaseRes.Init() // call base init, b/c we're overriding
+}
+
+// Validate if the params passed in are valid data.
+// FIXME: where should this get called ?
+func (obj *PasswordRes) Validate() error {
+	return nil
+}
+
+func (obj *PasswordRes) read() (string, error) {
+	file, err := os.Open(obj.path) // open a handle to read the file
+	if err != nil {
+		return "", err
+	}
+	defer file.Close()
+	data, err := ioutil.ReadAll(file)
+	if err != nil {
+		return "", errwrap.Wrapf(err, "could not read from file")
+	}
+	return strings.TrimSpace(string(data)), nil
+}
+
+func (obj *PasswordRes) write(password string) (int, error) {
+	file, err := os.Create(obj.path) // open a handle to create the file
+	if err != nil {
+		return -1, errwrap.Wrapf(err, "can't create file")
+	}
+	defer file.Close()
+	var c int
+	if c, err = file.Write([]byte(password + newline)); err != nil {
+		return c, errwrap.Wrapf(err, "can't write file")
+	}
+	return c, file.Sync()
+}
+
+// generate generates a new password.
+func (obj *PasswordRes) generate() (string, error) {
+	max := len(alphabet) - 1 // last index
+	output := ""
+
+	// FIXME: have someone verify this is cryptographically secure & correct
+	for i := uint16(0); i < obj.Length; i++ {
+		big, err := rand.Int(rand.Reader, big.NewInt(int64(max)))
+		if err != nil {
+			return "", errwrap.Wrapf(err, "could not generate password")
+		}
+		ix := big.Int64()
+		output += string(alphabet[ix])
+	}
+
+	if output == "" { // safety against empty passwords
+		return "", fmt.Errorf("password is empty")
+	}
+
+	if uint16(len(output)) != obj.Length { // safety against weird bugs
+		return "", fmt.Errorf("password length is too short") // bug!
+	}
+
+	return output, nil
+}
+
+// check validates a stored password string
+func (obj *PasswordRes) check(value string) error {
+	length := uint16(len(value))
+
+	if !obj.Saved && length == 0 { // expecting an empty string
+		return nil
+	}
+	if !obj.Saved && length != 0 { // should have no stored password
+		return fmt.Errorf("Expected empty token only!")
+	}
+
+	if length != obj.Length {
+		return fmt.Errorf("String length is not %d", obj.Length)
+	}
+Loop:
+	for i := uint16(0); i < length; i++ {
+		for j := 0; j < len(alphabet); j++ {
+			if value[i] == alphabet[j] {
+				continue Loop
+			}
+		}
+		// we couldn't find that character, so error!
+		return fmt.Errorf("Invalid character `%s`", string(value[i]))
+	}
+	return nil
+}
+
+// Watch is the primary listener for this resource and it outputs events.
+func (obj *PasswordRes) Watch(processChan chan event.Event) error {
+	cuid := obj.Converger() // get the converger uid used to report status
+
+	var err error
+	obj.recWatcher, err = recwatch.NewRecWatcher(obj.path, false)
+	if err != nil {
+		return err
+	}
+	defer obj.recWatcher.Close()
+
+	// notify engine that we're running
+	if err := obj.Running(processChan); err != nil {
+		return err // bubble up a NACK...
+	}
+
+	var send = false // send event?
+	var exit = false
+	for {
+		obj.SetState(ResStateWatching) // reset
+		select {
+		// NOTE: this part is very similar to the file resource code
+		case event, ok := <-obj.recWatcher.Events():
+			if !ok { // channel shutdown
+				return nil
+			}
+			cuid.SetConverged(false)
+			if err := event.Error; err != nil {
+				return errwrap.Wrapf(err, "Unknown %s[%s] watcher error", obj.Kind(), obj.GetName())
+			}
+			send = true
+			obj.StateOK(false) // dirty
+
+		case event := <-obj.Events():
+			cuid.SetConverged(false)
+			// we avoid sending events on unpause
+			if exit, send = obj.ReadEvent(&event); exit {
+				return nil // exit
+			}
+
+		case <-cuid.ConvergedTimer():
+			cuid.SetConverged(true) // converged!
+			continue
+		}
+
+		// do all our event sending all together to avoid duplicate msgs
+		if send {
+			send = false
+			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
+				return err // we exit or bubble up a NACK...
+			}
+		}
+	}
+}
+
+// CheckApply method for Password resource. Does nothing, returns happy!
+func (obj *PasswordRes) CheckApply(apply bool) (checkOK bool, err error) {
+
+	var refresh = obj.Refresh() // do we have a pending reload to apply?
+	var exists = true           // does the file (aka the token) exist?
+	var generate bool           // do we need to generate a new password?
+	var write bool              // do we need to write out to disk?
+
+	password, err := obj.read() // password might be empty if just a token
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return false, errwrap.Wrapf(err, "unknown read error")
+		}
+		exists = false
+	}
+
+	if exists {
+		if err := obj.check(password); err != nil {
+			if !obj.CheckRecovery {
+				return false, errwrap.Wrapf(err, "check failed")
+			}
+			log.Printf("%s[%s]: Integrity check failed", obj.Kind(), obj.GetName())
+			generate = true // okay to build a new one
+			write = true    // make sure to write over the old one
+		}
+	} else { // doesn't exist, write one
+		write = true
+	}
+
+	// if we previously had !obj.Saved, and now we want it, we re-generate!
+	if refresh || !exists || (obj.Saved && password == "") {
+		generate = true
+	}
+
+	// stored password isn't consistent with memory
+	if p := obj.Password; obj.Saved && (p != nil && *p != password) {
+		write = true
+	}
+
+	if !refresh && exists && !generate && !write { // nothing to do, done!
+		return true, nil
+	}
+	// a refresh was requested, the token doesn't exist, or the check failed
+
+	if !apply {
+		return false, nil
+	}
+
+	if generate {
+		// we'll need to write this out...
+		if obj.Saved || (!obj.Saved && password != "") {
+			write = true
+		}
+		// generate the actual password
+		var err error
+		log.Printf("%s[%s]: Generating new password...", obj.Kind(), obj.GetName())
+		if password, err = obj.generate(); err != nil { // generate one!
+			return false, errwrap.Wrapf(err, "could not generate password")
+		}
+	}
+
+	obj.Password = &password // save in memory
+
+	var output string // the string to write out
+
+	// if memory value != value on disk, save it
+	if write {
+		if obj.Saved { // save password as clear text
+			// TODO: would it make sense to encrypt this password?
+			output = password
+		}
+		// write either an empty token, or the password
+		log.Printf("%s[%s]: Writing password token...", obj.Kind(), obj.GetName())
+		if _, err := obj.write(output); err != nil {
+			return false, errwrap.Wrapf(err, "can't write to file")
+		}
+	}
+
+	return false, nil
+}
+
+// PasswordUID is the UID struct for PasswordRes.
+type PasswordUID struct {
+	BaseUID
+	name string
+}
+
+// AutoEdges returns the AutoEdge interface. In this case no autoedges are used.
+func (obj *PasswordRes) AutoEdges() AutoEdge {
+	return nil
+}
+
+// GetUIDs includes all params to make a unique identification of this object.
+// Most resources only return one, although some resources can return multiple.
+func (obj *PasswordRes) GetUIDs() []ResUID {
+	x := &PasswordUID{
+		BaseUID: BaseUID{name: obj.GetName(), kind: obj.Kind()},
+		name:    obj.Name,
+	}
+	return []ResUID{x}
+}
+
+// GroupCmp returns whether two resources can be grouped together or not.
+func (obj *PasswordRes) GroupCmp(r Res) bool {
+	_, ok := r.(*PasswordRes)
+	if !ok {
+		return false
+	}
+	return false // TODO: this is doable, but probably not very useful
+	// TODO: it could be useful to group our tokens into a single write, and
+	// as a result, we save inotify watches too!
+}
+
+// Compare two resources and return if they are equivalent.
+func (obj *PasswordRes) Compare(res Res) bool {
+	switch res.(type) {
+	// we can only compare PasswordRes to others of the same resource
+	case *PasswordRes:
+		res := res.(*PasswordRes)
+		if !obj.BaseRes.Compare(res) { // call base Compare
+			return false
+		}
+
+		if obj.Name != res.Name {
+			return false
+		}
+		if obj.Length != res.Length {
+			return false
+		}
+		// TODO: we *could* optimize by allowing CheckApply to move from
+		// saved->!saved, by removing the file, but not likely worth it!
+		if obj.Saved != res.Saved {
+			return false
+		}
+		if obj.CheckRecovery != res.CheckRecovery {
+			return false
+		}
+	default:
+		return false
+	}
+	return true
+}

resources/pkg.go

@@ -15,131 +15,134 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package main
+package resources
 
 import (
-	//"packagekit" // TODO
 	"encoding/gob"
-	"errors"
 	"fmt"
 	"log"
 	"path"
 	"strings"
+
+	"github.com/purpleidea/mgmt/event"
+	"github.com/purpleidea/mgmt/resources/packagekit"
+	"github.com/purpleidea/mgmt/util"
+
+	errwrap "github.com/pkg/errors"
 )
 
 func init() {
 	gob.Register(&PkgRes{})
 }
 
+// PkgRes is a package resource for packagekit.
 type PkgRes struct {
 	BaseRes          `yaml:",inline"`
 	State            string `yaml:"state"`            // state: installed, uninstalled, newest, <version>
 	AllowUntrusted   bool   `yaml:"allowuntrusted"`   // allow untrusted packages to be installed?
 	AllowNonFree     bool   `yaml:"allownonfree"`     // allow nonfree packages to be found?
 	AllowUnsupported bool   `yaml:"allowunsupported"` // allow unsupported packages to be found?
-	//bus              *Conn    // pk bus connection
+	//bus              *packagekit.Conn    // pk bus connection
 	fileList []string // FIXME: update if pkg changes
 }
 
-// helper function for creating new pkg resources that calls Init()
-func NewPkgRes(name, state string, allowuntrusted, allownonfree, allowunsupported bool) *PkgRes {
+// NewPkgRes is a constructor for this resource. It also calls Init() for you.
+func NewPkgRes(name, state string, allowuntrusted, allownonfree, allowunsupported bool) (*PkgRes, error) {
 	obj := &PkgRes{
 		BaseRes: BaseRes{
 			Name: name,
-			events: make(chan Event),
-			vertex: nil,
 		},
 		State:            state,
 		AllowUntrusted:   allowuntrusted,
 		AllowNonFree:     allownonfree,
 		AllowUnsupported: allowunsupported,
 	}
-	obj.Init()
-	return obj
+	return obj, obj.Init()
 }
 
-func (obj *PkgRes) Init() {
+// Init runs some startup code for this resource.
+func (obj *PkgRes) Init() error {
 	obj.BaseRes.kind = "Pkg"
-	obj.BaseRes.Init() // call base init, b/c we're overriding
+	if err := obj.BaseRes.Init(); err != nil { // call base init, b/c we're overriding
+		return err
+	}
 
-	bus := NewBus()
+	bus := packagekit.NewBus()
 	if bus == nil {
-		log.Fatal("Can't connect to PackageKit bus.")
+		return fmt.Errorf("Can't connect to PackageKit bus.")
 	}
 	defer bus.Close()
 
 	result, err := obj.pkgMappingHelper(bus)
 	if err != nil {
-		// FIXME: return error?
-		log.Fatalf("The pkgMappingHelper failed with: %v.", err)
-		return
+		return errwrap.Wrapf(err, "The pkgMappingHelper failed")
 	}
 
 	data, ok := result[obj.Name] // lookup single package (init does just one)
 	// package doesn't exist, this is an error!
 	if !ok || !data.Found {
-		// FIXME: return error?
-		log.Fatalf("Can't find package named '%s'.", obj.Name)
-		return
+		return fmt.Errorf("Can't find package named '%s'.", obj.Name)
 	}
 
 	packageIDs := []string{data.PackageID} // just one for now
 	filesMap, err := bus.GetFilesByPackageID(packageIDs)
 	if err != nil {
-		// FIXME: return error?
-		log.Fatalf("Can't run GetFilesByPackageID: %v", err)
-		return
+		return errwrap.Wrapf(err, "Can't run GetFilesByPackageID")
 	}
 	if files, ok := filesMap[data.PackageID]; ok {
-		obj.fileList = DirifyFileList(files, false)
+		obj.fileList = util.DirifyFileList(files, false)
 	}
+	return nil
 }
 
-func (obj *PkgRes) Validate() bool {
-
+// Validate checks if the resource data structure was populated correctly.
+func (obj *PkgRes) Validate() error {
 	if obj.State == "" {
-		return false
+		return fmt.Errorf("State cannot be empty!")
 	}
 
-	return true
+	return nil
 }
 
-// use UpdatesChanged signal to watch for changes
+// Watch is the primary listener for this resource and it outputs events.
+// It uses the PackageKit UpdatesChanged signal to watch for changes.
 // TODO: https://github.com/hughsie/PackageKit/issues/109
 // TODO: https://github.com/hughsie/PackageKit/issues/110
-func (obj *PkgRes) Watch(processChan chan Event) {
-	if obj.IsWatching() {
-		return
-	}
-	obj.SetWatching(true)
-	defer obj.SetWatching(false)
+func (obj *PkgRes) Watch(processChan chan event.Event) error {
+	cuid := obj.Converger() // get the converger uid used to report status
 
-	bus := NewBus()
+	bus := packagekit.NewBus()
 	if bus == nil {
-		log.Fatal("Can't connect to PackageKit bus.")
+		return fmt.Errorf("Can't connect to PackageKit bus.")
 	}
 	defer bus.Close()
 
 	ch, err := bus.WatchChanges()
 	if err != nil {
-		log.Fatalf("Error adding signal match: %v", err)
+		return errwrap.Wrapf(err, "Error adding signal match")
+	}
+
+	// notify engine that we're running
+	if err := obj.Running(processChan); err != nil {
+		return err // bubble up a NACK...
 	}
 
 	var send = false // send event?
 	var exit = false
-	var dirty = false
 
 	for {
-		if DEBUG {
-			log.Printf("%v: Watching...", obj.fmtNames(obj.getNames()))
+		if obj.debug {
+			log.Printf("%s: Watching...", obj.fmtNames(obj.getNames()))
 		}
 
-		obj.SetState(resStateWatching) // reset
+		obj.SetState(ResStateWatching) // reset
 		select {
 		case event := <-ch:
+			cuid.SetConverged(false)
+
 			// FIXME: ask packagekit for info on what packages changed
-			if DEBUG {
-				log.Printf("%v: Event: %v", obj.fmtNames(obj.getNames()), event.Name)
+			if obj.debug {
+				log.Printf("%s: Event: %v", obj.fmtNames(obj.getNames()), event.Name)
 			}
 
 			// since the chan is buffered, remove any supplemental
@@ -148,34 +151,27 @@ func (obj *PkgRes) Watch(processChan chan Event) {
 				<-ch // discard
 			}
 
-			obj.SetConvergedState(resConvergedNil)
 			send = true
-			dirty = true
+			obj.StateOK(false) // dirty
 
-		case event := <-obj.events:
-			obj.SetConvergedState(resConvergedNil)
+		case event := <-obj.Events():
+			cuid.SetConverged(false)
 			if exit, send = obj.ReadEvent(&event); exit {
-				return // exit
+				return nil // exit
 			}
-			//dirty = false // these events don't invalidate state
+			//obj.StateOK(false) // these events don't invalidate state
 
-		case _ = <-TimeAfterOrBlock(obj.ctimeout):
-			obj.SetConvergedState(resConvergedTimeout)
-			obj.converged <- true
+		case <-cuid.ConvergedTimer():
+			cuid.SetConverged(true) // converged!
 			continue
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
 			send = false
-			// only invalid state on certain types of events
-			if dirty {
-				dirty = false
-				obj.isStateOK = false // something made state dirty
+			if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
+				return err // we exit or bubble up a NACK...
 			}
-			resp := NewResp()
-			processChan <- Event{eventNil, resp, "", true} // trigger process
-			resp.ACKWait()                                 // wait for the ACK()
 		}
 	}
 }
@@ -198,9 +194,9 @@ func (obj *PkgRes) getNames() []string {
 // pretty print for header values
 func (obj *PkgRes) fmtNames(names []string) string {
 	if len(obj.GetGroup()) > 0 { // grouped elements
-		return fmt.Sprintf("%v[autogroup:(%v)]", obj.Kind(), strings.Join(names, ","))
+		return fmt.Sprintf("%s[autogroup:(%v)]", obj.Kind(), strings.Join(names, ","))
 	}
-	return fmt.Sprintf("%v[%v]", obj.Kind(), obj.GetName())
+	return fmt.Sprintf("%s[%s]", obj.Kind(), obj.GetName())
 }
 
 func (obj *PkgRes) groupMappingHelper() map[string]string {
@@ -209,7 +205,7 @@ func (obj *PkgRes) groupMappingHelper() map[string]string {
 		for _, x := range g {
 			pkg, ok := x.(*PkgRes) // convert from Res
 			if !ok {
-				log.Fatalf("Grouped member %v is not a %v", x, obj.Kind())
+				log.Fatalf("Grouped member %v is not a %s", x, obj.Kind())
 			}
 			result[pkg.Name] = pkg.State
 		}
@@ -217,67 +213,61 @@ func (obj *PkgRes) groupMappingHelper() map[string]string {
 	return result
 }
 
-func (obj *PkgRes) pkgMappingHelper(bus *Conn) (map[string]*PkPackageIDActionData, error) {
+func (obj *PkgRes) pkgMappingHelper(bus *packagekit.Conn) (map[string]*packagekit.PkPackageIDActionData, error) {
 	packageMap := obj.groupMappingHelper()   // get the grouped values
 	packageMap[obj.Name] = obj.State         // key is pkg name, value is pkg state
 	var filter uint64                        // initializes at the "zero" value of 0
-	filter += PK_FILTER_ENUM_ARCH          // always search in our arch (optional!)
+	filter += packagekit.PK_FILTER_ENUM_ARCH // always search in our arch (optional!)
 	// we're requesting latest version, or to narrow down install choices!
 	if obj.State == "newest" || obj.State == "installed" {
 		// if we add this, we'll still see older packages if installed
 		// this is an optimization, and is *optional*, this logic is
 		// handled inside of PackagesToPackageIDs now automatically!
-		filter += PK_FILTER_ENUM_NEWEST // only search for newest packages
+		filter += packagekit.PK_FILTER_ENUM_NEWEST // only search for newest packages
 	}
 	if !obj.AllowNonFree {
-		filter += PK_FILTER_ENUM_FREE
+		filter += packagekit.PK_FILTER_ENUM_FREE
 	}
 	if !obj.AllowUnsupported {
-		filter += PK_FILTER_ENUM_SUPPORTED
+		filter += packagekit.PK_FILTER_ENUM_SUPPORTED
 	}
-	result, e := bus.PackagesToPackageIDs(packageMap, filter)
-	if e != nil {
-		return nil, fmt.Errorf("Can't run PackagesToPackageIDs: %v", e)
+	result, err := bus.PackagesToPackageIDs(packageMap, filter)
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "Can't run PackagesToPackageIDs")
 	}
 	return result, nil
 }
 
-func (obj *PkgRes) CheckApply(apply bool) (stateok bool, err error) {
-	log.Printf("%v: CheckApply(%t)", obj.fmtNames(obj.getNames()), apply)
+// CheckApply checks the resource state and applies the resource if the bool
+// input is true. It returns error info and if the state check passed or not.
+func (obj *PkgRes) CheckApply(apply bool) (checkOK bool, err error) {
+	log.Printf("%s: Check", obj.fmtNames(obj.getNames()))
 
-	if obj.State == "" { // TODO: Validate() should replace this check!
-		log.Fatalf("%v: Package state is undefined!", obj.fmtNames(obj.getNames()))
-	}
-
-	if obj.isStateOK { // cache the state
-		return true, nil
-	}
-
-	bus := NewBus()
+	bus := packagekit.NewBus()
 	if bus == nil {
-		return false, errors.New("Can't connect to PackageKit bus.")
+		return false, fmt.Errorf("Can't connect to PackageKit bus.")
 	}
 	defer bus.Close()
 
 	result, err := obj.pkgMappingHelper(bus)
 	if err != nil {
-		return false, fmt.Errorf("The pkgMappingHelper failed with: %v.", err)
+		return false, errwrap.Wrapf(err, "The pkgMappingHelper failed")
 	}
 
 	packageMap := obj.groupMappingHelper() // map[string]string
 	packageList := []string{obj.Name}
-	packageList = append(packageList, StrMapKeys(packageMap)...)
+	packageList = append(packageList, util.StrMapKeys(packageMap)...)
 	//stateList := []string{obj.State}
-	//stateList = append(stateList, StrMapValues(packageMap)...)
+	//stateList = append(stateList, util.StrMapValues(packageMap)...)
 
 	// TODO: at the moment, all the states are the same, but
 	// eventually we might be able to drop this constraint!
-	states, err := FilterState(result, packageList, obj.State)
+	states, err := packagekit.FilterState(result, packageList, obj.State)
 	if err != nil {
-		return false, fmt.Errorf("The FilterState method failed with: %v.", err)
+		return false, errwrap.Wrapf(err, "The FilterState method failed")
 	}
 	data, _ := result[obj.Name] // if above didn't error, we won't either!
-	validState := BoolMapTrue(BoolMapValues(states))
+	validState := util.BoolMapTrue(util.BoolMapValues(states))
 
 	// obj.State == "installed" || "uninstalled" || "newest" || "4.2-1.fc23"
 	switch obj.State {
@@ -301,21 +291,21 @@ func (obj *PkgRes) CheckApply(apply bool) (stateok bool, err error) {
 	}
 
 	// apply portion
-	log.Printf("%v: Apply", obj.fmtNames(obj.getNames()))
-	readyPackages, err := FilterPackageState(result, packageList, obj.State)
+	log.Printf("%s: Apply", obj.fmtNames(obj.getNames()))
+	readyPackages, err := packagekit.FilterPackageState(result, packageList, obj.State)
 	if err != nil {
 		return false, err // fail
 	}
 	// these are the packages that actually need their states applied!
-	applyPackages := StrFilterElementsInList(readyPackages, packageList)
-	packageIDs, _ := FilterPackageIDs(result, applyPackages) // would be same err as above
+	applyPackages := util.StrFilterElementsInList(readyPackages, packageList)
+	packageIDs, _ := packagekit.FilterPackageIDs(result, applyPackages) // would be same err as above
 
 	var transactionFlags uint64 // initializes at the "zero" value of 0
 	if !obj.AllowUntrusted {    // allow
-		transactionFlags += PK_TRANSACTION_FLAG_ENUM_ONLY_TRUSTED
+		transactionFlags += packagekit.PK_TRANSACTION_FLAG_ENUM_ONLY_TRUSTED
 	}
 	// apply correct state!
-	log.Printf("%v: Set: %v...", obj.fmtNames(StrListIntersection(applyPackages, obj.getNames())), obj.State)
+	log.Printf("%s: Set: %v...", obj.fmtNames(util.StrListIntersection(applyPackages, obj.getNames())), obj.State)
 	switch obj.State {
 	case "uninstalled": // run remove
 		// NOTE: packageID is different than when installed, because now
@@ -333,20 +323,20 @@ func (obj *PkgRes) CheckApply(apply bool) (stateok bool, err error) {
 	if err != nil {
 		return false, err // fail
 	}
-	log.Printf("%v: Set: %v success!", obj.fmtNames(StrListIntersection(applyPackages, obj.getNames())), obj.State)
+	log.Printf("%s: Set: %v success!", obj.fmtNames(util.StrListIntersection(applyPackages, obj.getNames())), obj.State)
 	return false, nil // success
 }
 
-type PkgUUID struct {
-	BaseUUID
+// PkgUID is the UID struct for PkgRes.
+type PkgUID struct {
+	BaseUID
 	name  string // pkg name
 	state string // pkg state or "version"
 }
 
-// if and only if they are equivalent, return true
-// if they are not equivalent, return false
-func (obj *PkgUUID) IFF(uuid ResUUID) bool {
-	res, ok := uuid.(*PkgUUID)
+// IFF aka if and only if they are equivalent, return true. If not, false.
+func (obj *PkgUID) IFF(uid ResUID) bool {
+	res, ok := uid.(*PkgUID)
 	if !ok {
 		return false
 	}
@@ -354,31 +344,33 @@ func (obj *PkgUUID) IFF(uuid ResUUID) bool {
 	return obj.name == res.name
 }
 
+// PkgResAutoEdges holds the state of the auto edge generator.
 type PkgResAutoEdges struct {
 	fileList   []string
-	svcUUIDs   []ResUUID
+	svcUIDs    []ResUID
 	testIsNext bool   // safety
 	name       string // saved data from PkgRes obj
 	kind       string
 }
 
-func (obj *PkgResAutoEdges) Next() []ResUUID {
+// Next returns the next automatic edge.
+func (obj *PkgResAutoEdges) Next() []ResUID {
 	if obj.testIsNext {
 		log.Fatal("Expecting a call to Test()")
 	}
 	obj.testIsNext = true // set after all the errors paths are past
 
-	// first return any matching svcUUIDs
-	if x := obj.svcUUIDs; len(x) > 0 {
+	// first return any matching svcUIDs
+	if x := obj.svcUIDs; len(x) > 0 {
 		return x
 	}
 
-	var result []ResUUID
-	// return UUID's for whatever is in obj.fileList
+	var result []ResUID
+	// return UID's for whatever is in obj.fileList
 	for _, x := range obj.fileList {
 		var reversed = false // cheat by passing a pointer
-		result = append(result, &FileUUID{
-			BaseUUID: BaseUUID{
+		result = append(result, &FileUID{
+			BaseUID: BaseUID{
 				name:     obj.name,
 				kind:     obj.kind,
 				reversed: &reversed,
@@ -389,17 +381,18 @@ func (obj *PkgResAutoEdges) Next() []ResUUID {
 	return result
 }
 
+// Test gets results of the earlier Next() call, & returns if we should continue!
 func (obj *PkgResAutoEdges) Test(input []bool) bool {
 	if !obj.testIsNext {
 		log.Fatal("Expecting a call to Next()")
 	}
 
-	// ack the svcUUID's...
-	if x := obj.svcUUIDs; len(x) > 0 {
+	// ack the svcUID's...
+	if x := obj.svcUIDs; len(x) > 0 {
 		if y := len(x); y != len(input) {
 			log.Fatalf("Expecting %d value(s)!", y)
 		}
-		obj.svcUUIDs = []ResUUID{} // empty
+		obj.svcUIDs = []ResUID{} // empty
 		obj.testIsNext = false
 		return true
 	}
@@ -422,16 +415,16 @@ func (obj *PkgResAutoEdges) Test(input []bool) bool {
 	var dirs = make([]string, count)
 	done := []string{}
 	for i := 0; i < count; i++ {
-		dir := Dirname(obj.fileList[i]) // dirname of /foo/ should be /
+		dir := util.Dirname(obj.fileList[i]) // dirname of /foo/ should be /
 		dirs[i] = dir
 		if input[i] {
 			done = append(done, dir)
 		}
 	}
-	nodupes := StrRemoveDuplicatesInList(dirs)                // remove duplicates
-	nodones := StrFilterElementsInList(done, nodupes)         // filter out done
-	noempty := StrFilterElementsInList([]string{""}, nodones) // remove the "" from /
-	obj.fileList = RemoveCommonFilePrefixes(noempty)          // magic
+	nodupes := util.StrRemoveDuplicatesInList(dirs)                // remove duplicates
+	nodones := util.StrFilterElementsInList(done, nodupes)         // filter out done
+	noempty := util.StrFilterElementsInList([]string{""}, nodones) // remove the "" from /
+	obj.fileList = util.RemoveCommonFilePrefixes(noempty)          // magic
 
 	if len(obj.fileList) == 0 { // nothing more, don't continue
 		return false
@@ -439,46 +432,49 @@ func (obj *PkgResAutoEdges) Test(input []bool) bool {
 	return true // continue, there are more files!
 }
 
-// produce an object which generates a minimal pkg file optimization sequence
+// AutoEdges produces an object which generates a minimal pkg file optimization
+// sequence of edges.
 func (obj *PkgRes) AutoEdges() AutoEdge {
 	// in contrast with the FileRes AutoEdges() function which contains
 	// more of the mechanics, most of the AutoEdge mechanics for the PkgRes
 	// is contained in the Test() method! This design is completely okay!
 
 	// add matches for any svc resources found in pkg definition!
-	var svcUUIDs []ResUUID
+	var svcUIDs []ResUID
 	for _, x := range ReturnSvcInFileList(obj.fileList) {
 		var reversed = false
-		svcUUIDs = append(svcUUIDs, &SvcUUID{
-			BaseUUID: BaseUUID{
+		svcUIDs = append(svcUIDs, &SvcUID{
+			BaseUID: BaseUID{
 				name:     obj.GetName(),
 				kind:     obj.Kind(),
 				reversed: &reversed,
 			},
-			name: x, // the svc name itself in the SvcUUID object!
+			name: x, // the svc name itself in the SvcUID object!
 		}) // build list
 	}
 
 	return &PkgResAutoEdges{
-		fileList:   RemoveCommonFilePrefixes(obj.fileList), // clean start!
-		svcUUIDs:   svcUUIDs,
+		fileList:   util.RemoveCommonFilePrefixes(obj.fileList), // clean start!
+		svcUIDs:    svcUIDs,
 		testIsNext: false,         // start with Next() call
 		name:       obj.GetName(), // save data for PkgResAutoEdges obj
 		kind:       obj.Kind(),
 	}
 }
 
-// include all params to make a unique identification of this object
-func (obj *PkgRes) GetUUIDs() []ResUUID {
-	x := &PkgUUID{
-		BaseUUID: BaseUUID{name: obj.GetName(), kind: obj.Kind()},
+// GetUIDs includes all params to make a unique identification of this object.
+// Most resources only return one, although some resources can return multiple.
+func (obj *PkgRes) GetUIDs() []ResUID {
+	x := &PkgUID{
+		BaseUID: BaseUID{name: obj.GetName(), kind: obj.Kind()},
 		name:    obj.Name,
 		state:   obj.State,
 	}
-	result := []ResUUID{x}
+	result := []ResUID{x}
 	return result
 }
 
+// GroupCmp returns whether two resources can be grouped together or not.
 // can these two resources be merged ?
 // (aka does this resource support doing so?)
 // will resource allow itself to be grouped _into_ this obj?
@@ -500,10 +496,15 @@ func (obj *PkgRes) GroupCmp(r Res) bool {
 	return true
 }
 
+// Compare two resources and return if they are equivalent.
 func (obj *PkgRes) Compare(res Res) bool {
 	switch res.(type) {
 	case *PkgRes:
 		res := res.(*PkgRes)
+		if !obj.BaseRes.Compare(res) { // call base Compare
+			return false
+		}
+
 		if obj.Name != res.Name {
 			return false
 		}
@@ -525,7 +526,7 @@ func (obj *PkgRes) Compare(res Res) bool {
 	return true
 }
 
-// return a list of svc names for matches like /usr/lib/systemd/system/*.service
+// ReturnSvcInFileList returns a list of svc names for matches like: `/usr/lib/systemd/system/*.service`.
 func ReturnSvcInFileList(fileList []string) []string {
 	result := []string{}
 	for _, x := range fileList {
@@ -537,7 +538,7 @@ func ReturnSvcInFileList(fileList []string) []string {
 		if !strings.HasSuffix(basename, ".service") {
 			continue
 		}
-		if s := strings.TrimSuffix(basename, ".service"); !StrInList(s, result) {
+		if s := strings.TrimSuffix(basename, ".service"); !util.StrInList(s, result) {
 			result = append(result, s)
 		}
 	}

resources/refresh.go

@@ -0,0 +1,104 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strings"
+
+	errwrap "github.com/pkg/errors"
+)
+
+// Refresh returns the pending state of a notification. It should only be called
+// in the CheckApply portion of a resource where a refresh should be acted upon.
+func (obj *BaseRes) Refresh() bool {
+	return obj.refresh
+}
+
+// SetRefresh sets the pending state of a notification. It should only be called
+// by the mgmt engine.
+func (obj *BaseRes) SetRefresh(b bool) {
+	obj.refresh = b
+}
+
+// StatefulBool is an interface for storing a boolean flag in a permanent spot.
+type StatefulBool interface {
+	Get() (bool, error) // get value of token
+	Set() error         // set token to true
+	Del() error         // rm token if it exists
+}
+
+// DiskBool stores a boolean variable on disk for stateful access across runs.
+// The absence of the path is treated as false. If the path contains a special
+// value, then it is treated as true. All the other non-error cases are false.
+type DiskBool struct {
+	Path string // path to token
+}
+
+// str returns the string data which represents true (aka set).
+func (obj *DiskBool) str() string {
+	const TrueToken = "true"
+	const newline = "\n"
+	return TrueToken + newline
+}
+
+// Get returns if the boolean setting, if no error reading the value occurs.
+func (obj *DiskBool) Get() (bool, error) {
+	file, err := os.Open(obj.Path) // open a handle to read the file
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil // no token means value is false
+		}
+		return false, errwrap.Wrapf(err, "could not read token")
+	}
+	defer file.Close()
+	data, err := ioutil.ReadAll(file)
+	if err != nil {
+		return false, errwrap.Wrapf(err, "could not read from file")
+	}
+	return strings.TrimSpace(string(data)) == strings.TrimSpace(obj.str()), nil
+}
+
+// Set stores the true boolean value, if no error setting the value occurs.
+func (obj *DiskBool) Set() error {
+	file, err := os.Create(obj.Path) // open a handle to create the file
+	if err != nil {
+		return errwrap.Wrapf(err, "can't create file")
+	}
+	defer file.Close()
+	str := obj.str()
+	if c, err := file.Write([]byte(str)); err != nil {
+		return errwrap.Wrapf(err, "error writing to file")
+	} else if l := len(str); c != l {
+		return fmt.Errorf("wrote %d bytes instead of %d", c, l)
+	}
+	return file.Sync() // guarantee it!
+}
+
+// Del stores the false boolean value, if no error clearing the value occurs.
+func (obj *DiskBool) Del() error {
+	if err := os.Remove(obj.Path); err != nil { // remove the file
+		if os.IsNotExist(err) {
+			return nil // no file means this is already fine
+		}
+		return errwrap.Wrapf(err, "could not delete token")
+	}
+	return nil
+}

resources/resources.go

@@ -0,0 +1,471 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// Package resources provides the resource framework and idempotent primitives.
+package resources
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/gob"
+	"fmt"
+	"log"
+	"os"
+	"path"
+
+	// TODO: should each resource be a sub-package?
+	"github.com/purpleidea/mgmt/converger"
+	"github.com/purpleidea/mgmt/event"
+
+	errwrap "github.com/pkg/errors"
+)
+
+//go:generate stringer -type=ResState -output=resstate_stringer.go
+
+// The ResState type represents the current activity state of each resource.
+type ResState int
+
+// Each ResState should be set properly in the relevant part of the resource.
+const (
+	ResStateNil ResState = iota
+	ResStateWatching
+	ResStateEvent // an event has happened, but we haven't poked yet
+	ResStateCheckApply
+	ResStatePoking
+)
+
+const refreshPathToken = "refresh"
+
+// Data is the set of input values passed into the pgraph for the resources.
+type Data struct {
+	//Hostname string         // uuid for the host
+	//Noop     bool
+	Converger converger.Converger
+	Prefix    string // the prefix to be used for the pgraph namespace
+	Debug     bool
+	// NOTE: we can add more fields here if needed for the resources.
+}
+
+// ResUID is a unique identifier for a resource, namely it's name, and the kind ("type").
+type ResUID interface {
+	GetName() string
+	Kind() string
+	IFF(ResUID) bool
+
+	Reversed() bool // true means this resource happens before the generator
+}
+
+// The BaseUID struct is used to provide a unique resource identifier.
+type BaseUID struct {
+	name string // name and kind are the values of where this is coming from
+	kind string
+
+	reversed *bool // piggyback edge information here
+}
+
+// The AutoEdge interface is used to implement the autoedges feature.
+type AutoEdge interface {
+	Next() []ResUID   // call to get list of edges to add
+	Test([]bool) bool // call until false
+}
+
+// MetaParams is a struct will all params that apply to every resource.
+type MetaParams struct {
+	AutoEdge  bool `yaml:"autoedge"`  // metaparam, should we generate auto edges?
+	AutoGroup bool `yaml:"autogroup"` // metaparam, should we auto group?
+	Noop      bool `yaml:"noop"`
+	// NOTE: there are separate Watch and CheckApply retry and delay values,
+	// but I've decided to use the same ones for both until there's a proper
+	// reason to want to do something differently for the Watch errors.
+	Retry int16  `yaml:"retry"` // metaparam, number of times to retry on error. -1 for infinite
+	Delay uint64 `yaml:"delay"` // metaparam, number of milliseconds to wait between retries
+}
+
+// UnmarshalYAML is the custom unmarshal handler for the MetaParams struct. It
+// is primarily useful for setting the defaults.
+func (obj *MetaParams) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type rawMetaParams MetaParams           // indirection to avoid infinite recursion
+	raw := rawMetaParams(DefaultMetaParams) // convert; the defaults go here
+
+	if err := unmarshal(&raw); err != nil {
+		return err
+	}
+
+	*obj = MetaParams(raw) // restore from indirection with type conversion!
+	return nil
+}
+
+// DefaultMetaParams are the defaults to be used for undefined metaparams.
+var DefaultMetaParams = MetaParams{
+	AutoEdge:  true,
+	AutoGroup: true,
+	Noop:      false,
+	Retry:     0, // TODO: is this a good default?
+	Delay:     0, // TODO: is this a good default?
+}
+
+// The Base interface is everything that is common to all resources.
+// Everything here only needs to be implemented once, in the BaseRes.
+type Base interface {
+	GetName() string // can't be named "Name()" because of struct field
+	SetName(string)
+	SetKind(string)
+	Kind() string
+	Meta() *MetaParams
+	Events() chan event.Event
+	AssociateData(*Data)
+	IsWatching() bool
+	SetWatching(bool)
+	RegisterConverger()
+	UnregisterConverger()
+	Converger() converger.ConvergerUID
+	GetState() ResState
+	SetState(ResState)
+	DoSend(chan event.Event, string) (bool, error)
+	SendEvent(event.EventName, bool, bool) bool
+	ReadEvent(*event.Event) (bool, bool)   // TODO: optional here?
+	Refresh() bool                         // is there a pending refresh to run?
+	SetRefresh(bool)                       // set the refresh state of this resource
+	SendRecv(Res) (map[string]bool, error) // send->recv data passing function
+	IsStateOK() bool
+	StateOK(b bool)
+	GroupCmp(Res) bool  // TODO: is there a better name for this?
+	GroupRes(Res) error // group resource (arg) into self
+	IsGrouped() bool    // am I grouped?
+	SetGrouped(bool)    // set grouped bool
+	GetGroup() []Res    // return everyone grouped inside me
+	SetGroup([]Res)
+	VarDir(string) (string, error)
+	Running(chan event.Event) error // notify the engine that Watch started
+	Started() <-chan struct{}       // returns when the resource has started
+	Starter(bool)
+}
+
+// Res is the minimum interface you need to implement to define a new resource.
+type Res interface {
+	Base // include everything from the Base interface
+	Init() error
+	//Validate() error    // TODO: this might one day be added
+	GetUIDs() []ResUID            // most resources only return one
+	Watch(chan event.Event) error // send on channel to signal process() events
+	CheckApply(apply bool) (checkOK bool, err error)
+	AutoEdges() AutoEdge
+	Compare(Res) bool
+	CollectPattern(string) // XXX: temporary until Res collection is more advanced
+}
+
+// BaseRes is the base struct that gets used in every resource.
+type BaseRes struct {
+	Name       string           `yaml:"name"`
+	MetaParams MetaParams       `yaml:"meta"` // struct of all the metaparams
+	Recv       map[string]*Send // mapping of key to receive on from value
+
+	kind      string
+	events    chan event.Event
+	converger converger.Converger // converged tracking
+	cuid      converger.ConvergerUID
+	prefix    string // base prefix for this resource
+	debug     bool
+	state     ResState
+	watching  bool          // is Watch() loop running ?
+	started   chan struct{} // closed when worker is started/running
+	starter   bool          // does this have indegree == 0 ? XXX: usually?
+	isStateOK bool          // whether the state is okay based on events or not
+	isGrouped bool          // am i contained within a group?
+	grouped   []Res         // list of any grouped resources
+	refresh   bool          // does this resource have a refresh to run?
+	//refreshState StatefulBool // TODO: future stateful bool
+}
+
+// UIDExistsInUIDs wraps the IFF method when used with a list of UID's.
+func UIDExistsInUIDs(uid ResUID, uids []ResUID) bool {
+	for _, u := range uids {
+		if uid.IFF(u) {
+			return true
+		}
+	}
+	return false
+}
+
+// GetName returns the name of the resource.
+func (obj *BaseUID) GetName() string {
+	return obj.name
+}
+
+// Kind returns the kind of resource.
+func (obj *BaseUID) Kind() string {
+	return obj.kind
+}
+
+// IFF looks at two UID's and if and only if they are equivalent, returns true.
+// If they are not equivalent, it returns false.
+// Most resources will want to override this method, since it does the important
+// work of actually discerning if two resources are identical in function.
+func (obj *BaseUID) IFF(uid ResUID) bool {
+	res, ok := uid.(*BaseUID)
+	if !ok {
+		return false
+	}
+	return obj.name == res.name
+}
+
+// Reversed is part of the ResUID interface, and true means this resource
+// happens before the generator.
+func (obj *BaseUID) Reversed() bool {
+	if obj.reversed == nil {
+		log.Fatal("Programming error!")
+	}
+	return *obj.reversed
+}
+
+// Init initializes structures like channels if created without New constructor.
+func (obj *BaseRes) Init() error {
+	if obj.kind == "" {
+		return fmt.Errorf("Resource did not set kind!")
+	}
+	obj.events = make(chan event.Event) // unbuffered chan to avoid stale events
+	obj.started = make(chan struct{})   // closes when started
+	//dir, err := obj.VarDir("")
+	//if err != nil {
+	//	return errwrap.Wrapf(err, "VarDir failed in Init()")
+	//}
+	// TODO: this StatefulBool implementation could be eventually swappable
+	//obj.refreshState = &DiskBool{Path: path.Join(dir, refreshPathToken)}
+	return nil
+}
+
+// GetName is used by all the resources to Get the name.
+func (obj *BaseRes) GetName() string {
+	return obj.Name
+}
+
+// SetName is used to set the name of the resource.
+func (obj *BaseRes) SetName(name string) {
+	obj.Name = name
+}
+
+// SetKind sets the kind. This is used internally for exported resources.
+func (obj *BaseRes) SetKind(kind string) {
+	obj.kind = kind
+}
+
+// Kind returns the kind of resource this is.
+func (obj *BaseRes) Kind() string {
+	return obj.kind
+}
+
+// Meta returns the MetaParams as a reference, which we can then get/set on.
+func (obj *BaseRes) Meta() *MetaParams {
+	return &obj.MetaParams
+}
+
+// Events returns the channel of events to listen on.
+func (obj *BaseRes) Events() chan event.Event {
+	return obj.events
+}
+
+// AssociateData associates some data with the object in question.
+func (obj *BaseRes) AssociateData(data *Data) {
+	obj.converger = data.Converger
+	obj.prefix = data.Prefix
+	obj.debug = data.Debug
+}
+
+// IsWatching tells us if the Worker() function is running.
+func (obj *BaseRes) IsWatching() bool {
+	return obj.watching
+}
+
+// SetWatching stores the status of if the Worker() function is running.
+func (obj *BaseRes) SetWatching(b bool) {
+	obj.watching = b
+}
+
+// RegisterConverger sets up the cuid for the resource. This is a helper
+// function for the engine, and shouldn't be called by the resources directly.
+func (obj *BaseRes) RegisterConverger() {
+	obj.cuid = obj.converger.Register()
+}
+
+// UnregisterConverger tears down the cuid for the resource. This is a helper
+// function for the engine, and shouldn't be called by the resources directly.
+func (obj *BaseRes) UnregisterConverger() {
+	obj.cuid.Unregister()
+}
+
+// Converger returns the ConvergerUID for the resource. This should be called
+// by the Watch method of the resource to set the converged state.
+func (obj *BaseRes) Converger() converger.ConvergerUID {
+	return obj.cuid
+}
+
+// GetState returns the state of the resource.
+func (obj *BaseRes) GetState() ResState {
+	return obj.state
+}
+
+// SetState sets the state of the resource.
+func (obj *BaseRes) SetState(state ResState) {
+	if obj.debug {
+		log.Printf("%s[%s]: State: %v -> %v", obj.Kind(), obj.GetName(), obj.GetState(), state)
+	}
+	obj.state = state
+}
+
+// IsStateOK returns the cached state value.
+func (obj *BaseRes) IsStateOK() bool {
+	return obj.isStateOK
+}
+
+// StateOK sets the cached state value.
+func (obj *BaseRes) StateOK(b bool) {
+	obj.isStateOK = b
+}
+
+// GroupCmp compares two resources and decides if they're suitable for grouping
+// You'll probably want to override this method when implementing a resource...
+func (obj *BaseRes) GroupCmp(res Res) bool {
+	return false // base implementation assumes false, override me!
+}
+
+// GroupRes groups resource (arg) into self.
+func (obj *BaseRes) GroupRes(res Res) error {
+	if l := len(res.GetGroup()); l > 0 {
+		return fmt.Errorf("Res: %v already contains %d grouped resources!", res, l)
+	}
+	if res.IsGrouped() {
+		return fmt.Errorf("Res: %v is already grouped!", res)
+	}
+
+	obj.grouped = append(obj.grouped, res)
+	res.SetGrouped(true) // i am contained _in_ a group
+	return nil
+}
+
+// IsGrouped determines if we are grouped.
+func (obj *BaseRes) IsGrouped() bool { // am I grouped?
+	return obj.isGrouped
+}
+
+// SetGrouped sets a flag to tell if we are grouped.
+func (obj *BaseRes) SetGrouped(b bool) {
+	obj.isGrouped = b
+}
+
+// GetGroup returns everyone grouped inside me.
+func (obj *BaseRes) GetGroup() []Res { // return everyone grouped inside me
+	return obj.grouped
+}
+
+// SetGroup sets the grouped resources into me.
+func (obj *BaseRes) SetGroup(g []Res) {
+	obj.grouped = g
+}
+
+// Compare is the base compare method, which also handles the metaparams cmp.
+func (obj *BaseRes) Compare(res Res) bool {
+	// TODO: should the AutoEdge values be compared?
+	if obj.Meta().AutoEdge != res.Meta().AutoEdge {
+		return false
+	}
+	if obj.Meta().AutoGroup != res.Meta().AutoGroup {
+		return false
+	}
+	if obj.Meta().Noop != res.Meta().Noop {
+		// obj is the existing res, res is the *new* resource
+		// if we go from no-noop -> noop, we can re-use the obj
+		// if we go from noop -> no-noop, we need to regenerate
+		if obj.Meta().Noop { // asymmetrical
+			return false // going from noop to no-noop!
+		}
+	}
+	if obj.Meta().Retry != res.Meta().Retry {
+		return false
+	}
+	if obj.Meta().Delay != res.Meta().Delay {
+		return false
+	}
+	return true
+}
+
+// CollectPattern is used for resource collection.
+func (obj *BaseRes) CollectPattern(pattern string) {
+	// XXX: default method is empty
+}
+
+// VarDir returns the path to a working directory for the resource. It will try
+// and create the directory first, and return an error if this failed.
+func (obj *BaseRes) VarDir(extra string) (string, error) {
+	// Using extra adds additional dirs onto our namespace. An empty extra
+	// adds no additional directories.
+	if obj.prefix == "" {
+		return "", fmt.Errorf("VarDir prefix is empty!")
+	}
+	if obj.Kind() == "" {
+		return "", fmt.Errorf("VarDir kind is empty!")
+	}
+	if obj.GetName() == "" {
+		return "", fmt.Errorf("VarDir name is empty!")
+	}
+
+	// FIXME: is obj.GetName() sufficiently unique to use as a UID here?
+	uid := obj.GetName()
+	p := fmt.Sprintf("%s/", path.Join(obj.prefix, obj.Kind(), uid, extra))
+	if err := os.MkdirAll(p, 0770); err != nil {
+		return "", errwrap.Wrapf(err, "Can't create prefix for %s[%s]", obj.Kind(), obj.GetName())
+	}
+	return p, nil
+}
+
+// Started returns a channel that closes when the resource has started up.
+func (obj *BaseRes) Started() <-chan struct{} { return obj.started }
+
+// Starter sets the starter bool. This defines if a vertex has an indegree of 0.
+// If we have an indegree of 0, we'll need to be a poke initiator in the graph.
+func (obj *BaseRes) Starter(b bool) { obj.starter = b }
+
+// ResToB64 encodes a resource to a base64 encoded string (after serialization)
+func ResToB64(res Res) (string, error) {
+	b := bytes.Buffer{}
+	e := gob.NewEncoder(&b)
+	err := e.Encode(&res) // pass with &
+	if err != nil {
+		return "", fmt.Errorf("Gob failed to encode: %v", err)
+	}
+	return base64.StdEncoding.EncodeToString(b.Bytes()), nil
+}
+
+// B64ToRes decodes a resource from a base64 encoded string (after deserialization)
+func B64ToRes(str string) (Res, error) {
+	var output interface{}
+	bb, err := base64.StdEncoding.DecodeString(str)
+	if err != nil {
+		return nil, fmt.Errorf("Base64 failed to decode: %v", err)
+	}
+	b := bytes.NewBuffer(bb)
+	d := gob.NewDecoder(b)
+	err = d.Decode(&output) // pass with &
+	if err != nil {
+		return nil, fmt.Errorf("Gob failed to decode: %v", err)
+	}
+	res, ok := output.(Res)
+	if !ok {
+		return nil, fmt.Errorf("Output %v is not a Res", res)
+
+	}
+	return res, nil
+}

resources/resources_test.go

@@ -15,7 +15,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package main
+package resources
 
 import (
 	"bytes"
@@ -103,3 +103,71 @@ func TestMiscEncodeDecode2(t *testing.T) {
 		t.Error("The input and output Res values do not match!")
 	}
 }
+
+func TestIFF(t *testing.T) {
+	uid := &BaseUID{name: "/tmp/unit-test"}
+	same := &BaseUID{name: "/tmp/unit-test"}
+	diff := &BaseUID{name: "/tmp/other-file"}
+
+	if !uid.IFF(same) {
+		t.Error("basic resource UIDs with the same name should satisfy each other's IFF condition.")
+	}
+
+	if uid.IFF(diff) {
+		t.Error("basic resource UIDs with different names should NOT satisfy each other's IFF condition.")
+	}
+}
+
+func TestReadEvent(t *testing.T) {
+	res := FileRes{}
+
+	shouldExit := map[eventName]bool{
+		eventStart:    false,
+		eventPoke:     false,
+		eventBackPoke: false,
+		eventExit:     true,
+	}
+	shouldPoke := map[eventName]bool{
+		eventStart:    true,
+		eventPoke:     true,
+		eventBackPoke: true,
+		eventExit:     false,
+	}
+
+	for event := range shouldExit {
+		exit, poke := res.ReadEvent(&Event{Name: event})
+		if exit != shouldExit[event] {
+			t.Errorf("resource.ReadEvent returned wrong exit flag for a %v event (%v, should be %v)",
+				event, exit, shouldExit[event])
+		}
+		if poke != shouldPoke[event] {
+			t.Errorf("resource.ReadEvent returned wrong poke flag for a %v event (%v, should be %v)",
+				event, poke, shouldPoke[event])
+		}
+	}
+
+	res.Init()
+	res.SetWatching(true)
+
+	// test result when a pause event is followed by start
+	go res.SendEvent(eventStart, false, false)
+	exit, poke := res.ReadEvent(&Event{Name: eventPause})
+	if exit {
+		t.Error("resource.ReadEvent returned wrong exit flag for a pause+start event (true, should be false)")
+	}
+	if poke {
+		t.Error("resource.ReadEvent returned wrong poke flag for a pause+start event (true, should be false)")
+	}
+
+	// test result when a pause event is followed by exit
+	go res.SendEvent(eventExit, false, false)
+	exit, poke = res.ReadEvent(&Event{Name: eventPause})
+	if !exit {
+		t.Error("resource.ReadEvent returned wrong exit flag for a pause+start event (false, should be true)")
+	}
+	if poke {
+		t.Error("resource.ReadEvent returned wrong poke flag for a pause+start event (true, should be false)")
+	}
+
+	// TODO: create a wrapper API around log, so that Fatals can be mocked and tested
+}

resources/sendrecv.go

@@ -0,0 +1,216 @@
+// Mgmt
+// Copyright (C) 2013-2016+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"fmt"
+	"log"
+	"reflect"
+
+	"github.com/purpleidea/mgmt/event"
+
+	multierr "github.com/hashicorp/go-multierror"
+	errwrap "github.com/pkg/errors"
+)
+
+// SendEvent pushes an event into the message queue for a particular vertex
+func (obj *BaseRes) SendEvent(ev event.EventName, sync bool, activity bool) bool {
+	// TODO: isn't this race-y ?
+	if !obj.IsWatching() { // element has already exited
+		return false // if we don't return, we'll block on the send
+	}
+	if !sync {
+		obj.events <- event.Event{Name: ev, Resp: nil, Msg: "", Activity: activity}
+		return true
+	}
+
+	resp := event.NewResp()
+	obj.events <- event.Event{Name: ev, Resp: resp, Msg: "", Activity: activity}
+	resp.ACKWait() // waits until true (nil) value
+	return true
+}
+
+// DoSend sends off an event, but doesn't block the incoming event queue.
+func (obj *BaseRes) DoSend(processChan chan event.Event, comment string) (exit bool, err error) {
+	resp := event.NewResp()
+	processChan <- event.Event{Name: event.EventNil, Resp: resp, Activity: false, Msg: comment} // trigger process
+	e := resp.Wait()
+	return false, e // XXX: at the moment, we don't use the exit bool.
+}
+
+// ReadEvent processes events when a select gets one, and handles the pause
+// code too! The return values specify if we should exit and poke respectively.
+func (obj *BaseRes) ReadEvent(ev *event.Event) (exit, send bool) {
+	ev.ACK()
+	var poke bool
+	// ensure that a CheckApply runs by sending with a dirty state...
+	if ev.GetActivity() { // if previous node did work, and we were notified...
+		//obj.StateOK(false) // not necessarily
+		poke = true // poke!
+		// XXX: this should be elsewhere in case Watch isn't used (eg: Polling instead...)
+		// XXX: unless this is used in our "fallback" polling implementation???
+		//obj.SetRefresh(true) // TODO: is this redundant?
+	}
+
+	switch ev.Name {
+	case event.EventStart:
+		send = true || poke
+		return
+
+	case event.EventPoke:
+		send = true || poke
+		return
+
+	case event.EventBackPoke:
+		send = true || poke
+		return // forward poking in response to a back poke!
+
+	case event.EventExit:
+		// FIXME: what do we do if we have a pending refresh (poke) and an exit?
+		return true, false
+
+	case event.EventPause:
+		// wait for next event to continue
+		select {
+		case e, ok := <-obj.Events():
+			if !ok { // shutdown
+				return true, false
+			}
+			e.ACK()
+			if e.Name == event.EventExit {
+				return true, false
+			} else if e.Name == event.EventStart { // eventContinue
+				return false, false // don't poke on unpause!
+			} else {
+				// if we get a poke event here, it's a bug!
+				log.Fatalf("%s[%s]: Unknown event: %v, while paused!", obj.Kind(), obj.GetName(), e)
+			}
+		}
+
+	default:
+		log.Fatal("Unknown event: ", ev)
+	}
+	return true, false // required to keep the stupid go compiler happy
+}
+
+// Running is called by the Watch method of the resource once it has started up.
+// This signals to the engine to kick off the initial CheckApply resource check.
+func (obj *BaseRes) Running(processChan chan event.Event) error {
+	obj.StateOK(false)       // assume we're initially dirty
+	cuid := obj.Converger()  // get the converger uid used to report status
+	cuid.SetConverged(false) // a reasonable initial assumption
+	close(obj.started)       // send started signal
+
+	// FIXME: exit return value is unused atm, so ignore it for now...
+	//if exit, err := obj.DoSend(processChan, ""); exit || err != nil {
+	var err error
+	if obj.starter { // vertices of indegree == 0 should send initial pokes
+		_, err = obj.DoSend(processChan, "") // trigger a CheckApply
+	}
+	return err // bubble up any possible error (or nil)
+}
+
+// Send points to a value that a resource will send.
+type Send struct {
+	Res Res    // a handle to the resource which is sending a value
+	Key string // the key in the resource that we're sending
+
+	Changed bool // set to true if this key was updated, read only!
+}
+
+// SendRecv pulls in the sent values into the receive slots. It is called by the
+// receiver and must be given as input the full resource struct to receive on.
+func (obj *BaseRes) SendRecv(res Res) (map[string]bool, error) {
+	if obj.debug {
+		// NOTE: this could expose private resource data like passwords
+		log.Printf("%s[%s]: SendRecv: %+v", obj.Kind(), obj.GetName(), obj.Recv)
+	}
+	var updated = make(map[string]bool) // list of updated keys
+	var err error
+	for k, v := range obj.Recv {
+		updated[k] = false // default
+		v.Changed = false  // reset to the default
+		// send
+		obj1 := reflect.Indirect(reflect.ValueOf(v.Res))
+		type1 := obj1.Type()
+		value1 := obj1.FieldByName(v.Key)
+		kind1 := value1.Kind()
+
+		// recv
+		obj2 := reflect.Indirect(reflect.ValueOf(res)) // pass in full struct
+		type2 := obj2.Type()
+		value2 := obj2.FieldByName(k)
+		kind2 := value2.Kind()
+
+		if obj.debug {
+			log.Printf("Send(%s) has %v: %v", type1, kind1, value1)
+			log.Printf("Recv(%s) has %v: %v", type2, kind2, value2)
+		}
+
+		// i think we probably want the same kind, at least for now...
+		if kind1 != kind2 {
+			e := fmt.Errorf("Kind mismatch between %s[%s]: %s and %s[%s]: %s", v.Res.Kind(), v.Res.GetName(), kind1, obj.Kind(), obj.GetName(), kind2)
+			err = multierr.Append(err, e) // list of errors
+			continue
+		}
+
+		// if the types don't match, we can't use send->recv
+		// TODO: do we want to relax this for string -> *string ?
+		if e := TypeCmp(value1, value2); e != nil {
+			e := errwrap.Wrapf(e, "Type mismatch between %s[%s] and %s[%s]", v.Res.Kind(), v.Res.GetName(), obj.Kind(), obj.GetName())
+			err = multierr.Append(err, e) // list of errors
+			continue
+		}
+
+		// if we can't set, then well this is pointless!
+		if !value2.CanSet() {
+			e := fmt.Errorf("Can't set %s[%s].%s", obj.Kind(), obj.GetName(), k)
+			err = multierr.Append(err, e) // list of errors
+			continue
+		}
+
+		// if we can't interface, we can't compare...
+		if !value1.CanInterface() || !value2.CanInterface() {
+			e := fmt.Errorf("Can't interface %s[%s].%s", obj.Kind(), obj.GetName(), k)
+			err = multierr.Append(err, e) // list of errors
+			continue
+		}
+
+		// if the values aren't equal, we're changing the receiver
+		if !reflect.DeepEqual(value1.Interface(), value2.Interface()) {
+			// TODO: can we catch the panics here in case they happen?
+			value2.Set(value1) // do it for all types that match
+			updated[k] = true  // we updated this key!
+			v.Changed = true   // tag this key as updated!
+			log.Printf("SendRecv: %s[%s].%s -> %s[%s].%s", v.Res.Kind(), v.Res.GetName(), v.Key, obj.Kind(), obj.GetName(), k)
+		}
+	}
+	return updated, err
+}
+
+// TypeCmp compares two reflect values to see if they are the same Kind. It can
+// look into a ptr Kind to see if the underlying pair of ptr's can TypeCmp too!
+func TypeCmp(a, b reflect.Value) error {
+	ta, tb := a.Type(), b.Type()
+	if ta != tb {
+		return fmt.Errorf("Type mismatch: %s != %s", ta, tb)
+	}
+	// NOTE: it seems we don't need to recurse into pointers to sub check!
+
+	return nil // identical Type()'s
+}