modules: Add a small stub for CUPS

This is definitely not perfect, but it's a simple stub which we can
expand on.

.ackrc

@@ -1,3 +1,6 @@
 --ignore-dir=old/
 --ignore-dir=tmp/
 --ignore-dir=vendor/
+--ignore-dir=releases/
+--ignore-dir=rpmbuild/
+--type-set=mcl:ext:mcl

.editorconfig

@@ -23,3 +23,6 @@ indent_style = tab
 
 [*.mcl]
 indent_style = tab
+
+[*.txtar]
+indent_style = tab

.github/FUNDING.yml

@@ -0,0 +1,5 @@
+# You can add one username per supported platform and one custom link.
+custom: "https://paypal.me/purpleidea"
+github: purpleidea
+liberapay: purpleidea
+patreon: purpleidea

.github/settings.yml

@@ -68,6 +68,8 @@ labels:
     color: e11d21
   - name: question
     color: cc317c
+  - name: needinfo
+    color: fbca04
   - name: wontfix
     color: ffffff
 #  - name: first-timers-only

.github/workflows/test.yaml

@@ -0,0 +1,68 @@
+# Docs: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+
+# If the name is omitted, it uses the filename instead.
+#name: Test
+on:
+  # Run on all pull requests.
+  pull_request:
+    #branches:
+    #- master
+  # Run on all pushes.
+  push:
+  # Run daily at 4am.
+  schedule:
+  - cron: 0 4 * * *
+
+jobs:
+  maketest:
+    name: Test (${{ matrix.test_block }}) on ${{ matrix.os }} with golang ${{ matrix.golang_version }}
+    runs-on: ${{ matrix.os }}
+    env:
+      GOPATH: /home/runner/work/mgmt/mgmt/go
+    strategy:
+      matrix:
+        # TODO: Add tip when it's supported: https://github.com/actions/setup-go/issues/21
+        os:
+          - ubuntu-latest
+          # macos tests are currently failing in CI
+          #- macos-latest
+        golang_version:
+          # TODO: add 1.24.x and tip
+          # minimum required and latest published go_version
+          - "1.23"
+        test_block:
+          - basic
+          - shell
+          - race
+      fail-fast: false
+
+    steps:
+      # Do not shallow fetch. The path can't be absolute, so we need to move it
+      # to the expected location later.
+      - name: Clone mgmt
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          fetch-depth: 0
+          path: ./go/src/github.com/purpleidea/mgmt
+
+      - name: Install Go ${{ matrix.golang_version }}
+        uses: actions/setup-go@v4
+        with:
+          go-version: ${{ matrix.golang_version }}
+
+      # Install & configure ruby, fixes gem permissions error
+      - name: Install Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: head
+
+      - name: Install dependencies
+        working-directory: ./go/src/github.com/purpleidea/mgmt
+        run: |
+          make deps
+
+      - name: Run test
+        working-directory: ./go/src/github.com/purpleidea/mgmt
+        run: |
+          TEST_BLOCK="${{ matrix.test_block }}" make test

.gitignore

@@ -7,7 +7,6 @@ old/
 tmp/
 *WIP
 *_stringer.go
-bindata/*.go
 mgmt
 mgmt.static
 # crossbuild artifacts
@@ -17,3 +16,5 @@ rpmbuild/
 releases/
 # vim swap files
 .*.sw[op]
+# prevent `echo foo 2>1` typo errors by making this file read-only
+1

.gitmodules

@@ -1,33 +0,0 @@
-[submodule "vendor/github.com/coreos/etcd"]
-	path = vendor/github.com/coreos/etcd
-	url = https://github.com/coreos/etcd/
-[submodule "vendor/google.golang.org/grpc"]
-	path = vendor/google.golang.org/grpc
-	url = https://github.com/grpc/grpc-go
-[submodule "vendor/github.com/grpc-ecosystem/grpc-gateway"]
-	path = vendor/github.com/grpc-ecosystem/grpc-gateway
-	url = https://github.com/grpc-ecosystem/grpc-gateway
-[submodule "vendor/gopkg.in/fsnotify.v1"]
-	path = vendor/gopkg.in/fsnotify.v1
-	url = https://gopkg.in/fsnotify.v1
-[submodule "vendor/github.com/purpleidea/go-systemd"]
-	path = vendor/github.com/purpleidea/go-systemd
-	url = https://github.com/purpleidea/go-systemd
-[submodule "vendor/honnef.co/go/augeas"]
-	path = vendor/honnef.co/go/augeas
-	url = https://github.com/dominikh/go-augeas/
-[submodule "vendor/github.com/grpc-ecosystem/go-grpc-prometheus"]
-	path = vendor/github.com/grpc-ecosystem/go-grpc-prometheus
-	url = https://github.com/grpc-ecosystem/go-grpc-prometheus
-[submodule "vendor/github.com/ugorji/go"]
-	path = vendor/github.com/ugorji/go
-	url = https://github.com/ugorji/go
-[submodule "vendor/github.com/purpleidea/docker"]
-	path = vendor/github.com/docker/docker
-	url = https://github.com/purpleidea/docker
-[submodule "vendor/github.com/purpleidea/distribution"]
-	path = vendor/github.com/docker/distribution
-	url = https://github.com/purpleidea/distribution
-[submodule "vendor/github.com/purpleidea/go-connections"]
-	path = vendor/github.com/docker/go-connections
-	url = https://github.com/docker/go-connections

.lycheeignore

@@ -0,0 +1,3 @@
+# list URLs that should be excluded for lychee link checher
+https://roidelapluie.be
+https://github.com/purpleidea/mgmt/commit

.travis.yml

@@ -1,57 +0,0 @@
-language: go
-os:
-  - linux
-go:
-  - 1.10.x
-  - 1.11.x
-  - tip
-go_import_path: github.com/purpleidea/mgmt
-sudo: true
-dist: xenial
-# travis requires that you update manually, and provides this key to trigger it
-apt:
-  update: true
-before_install:
-  # print some debug information to help catch the constant travis regressions
-  - if [ -e /etc/apt/sources.list.d/ ]; then sudo ls -l /etc/apt/sources.list.d/; fi
-  # workaround broken travis NO_PUBKEY errors
-  - if [ -e /etc/apt/sources.list.d/rabbitmq_rabbitmq-server.list ]; then sudo rm -f /etc/apt/sources.list.d/rabbitmq_rabbitmq-server.list; fi
-  - if [ -e /etc/apt/sources.list.d/github_git-lfs.list ]; then sudo rm -f /etc/apt/sources.list.d/github_git-lfs.list; fi
-  # as per a number of comments online, this might mitigate some flaky fails...
-  - if [[ "$TRAVIS_OS_NAME" != "osx" ]]; then sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 0C49F3730359A14518585931BC711F9BA15703C6; fi
-  # apt update tends to be flaky in travis, retry up to 3 times on failure
-  # https://docs.travis-ci.com/user/common-build-problems/#travis_retry
-  - if [[ "$TRAVIS_OS_NAME" != "osx" ]]; then travis_retry travis_retry sudo apt update; fi
-  - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
-  - git fetch --unshallow
-install: 'make deps'
-script: 'make test'
-matrix:
-  fast_finish: false
-  allow_failures:
-    - go: 1.11.x
-    - go: tip
-    - os: osx
-  # include only one build for osx for a quicker build as the nr. of these runners are sparse
-  include:
-    - os: osx
-      go: 1.10.x
-
-# the "secure" channel value is the result of running: ./misc/travis-encrypt.sh
-# with a value of: irc.freenode.net#mgmtconfig to eliminate noise from forks...
-notifications:
-  irc:
-    channels:
-      - secure: htcuWAczm3C1zKC9vUfdRzhIXM1vtF+q0cLlQFXK1IQQlk693/pM30Mmf2L/9V2DVDeps+GyLdip0ARXD1DZEJV0lK+Ca1qbHdFP1r4Xv6l5+jaDb5Y88YU5LI8K758QShiZJojuQ1aO2j8xmmt9V0/5y5QwlpPeHbKYBOFPBX3HvlT9DhvwZNKGhBb4qJOEaPVOwq9IkN3DyQ456MHcJ3q3vF9Lb440uTuLsJNof2AbYZH8ZIHCSG2N8tBj2qhJOpWQboYtQJzE2pRaGkGBL4kYcHZSZMXX8sl4cBM1vx/IRUkvBxJUpLJz2gn/eRI+/gr59juZE2K0+FOLlx9dLnX626Y9xSViopBI6JsIoHJDqNC7aGaF2qaYulGYN65VNKVqmghjgt6JLmmiKeH10hYrJMMvt2rms8l4+5iwmCwXvhH/WU9edzk2p5wqERMnostJFEJib0zI3yzLoF0sdJs+veKtagzfayY2d2l7hlmt951IpqqVWldVgWUcQKVvi8gmRarbwFlK+5D7BEnkUDcLNly/cqf7BgEeX6YfF+FiR4pgfOhYvGCD+2q91NgWQXHBCxbyN0be1TVdkXD94f0Lkn94VyEJJ+PkPlG+rPgFwGcjqN4oEGkJeJmES2If05q2Ms1dJLwYQDL3+Py4lNMSdSWj24TzlFVhtwHepuw=
-    template:
-      - "%{repository} (%{commit}: %{author}): %{message}"
-      - "More info : %{build_url}"
-    on_success: always
-    on_failure: always
-    use_notice: false
-    skip_join: false
-  email:
-    recipients:
-      - secure: qNkgP6QLl6VXpFQIxas2wggxvIiOmm1/hGRXm4BXsSFzHsJPvMamA3E1HEC7H+luiWTny1jtGSGgTJPV9CX1LtQV0g0S4ThaAvWuKvk3rXO8IVd++iA/Lh1s1H6JdKM0dJtLqFICawjeci4tOQzSvrM2eCBWqT0UYsrQsGHB6AF31GNAH0Acqd5cYeL+ZpbCN+hQEznAZQ7546N25TwqieI8Lg7nisA+lwYYwsaC2+f5RIeyvvKjQv3wzEdBAQ9CI9WQiTOUBnUnyYxMrdomQ/XGF66QnZy9vq5nEP83IFtuhPvSamL7ceT+yJW0jDyBi8sYEV7On7eXzjyHbiYpF4YHcJrFnf5RyV4kQGd6/SC8iZwK4Is4eyeAjDFTC+JafLajw9R9x9bK43BwlRAWOZxjFKe0cU/BVAjmlz87vHgUho2P41+0a5XfajfU6VhA5QFPK6rNH7W1CnA7D/0LmS0yaqJM1OCrm6LfoZEMhe0DxTJ9uWJbr0x1sYao6q8H4xYk+fyRgoBAr2TxYU7kXx8ThiRdzuQ8izdbojlzTYLe8liZMIsjL0axLsLK7YBWrjJUcDFDjR/DqmVxPrvbVFbCi9ChmBw0WmbJvDY0FV8T8dO8wCjg9JEmprAmWPyq0g/F87LFK4tAZqQFJGjP1qwsR9jdwdNTKeCdY656f/Y=
-    on_failure: change
-    on_success: change

AUTHORS

@@ -6,7 +6,9 @@ This list is sorted alphabetically by first name.
 
 Felix Frank
 James Shubin
+Joe Groocock
 Johan Bloemberg
 Jonathan Gold
 Julien Pivotto
 Paul Morgan
+Samuel Gélineau

COPYING

@@ -1,7 +1,7 @@
                     GNU GENERAL PUBLIC LICENSE
                        Version 3, 29 June 2007
 
- Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 
@@ -645,7 +645,7 @@ the "copyright" line and a pointer to where the full notice is found.
     GNU General Public License for more details.
 
     You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 Also add information on how to contact you by electronic and paper mail.
 
@@ -664,11 +664,11 @@ might be different; for a GUI interface, you would use an "about box".
   You should also get your employer (if you work as a programmer) or school,
 if any, to sign a "copyright disclaimer" for the program, if necessary.
 For more information on this, and how to apply and follow the GNU GPL, see
-<http://www.gnu.org/licenses/>.
+<https://www.gnu.org/licenses/>.
 
   The GNU General Public License does not permit incorporating your program
 into proprietary programs.  If your program is a subroutine library, you
 may consider it more useful to permit linking proprietary applications with
 the library.  If this is what you want to do, use the GNU Lesser General
 Public License instead of this License.  But first, please read
-<http://www.gnu.org/philosophy/why-not-lgpl.html>.
+<https://www.gnu.org/philosophy/why-not-lgpl.html>.

COPYRIGHT

@@ -1,5 +1,5 @@
 Mgmt
-Copyright (C) 2013-2018+ James Shubin and the project contributors
+Copyright (C) James Shubin and the project contributors
 Written by James Shubin <james@shubin.ca> and the project contributors
 
 This program is free software: you can redistribute it and/or modify
@@ -13,4 +13,16 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.
 
 You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+Additional permission under GNU GPL version 3 section 7
+
+If you modify this program, or any covered work, by linking or combining it
+with embedded mcl code and modules (and that the embedded mcl code and
+modules which link with this program, contain a copy of their source code in
+the authoritative form) containing parts covered by the terms of any other
+license, the licensors of this program grant you additional permission to
+convey the resulting work. Furthermore, the licensors of this program grant
+the original author, James Shubin, additional permission to update this
+additional permission if he deems it necessary to achieve the goals of this
+additional permission.

Makefile

@@ -1,5 +1,5 @@
 # Mgmt
-# Copyright (C) 2013-2018+ James Shubin and the project contributors
+# Copyright (C) James Shubin and the project contributors
 # Written by James Shubin <james@shubin.ca> and the project contributors
 #
 # This program is free software: you can redistribute it and/or modify
@@ -13,14 +13,31 @@
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+# Additional permission under GNU GPL version 3 section 7
+#
+# If you modify this program, or any covered work, by linking or combining it
+# with embedded mcl code and modules (and that the embedded mcl code and
+# modules which link with this program, contain a copy of their source code in
+# the authoritative form) containing parts covered by the terms of any other
+# license, the licensors of this program grant you additional permission to
+# convey the resulting work. Furthermore, the licensors of this program grant
+# the original author, James Shubin, additional permission to update this
+# additional permission if he deems it necessary to achieve the goals of this
+# additional permission.
 
 SHELL = /usr/bin/env bash
-.PHONY: all art cleanart version program lang path deps run race bindata generate build build-debug crossbuild clean test gofmt yamlfmt format docs rpmbuild mkdirs rpm srpm spec tar upload upload-sources upload-srpms upload-rpms copr tag release
-.SILENT: clean bindata
+.PHONY: all art cleanart version program lang path deps run race generate build build-debug crossbuild clean test gofmt yamlfmt format docs
+.PHONY: rpmbuild mkdirs rpm srpm spec tar upload upload-sources upload-srpms upload-rpms upload-releases copr tag
+.PHONY: mkosi mkosi_fedora-latest mkosi_fedora-older mkosi_stream-latest mkosi_debian-stable mkosi_ubuntu-latest mkosi_archlinux
+.PHONY: release release_test releases_path release_binary_amd64 release_binary_arm64 release_fedora-latest release_fedora-older release_stream-latest release_debian-stable release_ubuntu-latest release_archlinux
+.PHONY: funcgen
+.SILENT: clean
 
 # a large amount of output from this `find`, can cause `make` to be much slower!
 GO_FILES := $(shell find * -name '*.go' -not -path 'old/*' -not -path 'tmp/*')
+MCL_FILES := $(shell find lang/ -name '*.mcl' -not -path 'old/*' -not -path 'tmp/*')
 
 SVERSION := $(or $(SVERSION),$(shell git describe --match '[0-9]*\.[0-9]*\.[0-9]*' --tags --dirty --always))
 VERSION := $(or $(VERSION),$(shell git describe --match '[0-9]*\.[0-9]*\.[0-9]*' --tags --abbrev=0))
@@ -31,6 +48,13 @@ ifeq ($(VERSION),$(SVERSION))
 else
 	RELEASE = untagged
 endif
+# debugging is harder if -trimpath is set, so disable it if env var is set
+# this is force enabled automatically when using the release target
+ifeq ($(MGMT_NOTRIMPATH),true)
+	TRIMPATH =
+else
+	TRIMPATH = -trimpath
+endif
 ARCH = $(uname -m)
 SPEC = rpmbuild/SPECS/$(PROGRAM).spec
 SOURCE = rpmbuild/SOURCES/$(PROGRAM)-$(VERSION).tar.bz2
@@ -39,7 +63,7 @@ SRPM_BASE = $(PROGRAM)-$(VERSION)-$(RELEASE).src.rpm
 RPM = rpmbuild/RPMS/$(PROGRAM)-$(VERSION)-$(RELEASE).$(ARCH).rpm
 USERNAME := $(shell cat ~/.config/copr 2>/dev/null | grep username | awk -F '=' '{print $$2}' | tr -d ' ')
 SERVER = 'dl.fedoraproject.org'
-REMOTE_PATH = 'pub/alt/$(USERNAME)/$(PROGRAM)'
+REMOTE_PATH = '/srv/pub/alt/$(USERNAME)/$(PROGRAM)'
 ifneq ($(GOTAGS),)
 	BUILD_FLAGS = -tags '$(GOTAGS)'
 endif
@@ -48,9 +72,68 @@ GOOSARCHES ?= linux/amd64 linux/ppc64 linux/ppc64le linux/arm64 darwin/amd64
 GOHOSTOS = $(shell go env GOHOSTOS)
 GOHOSTARCH = $(shell go env GOHOSTARCH)
 
-RPM_PKG = releases/$(VERSION)/rpm/mgmt-$(VERSION)-1.x86_64.rpm
-DEB_PKG = releases/$(VERSION)/deb/mgmt_$(VERSION)_amd64.deb
-PACMAN_PKG = releases/$(VERSION)/pacman/mgmt-$(VERSION)-1-x86_64.pkg.tar.xz
+# The underscores separate the prefix name ("TOKEN") the distro ("BINARY",
+# "FEDORA-LATEST", etc...) and the arch ("AMD64"). The distro can have a dash.
+TOKEN_BINARY_AMD64 = $(shell grep -v '#' releases/binary-amd64.release)
+TOKEN_BINARY_ARM64 = $(shell grep -v '#' releases/binary-arm64.release)
+TOKEN_FEDORA-LATEST = $(shell grep -v '#' releases/fedora-latest.release)
+TOKEN_FEDORA-OLDER = $(shell grep -v '#' releases/fedora-older.release)
+TOKEN_STREAM-LATEST = $(shell grep -v '#' releases/stream-latest.release)
+TOKEN_DEBIAN-STABLE = $(shell grep -v '#' releases/debian-stable.release)
+TOKEN_UBUNTU-LATEST = $(shell grep -v '#' releases/ubuntu-latest.release)
+TOKEN_ARCHLINUX = $(shell grep -v '#' releases/archlinux.release)
+
+FILE_BINARY_AMD64 = mgmt-linux-amd64-$(VERSION)
+FILE_BINARY_ARM64 = mgmt-linux-arm64-$(VERSION)
+# TODO: add ARCH onto these at the end, eg _AMD64
+FILE_FEDORA-LATEST = mgmt-$(TOKEN_FEDORA-LATEST)-$(VERSION)-1.x86_64.rpm
+FILE_FEDORA-OLDER = mgmt-$(TOKEN_FEDORA-OLDER)-$(VERSION)-1.x86_64.rpm
+FILE_STREAM-LATEST = mgmt-$(TOKEN_STREAM-LATEST)-$(VERSION)-1.x86_64.rpm
+FILE_DEBIAN-STABLE = mgmt_$(TOKEN_DEBIAN-STABLE)_$(VERSION)_amd64.deb
+FILE_UBUNTU-LATEST = mgmt_$(TOKEN_UBUNTU-LATEST)_$(VERSION)_amd64.deb
+FILE_ARCHLINUX = mgmt-$(TOKEN_ARCHLINUX)-$(VERSION)-1-x86_64.pkg.tar.xz
+
+PKG_BINARY_AMD64 = releases/$(VERSION)/$(TOKEN_BINARY_AMD64)/$(FILE_BINARY_AMD64)
+PKG_BINARY_ARM64 = releases/$(VERSION)/$(TOKEN_BINARY_ARM64)/$(FILE_BINARY_ARM64)
+PKG_FEDORA-LATEST = releases/$(VERSION)/$(TOKEN_FEDORA-LATEST)/$(FILE_FEDORA-LATEST)
+PKG_FEDORA-OLDER = releases/$(VERSION)/$(TOKEN_FEDORA-OLDER)/$(FILE_FEDORA-OLDER)
+PKG_STREAM-LATEST = releases/$(VERSION)/$(TOKEN_STREAM-LATEST)/$(FILE_STREAM-LATEST)
+PKG_DEBIAN-STABLE = releases/$(VERSION)/$(TOKEN_DEBIAN-STABLE)/$(FILE_DEBIAN-STABLE)
+PKG_UBUNTU-LATEST = releases/$(VERSION)/$(TOKEN_UBUNTU-LATEST)/$(FILE_UBUNTU-LATEST)
+PKG_ARCHLINUX = releases/$(VERSION)/$(TOKEN_ARCHLINUX)/$(FILE_ARCHLINUX)
+
+DEP_BINARY_AMD64 =
+ifneq ($(TOKEN_BINARY_AMD64),)
+	DEP_BINARY_AMD64 = $(PKG_BINARY_AMD64)
+endif
+DEP_BINARY_ARM64 =
+ifneq ($(TOKEN_BINARY_ARM64),)
+	DEP_BINARY_ARM64 = $(PKG_BINARY_ARM64)
+endif
+DEP_FEDORA-LATEST =
+ifneq ($(TOKEN_FEDORA-LATEST),)
+	DEP_FEDORA-LATEST = $(PKG_FEDORA-LATEST)
+endif
+DEP_FEDORA-OLDER =
+ifneq ($(TOKEN_FEDORA-OLDER),)
+	DEP_FEDORA-OLDER = $(PKG_FEDORA-OLDER)
+endif
+DEP_STREAM-LATEST =
+ifneq ($(TOKEN_STREAM-LATEST),)
+	DEP_STREAM-LATEST = $(PKG_STREAM-LATEST)
+endif
+DEP_DEBIAN-STABLE =
+ifneq ($(TOKEN_DEBIAN-STABLE),)
+	DEP_DEBIAN-STABLE = $(PKG_DEBIAN-STABLE)
+endif
+DEP_UBUNTU-LATEST =
+ifneq ($(TOKEN_UBUNTU-LATEST),)
+	DEP_UBUNTU-LATEST = $(PKG_UBUNTU-LATEST)
+endif
+DEP_ARCHLINUX =
+ifneq ($(TOKEN_ARCHLINUX),)
+	DEP_ARCHLINUX = $(PKG_ARCHLINUX)
+endif
 
 SHA256SUMS = releases/$(VERSION)/SHA256SUMS
 SHA256SUMS_ASC = $(SHA256SUMS).asc
@@ -115,28 +198,21 @@ run: ## run mgmt
 race:
 	find . -maxdepth 1 -type f -name '*.go' -not -name '*_test.go' | xargs go run -race -ldflags "-X main.program=$(PROGRAM) -X main.version=$(SVERSION)"
 
-# generate go files from non-go source
-bindata: ## generate go files from non-go sources
-	@echo "Generating: bindata..."
-	$(MAKE) --quiet -C bindata
-	$(MAKE) --quiet -C lang/funcs
-
 generate:
 	go generate
 
 lang: ## generates the lexer/parser for the language frontend
 	@# recursively run make in child dir named lang
-	@echo "Generating: lang..."
-	$(MAKE) --quiet -C lang
+	@$(MAKE) --quiet -C lang
 
 # build a `mgmt` binary for current host os/arch
 $(PROGRAM): build/mgmt-${GOHOSTOS}-${GOHOSTARCH} ## build an mgmt binary for current host os/arch
 	cp -a $< $@
 
-$(PROGRAM).static: $(GO_FILES)
+$(PROGRAM).static: $(GO_FILES) $(MCL_FILES) go.mod go.sum
 	@echo "Building: $(PROGRAM).static, version: $(SVERSION)..."
 	go generate
-	go build -a -installsuffix cgo -tags netgo -ldflags '-extldflags "-static" -X main.program=$(PROGRAM) -X main.version=$(SVERSION) -s -w' -o $(PROGRAM).static $(BUILD_FLAGS);
+	go build $(TRIMPATH) -a -installsuffix cgo -tags netgo -ldflags '-extldflags "-static" -X main.program=$(PROGRAM) -X main.version=$(SVERSION) -s -w' -o $(PROGRAM).static $(BUILD_FLAGS);
 
 build: LDFLAGS=-s -w ## build a fresh mgmt binary
 build: $(PROGRAM)
@@ -148,24 +224,25 @@ build-debug: $(PROGRAM)
 # extract os and arch from target pattern
 GOOS=$(firstword $(subst -, ,$*))
 GOARCH=$(lastword $(subst -, ,$*))
-build/mgmt-%: $(GO_FILES) | bindata lang
+build/mgmt-%: $(GO_FILES) $(MCL_FILES) go.mod go.sum | lang funcgen
+	@# If you need to run `go mod tidy` then this can trigger.
+	@if [ "$(PKGNAME)" = "" ]; then echo "\$$(PKGNAME) is empty, test with: go list ."; exit 42; fi
 	@echo "Building: $(PROGRAM), os/arch: $*, version: $(SVERSION)..."
-	@# reassigning GOOS and GOARCH to make build command copy/pastable
-	@# go 1.10 requires specifying the package for ldflags
-	@if go version | grep -qE 'go1.9'; then \
-		time env GOOS=${GOOS} GOARCH=${GOARCH} go build -i -ldflags "-X main.program=$(PROGRAM) -X main.version=$(SVERSION) ${LDFLAGS}" -o $@ $(BUILD_FLAGS); \
-	else \
-		time env GOOS=${GOOS} GOARCH=${GOARCH} go build -i -ldflags=$(PKGNAME)="-X main.program=$(PROGRAM) -X main.version=$(SVERSION) ${LDFLAGS}" -o $@ $(BUILD_FLAGS); \
-	fi
+	time env GOOS=${GOOS} GOARCH=${GOARCH} go build $(TRIMPATH) -ldflags=$(PKGNAME)="-X main.program=$(PROGRAM) -X main.version=$(SVERSION) ${LDFLAGS}" -o $@ $(BUILD_FLAGS)
 
 # create a list of binary file names to use as make targets
+# to use this you might want to run something like:
+# GOOSARCHES='linux/arm64' GOTAGS='noaugeas novirt' make crossbuild
+# and the output will end up in build/
 crossbuild_targets = $(addprefix build/mgmt-,$(subst /,-,${GOOSARCHES}))
 crossbuild: ${crossbuild_targets}
 
 clean: ## clean things up
-	$(MAKE) --quiet -C bindata clean
-	$(MAKE) --quiet -C lang/funcs clean
+	$(MAKE) --quiet -C test clean
 	$(MAKE) --quiet -C lang clean
+	$(MAKE) --quiet -C misc/mkosi clean
+	rm -f lang/core/generated_funcs.go || true
+	rm -f lang/core/generated_funcs_test.go || true
 	[ ! -e $(PROGRAM) ] || rm $(PROGRAM)
 	rm -f *_stringer.go	# generated by `go generate`
 	rm -f *_mock.go		# generated by `go generate`
@@ -173,6 +250,8 @@ clean: ## clean things up
 	rm -f build/mgmt-*
 
 test: build ## run tests
+	@# recursively run make in child dir named test
+	@$(MAKE) --quiet -C test
 	./test.sh
 
 # create all test targets for make tab completion (eg: make test-gofmt)
@@ -188,8 +267,8 @@ $(addprefix test-shell-,${test_shell}): test-shell-%: build
 
 gofmt:
 	# TODO: remove gofmt once goimports has a -s option
-	find . -maxdepth 6 -type f -name '*.go' -not -path './old/*' -not -path './tmp/*' -not -path './vendor/*' -exec gofmt -s -w {} \;
-	find . -maxdepth 6 -type f -name '*.go' -not -path './old/*' -not -path './tmp/*' -not -path './vendor/*' -exec goimports -w {} \;
+	find . -maxdepth 9 -type f -name '*.go' -not -path './old/*' -not -path './tmp/*' -not -path './vendor/*' -exec gofmt -s -w {} \;
+	find . -maxdepth 9 -type f -name '*.go' -not -path './old/*' -not -path './tmp/*' -not -path './vendor/*' -exec goimports -w {} \;
 
 yamlfmt:
 	find . -maxdepth 3 -type f -name '*.yaml' -not -path './old/*' -not -path './tmp/*' -not -path './omv.yaml' -exec ruby -e "require 'yaml'; x=YAML.load_file('{}').to_yaml.each_line.map(&:rstrip).join(10.chr)+10.chr; File.open('{}', 'w').write x" \;
@@ -326,6 +405,10 @@ upload-rpms: rpmbuild/RPMS/ rpmbuild/RPMS/SHA256SUMS rpmbuild/RPMS/SHA256SUMS.as
 		rsync -avz --prune-empty-dirs rpmbuild/RPMS/ $(SERVER):$(REMOTE_PATH)/RPMS/; \
 	fi
 
+upload-releases:
+	echo Running releases/ upload...
+	rsync -avz --exclude '.mkdir' --exclude 'mgmt-release.url' releases/ $(SERVER):$(REMOTE_PATH)/releases/
+
 #
 #	copr build
 #
@@ -338,18 +421,105 @@ copr: upload-srpms ## build in copr
 tag: ## tags a new release
 	./misc/tag.sh
 
+#
+#	mkosi
+#
+mkosi: mkosi_fedora-latest mkosi_fedora-older mkosi_stream-latest mkosi_debian-stable mkosi_ubuntu-latest mkosi_archlinux ## builds distro packages via mkosi
+
+mkosi_fedora-latest: releases/$(VERSION)/.mkdir
+	@title='$@' ; echo "Generating: $${title#'mkosi_'} via mkosi..."
+	@title='$@' ; distro=$${title#'mkosi_'} ; ./misc/mkosi/make.sh $${distro} `realpath "releases/$(VERSION)/"`
+
+mkosi_fedora-older: releases/$(VERSION)/.mkdir
+	@title='$@' ; echo "Generating: $${title#'mkosi_'} via mkosi..."
+	@title='$@' ; distro=$${title#'mkosi_'} ; ./misc/mkosi/make.sh $${distro} `realpath "releases/$(VERSION)/"`
+
+mkosi_stream-latest: releases/$(VERSION)/.mkdir
+	@title='$@' ; echo "Generating: $${title#'mkosi_'} via mkosi..."
+	@title='$@' ; distro=$${title#'mkosi_'} ; ./misc/mkosi/make.sh $${distro} `realpath "releases/$(VERSION)/"`
+
+mkosi_debian-stable: releases/$(VERSION)/.mkdir
+	@title='$@' ; echo "Generating: $${title#'mkosi_'} via mkosi..."
+	@title='$@' ; distro=$${title#'mkosi_'} ; ./misc/mkosi/make.sh $${distro} `realpath "releases/$(VERSION)/"`
+
+mkosi_ubuntu-latest: releases/$(VERSION)/.mkdir
+	@title='$@' ; echo "Generating: $${title#'mkosi_'} via mkosi..."
+	@title='$@' ; distro=$${title#'mkosi_'} ; ./misc/mkosi/make.sh $${distro} `realpath "releases/$(VERSION)/"`
+
+mkosi_archlinux: releases/$(VERSION)/.mkdir
+	@title='$@' ; echo "Generating: $${title#'mkosi_'} via mkosi..."
+	@title='$@' ; distro=$${title#'mkosi_'} ; ./misc/mkosi/make.sh $${distro} `realpath "releases/$(VERSION)/"`
+
 #
 #	release
 #
+release: TRIMPATH = -trimpath
 release: releases/$(VERSION)/mgmt-release.url ## generates and uploads a release
 
-releases/$(VERSION)/mgmt-release.url: $(RPM_PKG) $(DEB_PKG) $(PACMAN_PKG) $(SHA256SUMS_ASC)
+releases_path:
+	@#Don't put any other output or dependencies in here or they'll show!
+	@echo "releases/$(VERSION)/"
+
+release_test: $(DEP_BINARY_AMD64) $(DEP_BINARY_ARM64) $(DEP_FEDORA-LATEST) $(DEP_FEDORA-OLDER) $(DEP_STREAM-LATEST) $(DEP_DEBIAN-STABLE) $(DEP_UBUNTU-LATEST) $(DEP_ARCHLINUX) $(SHA256SUMS_ASC)
+	@echo '$$< denotes ‘the first dependency of the current rule’.'
+	@echo '> '"$<"
+	@echo
+	@echo '$$@ denotes ‘the target of the current rule’.'
+	@echo '> '"$@"
+	@echo
+	@echo '$$^ denotes ‘the dependencies of the current rule’.'
+	@echo '> '"$^"
+	@echo
+	@echo '$$* denotes ‘the stem with which the pattern of the current rule matched’.'
+	@echo '> '"$*"
+	@echo
+	@echo "TOKEN_BINARY_AMD64: $(TOKEN_BINARY_AMD64)"
+	@echo "DEP_BINARY_AMD64: $(DEP_BINARY_AMD64)"
+	@echo
+	@echo "TOKEN_BINARY_ARM64: $(TOKEN_BINARY_ARM64)"
+	@echo "DEP_BINARY_ARM64: $(DEP_BINARY_ARM64)"
+	@echo
+	@echo "TOKEN_FEDORA-LATEST: $(TOKEN_FEDORA-LATEST)"
+	@echo "DEP_FEDORA-LATEST: $(DEP_FEDORA-LATEST)"
+	@echo
+	@echo "TOKEN_FEDORA-OLDER: $(TOKEN_FEDORA-OLDER)"
+	@echo "DEP_FEDORA-OLDER: $(DEP_FEDORA-OLDER)"
+	@echo
+	@echo "TOKEN_STREAM-LATEST: $(TOKEN_STREAM-LATEST)"
+	@echo "DEP_STREAM-LATEST: $(DEP_STREAM-LATEST)"
+	@echo
+	@echo "TOKEN_DEBIAN-STABLE: $(TOKEN_DEBIAN-STABLE)"
+	@echo "DEP_DEBIAN-STABLE: $(DEP_DEBIAN-STABLE)"
+	@echo
+	@echo "TOKEN_UBUNTU-LATEST: $(TOKEN_UBUNTU-LATEST)"
+	@echo "DEP_UBUNTU-LATEST: $(DEP_UBUNTU-LATEST)"
+	@echo
+	@echo "TOKEN_ARCHLINUX: $(TOKEN_ARCHLINUX)"
+	@echo "DEP_ARCHLINUX: $(DEP_ARCHLINUX)"
+
+release_binary_amd64: $(PKG_BINARY_AMD64)
+release_binary_arm64: $(PKG_BINARY_ARM64)
+release_fedora-latest: $(PKG_FEDORA-LATEST)
+release_fedora-older: $(PKG_FEDORA-OLDER)
+release_stream-latest: $(PKG_STREAM-LATEST)
+release_debian-stable: $(PKG_DEBIAN-STABLE)
+release_ubuntu-latest: $(PKG_UBUNTU-LATEST)
+release_archlinux: $(PKG_ARCHLINUX)
+
+releases/$(VERSION)/mgmt-release.url: $(DEP_BINARY_AMD64) $(DEP_BINARY_ARM64) $(DEP_FEDORA-LATEST) $(DEP_FEDORA-OLDER) $(DEP_STREAM-LATEST) $(DEP_DEBIAN-STABLE) $(DEP_UBUNTU-LATEST) $(DEP_ARCHLINUX) $(SHA256SUMS_ASC)
+	@echo "Pushing git tag $(VERSION) to origin..."
+	git push origin $(VERSION)
 	@echo "Creating github release..."
 	hub release create \
 		-F <( echo -e "$(VERSION)\n";echo "Verify the signatures of all packages before you use them. The signing key can be downloaded from https://purpleidea.com/contact/#pgp-key to verify the release." ) \
-		-a $(RPM_PKG) \
-		-a $(DEB_PKG) \
-		-a $(PACMAN_PKG) \
+		` [ -e $(PKG_BINARY_AMD64) ] && printf -- "-a $(PKG_BINARY_AMD64)" ` \
+		` [ -e $(PKG_BINARY_ARM64) ] && printf -- "-a $(PKG_BINARY_ARM64)" ` \
+		` [ -e $(PKG_FEDORA-LATEST) ] && printf -- "-a $(PKG_FEDORA-LATEST)" ` \
+		` [ -e $(PKG_FEDORA-OLDER) ] && printf -- "-a $(PKG_FEDORA-OLDER)" ` \
+		` [ -e $(PKG_STREAM-LATEST) ] && printf -- "-a $(PKG_STREAM-LATEST)" ` \
+		` [ -e $(PKG_DEBIAN-STABLE) ] && printf -- "-a $(PKG_DEBIAN-STABLE)" ` \
+		` [ -e $(PKG_UBUNTU-LATEST) ] && printf -- "-a $(PKG_UBUNTU-LATEST)" ` \
+		` [ -e $(PKG_ARCHLINUX) ] && printf -- "-a $(PKG_ARCHLINUX)" ` \
 		-a $(SHA256SUMS_ASC) \
 		$(VERSION) \
 		> releases/$(VERSION)/mgmt-release.url \
@@ -357,32 +527,92 @@ releases/$(VERSION)/mgmt-release.url: $(RPM_PKG) $(DEB_PKG) $(PACMAN_PKG) $(SHA2
 		|| rm -f releases/$(VERSION)/mgmt-release.url
 
 releases/$(VERSION)/.mkdir:
-	mkdir -p releases/$(VERSION)/{deb,rpm,pacman}/ && touch releases/$(VERSION)/.mkdir
+	mkdir -p \
+	` [ "$(TOKEN_BINARY_AMD64)" != "" ] && printf -- "releases/$(VERSION)/$(TOKEN_BINARY_AMD64)/" ` \
+	` [ "$(TOKEN_BINARY_ARM64)" != "" ] && printf -- "releases/$(VERSION)/$(TOKEN_BINARY_ARM64)/" ` \
+	` [ "$(TOKEN_FEDORA-LATEST)" != "" ] && printf -- "releases/$(VERSION)/$(TOKEN_FEDORA-LATEST)/" ` \
+	` [ "$(TOKEN_FEDORA-OLDER)" != "" ] && printf -- "releases/$(VERSION)/$(TOKEN_FEDORA-OLDER)/" ` \
+	` [ "$(TOKEN_STREAM-LATEST)" != "" ] && printf -- "releases/$(VERSION)/$(TOKEN_STREAM-LATEST)/" ` \
+	` [ "$(TOKEN_DEBIAN-STABLE)" != "" ] && printf -- "releases/$(VERSION)/$(TOKEN_DEBIAN-STABLE)/" ` \
+	` [ "$(TOKEN_UBUNTU-LATEST)" != "" ] && printf -- "releases/$(VERSION)/$(TOKEN_UBUNTU-LATEST)/" ` \
+	` [ "$(TOKEN_ARCHLINUX)" != "" ] && printf -- "releases/$(VERSION)/$(TOKEN_ARCHLINUX)/" ` \
+	&& touch releases/$(VERSION)/.mkdir
 
-releases/$(VERSION)/rpm/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir
-	@echo "Generating rpm changelog..."
-	./misc/make-rpm-changelog.sh $(VERSION)
+# These are defined conditionally, since if the token is empty, they warn!
+ifneq ($(TOKEN_BINARY_AMD64),)
+$(PKG_BINARY_AMD64): build/mgmt-linux-amd64 releases/$(VERSION)/.mkdir
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Building: $${distro} package..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; cp -a build/mgmt-linux-amd64 $(PKG_BINARY_AMD64)
+endif
+ifneq ($(TOKEN_BINARY_ARM64),)
+$(PKG_BINARY_ARM64): build/mgmt-linux-arm64 releases/$(VERSION)/.mkdir
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Building: $${distro} package..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; cp -a build/mgmt-linux-arm64 $(PKG_BINARY_ARM64)
+endif
+ifneq ($(TOKEN_FEDORA-LATEST),)
+releases/$(VERSION)/$(TOKEN_FEDORA-LATEST)/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Generating: $${distro} changelog..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/make-rpm-changelog.sh "$${distro}" $(VERSION)
 
-$(RPM_PKG): releases/$(VERSION)/rpm/changelog
-	@echo "Building rpm package..."
-	./misc/fpm-pack.sh rpm $(VERSION) libvirt-devel augeas-devel
+$(PKG_FEDORA-LATEST): releases/$(VERSION)/$(TOKEN_FEDORA-LATEST)/changelog
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Building: $${distro} package..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/fpm-pack.sh $${distro} $(VERSION) "$(FILE_FEDORA-LATEST)" libvirt-devel augeas-devel
+endif
+ifneq ($(TOKEN_FEDORA-OLDER),)
+releases/$(VERSION)/$(TOKEN_FEDORA-OLDER)/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Generating: $${distro} changelog..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/make-rpm-changelog.sh "$${distro}" $(VERSION)
 
-releases/$(VERSION)/deb/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir
-	@echo "Generating deb changelog..."
-	./misc/make-deb-changelog.sh $(VERSION)
+$(PKG_FEDORA-OLDER): releases/$(VERSION)/$(TOKEN_FEDORA-OLDER)/changelog
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Building: $${distro} package..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/fpm-pack.sh $${distro} $(VERSION) "$(FILE_FEDORA-OLDER)" libvirt-devel augeas-devel
+endif
+ifneq ($(TOKEN_STREAM-LATEST),)
+releases/$(VERSION)/$(TOKEN_STREAM-LATEST)/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Generating: $${distro} changelog..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/make-rpm-changelog.sh "$${distro}" $(VERSION)
 
-$(DEB_PKG): releases/$(VERSION)/deb/changelog
-	@echo "Building deb package..."
-	./misc/fpm-pack.sh deb $(VERSION) libvirt-dev libaugeas-dev
+$(PKG_STREAM-LATEST): releases/$(VERSION)/$(TOKEN_STREAM-LATEST)/changelog
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Building: $${distro} package..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/fpm-pack.sh $${distro} $(VERSION) "$(FILE_STREAM-LATEST)" libvirt-devel augeas-devel
+endif
+ifneq ($(TOKEN_DEBIAN-STABLE),)
+releases/$(VERSION)/$(TOKEN_DEBIAN-STABLE)/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Generating: $${distro} changelog..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/make-deb-changelog.sh "$${distro}" $(VERSION)
 
-$(PACMAN_PKG): $(PROGRAM) releases/$(VERSION)/.mkdir
-	@echo "Building pacman package..."
-	./misc/fpm-pack.sh pacman $(VERSION) libvirt augeas
+$(PKG_DEBIAN-STABLE): releases/$(VERSION)/$(TOKEN_DEBIAN-STABLE)/changelog
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Building: $${distro} package..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/fpm-pack.sh $${distro} $(VERSION) "$(FILE_DEBIAN-STABLE)" libvirt-dev libaugeas-dev
+endif
+ifneq ($(TOKEN_UBUNTU-LATEST),)
+releases/$(VERSION)/$(TOKEN_UBUNTU-LATEST)/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Generating: $${distro} changelog..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/make-deb-changelog.sh "$${distro}" $(VERSION)
 
-$(SHA256SUMS): $(RPM_PKG) $(DEB_PKG) $(PACMAN_PKG)
+$(PKG_UBUNTU-LATEST): releases/$(VERSION)/$(TOKEN_UBUNTU-LATEST)/changelog
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Building: $${distro} package..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/fpm-pack.sh $${distro} $(VERSION) "$(FILE_UBUNTU-LATEST)" libvirt-dev libaugeas-dev
+endif
+ifneq ($(TOKEN_ARCHLINUX),)
+$(PKG_ARCHLINUX): $(PROGRAM) releases/$(VERSION)/.mkdir
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Building: $${distro} package..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/fpm-pack.sh $${distro} $(VERSION) "$(FILE_ARCHLINUX)" libvirt augeas
+endif
+
+$(SHA256SUMS): $(DEP_BINARY_AMD64) $(DEP_BINARY_ARM64) $(DEP_FEDORA-LATEST) $(DEP_FEDORA-OLDER) $(DEP_STREAM-LATEST) $(DEP_DEBIAN-STABLE) $(DEP_UBUNTU-LATEST) $(DEP_ARCHLINUX)
 	@# remove the directory separator in the SHA256SUMS file
-	@echo "Generating sha256 sum..."
-	sha256sum $(RPM_PKG) $(DEB_PKG) $(PACMAN_PKG) | awk -F '/| ' '{print $$1"  "$$6}' > $(SHA256SUMS)
+	@echo "Generating: sha256 sum..."
+	sha256sum \
+	` [ -e $(PKG_BINARY_AMD64) ] && printf -- "$(PKG_BINARY_AMD64)" ` \
+	` [ -e $(PKG_BINARY_ARM64) ] && printf -- "$(PKG_BINARY_ARM64)" ` \
+	` [ -e $(PKG_FEDORA-LATEST) ] && printf -- "$(PKG_FEDORA-LATEST)" ` \
+	` [ -e $(PKG_FEDORA-OLDER) ] && printf -- "$(PKG_FEDORA-OLDER)" ` \
+	` [ -e $(PKG_STREAM-LATEST) ] && printf -- "$(PKG_STREAM-LATEST)" ` \
+	` [ -e $(PKG_DEBIAN-STABLE) ] && printf -- "$(PKG_DEBIAN-STABLE)" ` \
+	` [ -e $(PKG_UBUNTU-LATEST) ] && printf -- "$(PKG_UBUNTU-LATEST)" ` \
+	` [ -e $(PKG_ARCHLINUX) ] && printf -- "$(PKG_ARCHLINUX)" ` \
+	| awk -F '/| ' '{print $$1"  "$$6}' > $(SHA256SUMS)
 
 $(SHA256SUMS_ASC): $(SHA256SUMS)
 	@echo "Signing sha256 sum..."
@@ -408,4 +638,10 @@ help: ## show this help screen
 		awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
 	@echo ''
 
+funcgen: lang/core/generated_funcs.go
+
+lang/core/generated_funcs.go: lang/funcs/funcgen/*.go lang/core/funcgen.yaml lang/funcs/funcgen/templates/generated_funcs.go.tpl
+	@echo "Generating: funcs..."
+	@go run `find lang/funcs/funcgen/ -maxdepth 1 -type f -name '*.go' -not -name '*_test.go'` -templates=lang/funcs/funcgen/templates/generated_funcs.go.tpl >/dev/null
+
 # vim: ts=8

README.md

@@ -3,23 +3,80 @@
 [![mgmt!](art/mgmt.png)](art/)
 
 [![Go Report Card](https://goreportcard.com/badge/github.com/purpleidea/mgmt?style=flat-square)](https://goreportcard.com/report/github.com/purpleidea/mgmt)
-[![Build Status](https://img.shields.io/travis/purpleidea/mgmt/master.svg?style=flat-square)](http://travis-ci.org/purpleidea/mgmt)
-[![GoDoc](https://img.shields.io/badge/godoc-reference-5272B4.svg?style=flat-square)](https://godoc.org/github.com/purpleidea/mgmt)
-[![IRC](https://img.shields.io/badge/irc-%23mgmtconfig-orange.svg?style=flat-square)](https://webchat.freenode.net/?channels=#mgmtconfig)
+[![Build Status](https://github.com/purpleidea/mgmt/workflows/.github/workflows/test.yaml/badge.svg)](https://github.com/purpleidea/mgmt/actions/)
+[![GoDoc](https://img.shields.io/badge/godoc-reference-5272B4.svg?style=flat-square)](https://godocs.io/github.com/purpleidea/mgmt)
+[![Matrix](https://img.shields.io/badge/matrix-%23mgmtconfig-orange.svg?style=flat-square)](https://matrix.to/#/#mgmtconfig:matrix.org)
+[![IRC](https://img.shields.io/badge/irc-%23mgmtconfig-orange.svg?style=flat-square)](https://web.libera.chat/?channels=#mgmtconfig)
 [![Patreon](https://img.shields.io/badge/patreon-donate-yellow.svg?style=flat-square)](https://www.patreon.com/purpleidea)
 [![Liberapay](https://img.shields.io/badge/liberapay-donate-yellow.svg?style=flat-square)](https://liberapay.com/purpleidea/donate)
 
+> [!TIP]
+> [Resource reference guide now available!](https://mgmtconfig.com/docs/resources/)
+
+> [!TIP]
+> [Function reference guide now available!](https://mgmtconfig.com/docs/functions/)
+
+## About:
+
+`Mgmt` is a real-time automation tool. It is familiar to existing configuration
+management software, but is drastically more powerful as it can allow you to
+build real-time, closed-loop feedback systems, in a very safe way, and with a
+surprisingly small amount of our `mcl` code. For example, the following code
+will ensure that your file server is set to read-only when it's friday.
+
+```mcl
+import "datetime"
+$is_friday = datetime.weekday(datetime.now()) == "friday"
+file "/srv/files/" {
+	state => $const.res.file.state.exists,
+	mode => if $is_friday { # this updates the mode, the instant it changes!
+		"0550"
+	} else {
+		"0770"
+	},
+}
+```
+
+It can run continuously, intermittently, or on-demand, and in the first case, it
+will guarantee that your system is always in the desired state for that instant!
+In this mode it can run as a decentralized cluster of agents across your
+network, each exchanging information with the others in real-time, to respond to
+your changing needs. For example, if you want to ensure that some resource runs
+on a maximum of two hosts in your cluster, you can specify that as well:
+
+```mcl
+import "sys"
+import "world"
+
+# we'll set a few scheduling options:
+$opts = struct{strategy => "rr", max => 2, ttl => 10,}
+
+# schedule in a particular namespace with options:
+$set = world.schedule("xsched", $opts)
+
+if sys.hostname() in $set {
+	# use your imagination to put something more complex right here...
+	print "i got scheduled" {} # this will run on the chosen machines
+}
+```
+
+As you add and remove hosts from the cluster, the real-time `schedule` function
+will dynamically pick up to two hosts from the available pool. These specific
+functions aren't intrinsic to the core design, and new ones can be easily added.
+
+Please read on if you'd like to learn more...
+
 ## Community:
 
 Come join us in the `mgmt` community!
 
 | Medium | Link |
 |---|---|
-| IRC | [#mgmtconfig](https://webchat.freenode.net/?channels=#mgmtconfig) on Freenode |
+| Matrix | [#mgmtconfig](https://matrix.to/#/#mgmtconfig:matrix.org) on Matrix.org |
+| IRC | [#mgmtconfig](https://web.libera.chat/?channels=#mgmtconfig) on Libera.Chat |
 | Twitter | [@mgmtconfig](https://twitter.com/mgmtconfig) & [#mgmtconfig](https://twitter.com/hashtag/mgmtconfig) |
-| Mailing list | [mgmtconfig-list@redhat.com](https://www.redhat.com/mailman/listinfo/mgmtconfig-list) |
+| Mailing list | [looking for a new home, suggestions welcome](https://gitlab.freedesktop.org/freedesktop/freedesktop/-/issues/1082) |
 | Patreon | [purpleidea](https://www.patreon.com/purpleidea) on Patreon |
-| Liberapay | [purpleidea](https://liberapay.com/purpleidea/donate) on Liberapay |
 
 ## Status:
 
@@ -30,7 +87,7 @@ approach. The project contains an engine and a language.
 
 Mgmt is a fairly new project. It is usable today, but not yet feature complete.
 With your help you'll be able to influence our design and get us to 1.0 sooner!
-Interested developers should read the [quick start guide](docs/quick-start-guide.md).
+Interested users should read the [quick start guide](docs/quick-start-guide.md).
 
 ## Documentation:
 
@@ -38,17 +95,23 @@ Please read, enjoy and help improve our documentation!
 
 | Documentation | Additional Notes |
 |---|---|
-| [quick start guide](docs/quick-start-guide.md) | for mgmt developers |
+| [quick start guide](docs/quick-start-guide.md) | for everyone |
 | [frequently asked questions](docs/faq.md) | for everyone |
 | [general documentation](docs/documentation.md) | for everyone |
+| [resource reference](https://mgmtconfig.com/docs/resources/) | for everyone |
+| [function reference](https://mgmtconfig.com/docs/functions/) | for everyone |
 | [language guide](docs/language-guide.md) | for everyone |
 | [function guide](docs/function-guide.md) | for mgmt developers |
 | [resource guide](docs/resource-guide.md) | for mgmt developers |
 | [style guide](docs/style-guide.md) | for mgmt developers |
+| [contributing guide](docs/contributing.md) | for mgmt contributors |
+| [service API guide](docs/service-guide.md) | for external developers |
 | [godoc API reference](https://godoc.org/github.com/purpleidea/mgmt) | for mgmt developers |
 | [prometheus guide](docs/prometheus.md) | for everyone |
 | [puppet guide](docs/puppet-guide.md) | for puppet sysadmins |
 | [development](docs/development.md) | for mgmt developers |
+| [videos](docs/on-the-web.md) | for everyone |
+| [blogs](docs/on-the-web.md) | for everyone |
 
 ## Questions:
 
@@ -57,22 +120,18 @@ If you have a well phrased question that might benefit others, consider asking
 it by sending a patch to the [FAQ](docs/faq.md) section. I'll merge your
 question, and a patch with the answer!
 
-## Roadmap:
+## Get involved:
 
 Feel free to grab one of the straightforward [#mgmtlove](https://github.com/purpleidea/mgmt/labels/mgmtlove)
 issues if you're a first time contributor to the project or if you're unsure
-about what to hack on!
-Please see: [TODO.md](TODO.md) for a list of upcoming work and TODO items.
-Please get involved by working on one of these items or by suggesting something
-else!
+about what to hack on! Please get involved by working on one of these items or
+by suggesting something else! There are some lower priority issues and harder
+issues available in our [TODO](TODO.md) file. Please have a look.
 
 ## Bugs:
 
 Please set the `DEBUG` constant in [main.go](https://github.com/purpleidea/mgmt/blob/master/main.go)
 to `true`, and post the logs when you report the [issue](https://github.com/purpleidea/mgmt/issues).
-Bonus points if you provide a [shell](https://github.com/purpleidea/mgmt/tree/master/test/shell)
-or [OMV](https://github.com/purpleidea/mgmt/tree/master/test/omv) reproducible
-test case.
 Feel free to read my article on [debugging golang programs](https://purpleidea.com/blog/2016/02/15/debugging-golang-programs/).
 
 ## Patches:
@@ -81,6 +140,6 @@ We'd love to have your patches! Please send them by email, or as a pull request.
 
 ## On the web:
 
-[Read what people are saying and publishing about mgmt!](docs/on-the-web.md)
+[Blog posts and recorded talks about mgmt are listed here!](docs/on-the-web.md)
 
 Happy hacking!

TODO.md

@@ -1,10 +1,18 @@
 # TODO
 
-If you're looking for something to do, look here!
-Let us know if you're working on one of the items.
-If you'd like something to work on, ping @purpleidea and I'll create an issue
-tailored especially for you! Just let me know your approximate golang skill
-level and how many hours you'd like to spend on the patch.
+Here is a TODO list of longstanding items that are either lower-priority, or
+more involved in terms of time, skill-level, and/or motivation.
+
+Please have a look, and let us know if you're working on one of the items. It's
+best to open an issue to track your progress and to discuss any implementation
+questions you might have.
+
+Lastly, if you'd like something different to work on, please ping @purpleidea
+and I'll create an issue tailored especially for your approximate golang skill
+level and available time commitment in terms of hours you'd need to spend on the
+patch.
+
+Happy Hacking!
 
 ## Package resource
 
@@ -19,7 +27,7 @@ level and how many hours you'd like to spend on the patch.
 
 ## Svc resource
 
-- [ ] base resource improvements
+- [ ] refreshonly support [:heart:](https://github.com/purpleidea/mgmt/issues/464)
 
 ## Exec resource
 
@@ -33,33 +41,14 @@ level and how many hours you'd like to spend on the patch.
 
 - [ ] automatic edges to file resource [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
 
-## Virt (libvirt) resource
-
-- [ ] base resource improvements [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
-
-## Net (systemd-networkd) resource
-
-- [ ] base resource
-
-## Nspawn (systemd-nspawn) resource
-
-- [ ] base resource [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
-
-## Mount (systemd-mount) resource
-
-- [ ] base resource [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
-
-## Cron (systemd-timer) resource
-
-- [ ] base resource [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
-
 ## Http resource
 
 - [ ] base resource [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
 
 ## Etcd improvements
 
-- [ ] fix embedded etcd master race
+- [ ] fix etcd race bug that only happens during CI testing (intermittently
+failing test case issue)
 
 ## Torrent/dht file transfer
 
@@ -69,17 +58,33 @@ level and how many hours you'd like to spend on the patch.
 
 - [ ] base plumbing
 
+## Resource improvements
+
+- [ ] more reversible resources implemented
+- [ ] more "cloud" resources
+
 ## Language improvements
 
 - [ ] more core functions
 - [ ] automatic language formatter, ala `gofmt`
 - [ ] gedit/gnome-builder/gtksourceview syntax highlighting
 - [ ] vim syntax highlighting
-- [x] emacs syntax highlighting: see `misc/emacs/`
+- [ ] emacs syntax highlighting: see `misc/emacs/` (needs updating)
+- [ ] exposed $error variable for feedback in the language
+- [ ] improve the printf function to add %[]s, %[]f ([]str, []float) and map,
+struct, nested etc... %v would be nice too!
+- [ ] add line/col/file annotations to AST so we can get locations of errors
+that the parser finds
+- [ ] add more error messages with the `%error` pattern in parser.y
+- [ ] we should have helper functions or language sugar to pull a field out of a
+struct, or a value out of a map, or an index out of a list, etc...
+
+## Engine improvements
+
+- [ ] add a "waiting for func" message in the func engine to notify the user
+about slow functions...
 
 ## Other
 
-- [ ] better error/retry handling
-- [ ] deb package target in Makefile
 - [ ] reproducible builds
 - [ ] add your suggestions!

Vagrantfile

@@ -6,7 +6,7 @@ Vagrant.configure(2) do |config|
 	config.vm.synced_folder ".", "/vagrant", disabled: true
 
 	config.vm.define "mgmt-dev" do |instance|
-		instance.vm.box = "fedora/28-cloud-base"
+		instance.vm.box = "bento/fedora-31"
 	end
 
 	config.vm.provider "virtualbox" do |v|
@@ -23,8 +23,7 @@ Vagrant.configure(2) do |config|
 	config.vm.provision "file", source: "vagrant/mgmt.bashrc", destination: ".mgmt.bashrc"
 	config.vm.provision "file", source: "~/.gitconfig", destination: ".gitconfig"
 
-	# copied from make-deps.sh (with added git)
-	config.vm.provision "shell", inline: "dnf install -y libvirt-devel golang golang-googlecode-tools-stringer hg git make gem"
+	config.vm.provision "shell", inline: "dnf install -y golang git make"
 
 	# set up packagekit
 	config.vm.provision "shell" do |shell|
@@ -39,8 +38,10 @@ Vagrant.configure(2) do |config|
 	script = <<-SCRIPT
 		grep -q 'mgmt\.bashrc' ~/.bashrc || echo '. ~/.mgmt.bashrc' >>~/.bashrc
 		. ~/.mgmt.bashrc
-		go get -u github.com/purpleidea/mgmt
-		cd ~/gopath/src/github.com/purpleidea/mgmt
+		mkdir -p ~/gopath/src/github.com/purpleidea
+		cd ~/gopath/src/github.com/purpleidea
+		git clone https://github.com/purpleidea/mgmt --recursive
+		cd mgmt
 		make deps
 	SCRIPT
 	config.vm.provision "shell" do |shell|

bindata/Makefile

@@ -1,41 +0,0 @@
-# Mgmt
-# Copyright (C) 2013-2018+ James Shubin and the project contributors
-# Written by James Shubin <james@shubin.ca> and the project contributors
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-# The bindata target generates go files from any source defined below. To use
-# the files, import the generated "bindata" package and use:
-# `bytes, err := bindata.Asset("FILEPATH")`
-# where FILEPATH is the path of the original input file relative to `bindata/`.
-# To get a list of files stored in this "bindata" package, you can use:
-# `paths := bindata.AssetNames()` and `paths, err := bindata.AssetDir(name)`
-# to get a list of files with a directory prefix.
-
-.PHONY: build clean
-default: build
-
-build: bindata.go
-
-# add more input files as dependencies at the end here...
-bindata.go: ../COPYING
-	# go-bindata --pkg bindata -o <OUTPUT> <INPUT>
-	go-bindata --pkg bindata -o ./$@ $^
-	# gofmt the output file
-	gofmt -s -w $@
-	@ROOT=$$(dirname "$${BASH_SOURCE}")/.. && $$ROOT/misc/header.sh '$@'
-
-clean:
-	# remove generated bindata.go
-	@ROOT=$$(dirname "$${BASH_SOURCE}")/.. && rm -f bindata.go

cli/cli.go

@@ -0,0 +1,183 @@
+// Mgmt
+// Copyright (C) James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+// Package cli handles all of the core command line parsing. It's the first
+// entry point after the real main function, and it imports and runs our core
+// "lib".
+package cli
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	cliUtil "github.com/purpleidea/mgmt/cli/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
+
+	"github.com/alexflint/go-arg"
+)
+
+func init() {
+	if _, err := arg.NewParser(arg.Config{}, &Args{}); err != nil { // sanity check
+		panic(errwrap.Wrapf(err, "invalid args cli struct"))
+	}
+}
+
+// CLI is the entry point for using mgmt normally from the CLI.
+func CLI(ctx context.Context, data *cliUtil.Data) error {
+	// test for sanity
+	if data == nil {
+		return fmt.Errorf("this CLI was not run correctly")
+	}
+	if data.Program == "" || data.Version == "" {
+		return fmt.Errorf("program was not compiled correctly")
+	}
+	if data.Copying == "" {
+		return fmt.Errorf("program copyrights were removed, can't run")
+	}
+
+	args := Args{}
+	args.version = data.Version // copy this in
+	args.description = data.Tagline
+
+	config := arg.Config{
+		Program: data.Program,
+	}
+	parser, err := arg.NewParser(config, &args)
+	if err != nil {
+		// programming error
+		return errwrap.Wrapf(err, "cli config error")
+	}
+	err = parser.Parse(data.Args[1:]) // XXX: args[0] needs to be dropped
+	if err == arg.ErrHelp {
+		parser.WriteHelp(os.Stdout)
+		return nil
+	}
+	if err == arg.ErrVersion {
+		fmt.Printf("%s\n", data.Version) // byon: bring your own newline
+		return nil
+	}
+	if err != nil {
+		//parser.WriteHelp(os.Stdout) // TODO: is doing this helpful?
+		return cliUtil.CliParseError(err) // consistent errors
+	}
+
+	// display the license
+	if args.License {
+		fmt.Printf("%s", data.Copying) // file comes with a trailing nl
+		return nil
+	}
+
+	if ok, err := args.Run(ctx, data); err != nil {
+		return err
+	} else if ok { // did we activate one of the commands?
+		return nil
+	}
+
+	// print help if no subcommands are set
+	parser.WriteHelp(os.Stdout)
+
+	return nil
+}
+
+// Args is the CLI parsing structure and type of the parsed result. This
+// particular struct is the top-most one.
+type Args struct {
+	// XXX: We cannot have both subcommands and a positional argument.
+	// XXX: I think it's a bug of this library that it can't handle argv[0].
+	//Argv0 string `arg:"positional"`
+
+	License bool `arg:"--license" help:"display the license and exit"`
+
+	RunCmd *RunArgs `arg:"subcommand:run" help:"run code on this machine"`
+
+	DeployCmd *DeployArgs `arg:"subcommand:deploy" help:"deploy code into a cluster"`
+
+	SetupCmd *SetupArgs `arg:"subcommand:setup" help:"setup some bootstrapping tasks"`
+
+	FirstbootCmd *FirstbootArgs `arg:"subcommand:firstboot" help:"run some tasks on first boot"`
+
+	DocsCmd *DocsGenerateArgs `arg:"subcommand:docs" help:"generate documentation"`
+
+	// This never runs, it gets preempted in the real main() function.
+	// XXX: Can we do it nicely with the new arg parser? can it ignore all args?
+	EtcdCmd *EtcdArgs `arg:"subcommand:etcd" help:"run standalone etcd"`
+
+	// version is a private handle for our version string.
+	version string `arg:"-"` // ignored from parsing
+
+	// description is a private handle for our description string.
+	description string `arg:"-"` // ignored from parsing
+}
+
+// Version returns the version string. Implementing this signature is part of
+// the API for the cli library.
+func (obj *Args) Version() string {
+	return obj.version
+}
+
+// Description returns a description string. Implementing this signature is part
+// of the API for the cli library.
+func (obj *Args) Description() string {
+	return obj.description
+}
+
+// Run executes the correct subcommand. It errors if there's ever an error. It
+// returns true if we did activate one of the subcommands. It returns false if
+// we did not. This information is used so that the top-level parser can return
+// usage or help information if no subcommand activates.
+func (obj *Args) Run(ctx context.Context, data *cliUtil.Data) (bool, error) {
+	if cmd := obj.RunCmd; cmd != nil {
+		return cmd.Run(ctx, data)
+	}
+
+	if cmd := obj.DeployCmd; cmd != nil {
+		return cmd.Run(ctx, data)
+	}
+
+	if cmd := obj.SetupCmd; cmd != nil {
+		return cmd.Run(ctx, data)
+	}
+
+	if cmd := obj.FirstbootCmd; cmd != nil {
+		return cmd.Run(ctx, data)
+	}
+
+	if cmd := obj.DocsCmd; cmd != nil {
+		return cmd.Run(ctx, data)
+	}
+
+	// NOTE: we could return true, fmt.Errorf("...") if more than one did
+	return false, nil // nobody activated
+}
+
+// EtcdArgs is the CLI parsing structure and type of the parsed result. This
+// particular one is empty because the `etcd` subcommand is preempted in the
+// real main() function.
+type EtcdArgs struct{}

cli/deploy.go

@@ -0,0 +1,265 @@
+// Mgmt
+// Copyright (C) James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+package cli
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/signal"
+
+	cliUtil "github.com/purpleidea/mgmt/cli/util"
+	"github.com/purpleidea/mgmt/etcd/client"
+	"github.com/purpleidea/mgmt/etcd/deployer"
+	etcdfs "github.com/purpleidea/mgmt/etcd/fs"
+	"github.com/purpleidea/mgmt/gapi"
+	"github.com/purpleidea/mgmt/lib"
+	"github.com/purpleidea/mgmt/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
+
+	git "github.com/go-git/go-git/v5"
+	"github.com/google/uuid"
+)
+
+// DeployArgs is the CLI parsing structure and type of the parsed result. This
+// particular one contains all the common flags for the `deploy` subcommand
+// which all frontends can use.
+type DeployArgs struct {
+	Seeds []string `arg:"--seeds,env:MGMT_SEEDS" help:"default etc client endpoint"`
+	Noop  bool     `arg:"--noop" help:"globally force all resources into no-op mode"`
+	Sema  int      `arg:"--sema" default:"-1" help:"globally add a semaphore to all resources with this lock count"`
+	NoGit bool     `arg:"--no-git" help:"don't look at git commit id for safe deploys"`
+	Force bool     `arg:"--force" help:"force a new deploy, even if the safety chain would break"`
+
+	DeployEmpty      *cliUtil.EmptyArgs      `arg:"subcommand:empty" help:"deploy empty payload"`
+	DeployLang       *cliUtil.LangArgs       `arg:"subcommand:lang" help:"deploy lang (mcl) payload"`
+	DeployYaml       *cliUtil.YamlArgs       `arg:"subcommand:yaml" help:"deploy yaml graph payload"`
+	DeployPuppet     *cliUtil.PuppetArgs     `arg:"subcommand:puppet" help:"deploy puppet graph payload"`
+	DeployLangPuppet *cliUtil.LangPuppetArgs `arg:"subcommand:langpuppet" help:"deploy langpuppet graph payload"`
+}
+
+// Run executes the correct subcommand. It errors if there's ever an error. It
+// returns true if we did activate one of the subcommands. It returns false if
+// we did not. This information is used so that the top-level parser can return
+// usage or help information if no subcommand activates. This particular Run is
+// the run for the main `deploy` subcommand. This always requires a frontend to
+// deploy to the cluster, but if you don't want a graph, you can use the `empty`
+// frontend. The engine backend is agnostic to which frontend is deployed, in
+// fact, you can deploy with multiple different frontends, one after another, on
+// the same engine.
+func (obj *DeployArgs) Run(ctx context.Context, data *cliUtil.Data) (bool, error) {
+	var name string
+	var args interface{}
+	if cmd := obj.DeployEmpty; cmd != nil {
+		name = cliUtil.LookupSubcommand(obj, cmd) // "empty"
+		args = cmd
+	}
+	if cmd := obj.DeployLang; cmd != nil {
+		name = cliUtil.LookupSubcommand(obj, cmd) // "lang"
+		args = cmd
+	}
+	if cmd := obj.DeployYaml; cmd != nil {
+		name = cliUtil.LookupSubcommand(obj, cmd) // "yaml"
+		args = cmd
+	}
+	if cmd := obj.DeployPuppet; cmd != nil {
+		name = cliUtil.LookupSubcommand(obj, cmd) // "puppet"
+		args = cmd
+	}
+	if cmd := obj.DeployLangPuppet; cmd != nil {
+		name = cliUtil.LookupSubcommand(obj, cmd) // "langpuppet"
+		args = cmd
+	}
+
+	// XXX: workaround https://github.com/alexflint/go-arg/issues/239
+	gapiNames := gapi.Names() // list of registered names
+	if l := len(obj.Seeds); name == "" && l > 1 {
+		elem := obj.Seeds[l-2] // second to last element
+		if util.StrInList(elem, gapiNames) {
+			return false, cliUtil.CliParseError(cliUtil.MissingEquals) // consistent errors
+		}
+	}
+
+	fn, exists := gapi.RegisteredGAPIs[name]
+	if !exists {
+		return false, nil // did not activate
+	}
+	gapiObj := fn()
+
+	program, version := data.Program, data.Version
+	Logf := func(format string, v ...interface{}) {
+		data.Flags.Logf("deploy: "+format, v...)
+	}
+
+	// TODO: consider adding a timeout based on an args.Timeout flag ?
+	ctx, cancel := signal.NotifyContext(ctx, os.Interrupt)
+	defer cancel()
+
+	cliUtil.Hello(program, version, data.Flags) // say hello!
+	defer Logf("goodbye!")
+
+	var hash, pHash string
+	if !obj.NoGit {
+		wd, err := os.Getwd()
+		if err != nil {
+			return false, errwrap.Wrapf(err, "could not get current working directory")
+		}
+		repo, err := git.PlainOpen(wd)
+		if err != nil {
+			return false, errwrap.Wrapf(err, "could not open git repo")
+		}
+
+		head, err := repo.Head()
+		if err != nil {
+			return false, errwrap.Wrapf(err, "could not read git HEAD")
+		}
+
+		hash = head.Hash().String() // current commit id
+		Logf("hash: %s", hash)
+
+		lo := &git.LogOptions{
+			From: head.Hash(),
+		}
+		commits, err := repo.Log(lo)
+		if err != nil {
+			return false, errwrap.Wrapf(err, "could not read git log")
+		}
+		if _, err := commits.Next(); err != nil { // skip over HEAD
+			return false, errwrap.Wrapf(err, "could not read HEAD in git log") // weird!
+		}
+		commit, err := commits.Next()
+		if err == nil { // errors are okay, we might be empty
+			pHash = commit.Hash.String() // previous commit id
+		}
+		Logf("previous deploy hash: %s", pHash)
+		if obj.Force {
+			pHash = "" // don't check this :(
+		}
+		if hash == "" {
+			return false, errwrap.Wrapf(err, "could not get git deploy hash")
+		}
+	}
+
+	uniqueid := uuid.New() // panic's if it can't generate one :P
+
+	etcdClient := client.NewClientFromSeedsNamespace(
+		obj.Seeds, // endpoints
+		lib.NS,
+	)
+	if err := etcdClient.Init(); err != nil {
+		return false, errwrap.Wrapf(err, "client Init failed")
+	}
+	defer func() {
+		err := errwrap.Wrapf(etcdClient.Close(), "client Close failed")
+		if err != nil {
+			// TODO: cause the final exit code to be non-zero
+			Logf("client cleanup error: %+v", err)
+		}
+	}()
+
+	simpleDeploy := &deployer.SimpleDeploy{
+		Client: etcdClient,
+		Debug:  data.Flags.Debug,
+		Logf: func(format string, v ...interface{}) {
+			Logf("deploy: "+format, v...)
+		},
+	}
+	if err := simpleDeploy.Init(); err != nil {
+		return false, errwrap.Wrapf(err, "deploy Init failed")
+	}
+	defer func() {
+		err := errwrap.Wrapf(simpleDeploy.Close(), "deploy Close failed")
+		if err != nil {
+			// TODO: cause the final exit code to be non-zero
+			Logf("deploy cleanup error: %+v", err)
+		}
+	}()
+
+	// get max id (from all the previous deploys)
+	max, err := simpleDeploy.GetMaxDeployID(ctx)
+	if err != nil {
+		return false, errwrap.Wrapf(err, "error getting max deploy id")
+	}
+	// find the latest id
+	var id = max + 1 // next id
+	Logf("previous max deploy id: %d", max)
+
+	etcdFs := &etcdfs.Fs{
+		Client: etcdClient,
+		// TODO: using a uuid is meant as a temporary measure, i hate them
+		Metadata:   lib.MetadataPrefix + fmt.Sprintf("/deploy/%d-%s", id, uniqueid),
+		DataPrefix: lib.StoragePrefix,
+
+		Debug: data.Flags.Debug,
+		Logf: func(format string, v ...interface{}) {
+			Logf("fs: "+format, v...)
+		},
+	}
+
+	info := &gapi.Info{
+		Args: args,
+		Flags: &gapi.Flags{
+			Noop: obj.Noop,
+			Sema: obj.Sema,
+			//Update: obj.Update,
+		},
+
+		Fs:    etcdFs,
+		Debug: data.Flags.Debug,
+		Logf: func(format string, v ...interface{}) {
+			// TODO: is this a sane prefix to use here?
+			data.Flags.Logf("cli: "+format, v...)
+		},
+	}
+
+	deploy, err := gapiObj.Cli(info)
+	if err != nil {
+		return false, cliUtil.CliParseError(err) // consistent errors
+	}
+	if deploy == nil { // not used
+		return false, fmt.Errorf("not enough information specified")
+	}
+
+	// redundant
+	deploy.Noop = obj.Noop
+	deploy.Sema = obj.Sema
+
+	str, err := deploy.ToB64()
+	if err != nil {
+		return false, errwrap.Wrapf(err, "encoding error")
+	}
+
+	// this nominally checks the previous git hash matches our expectation
+	if err := simpleDeploy.AddDeploy(ctx, id, hash, pHash, &str); err != nil {
+		return false, errwrap.Wrapf(err, "could not create deploy id `%d`", id)
+	}
+	Logf("success, id: %d", id)
+	return true, nil
+}

cli/docs.go

@@ -0,0 +1,150 @@
+// Mgmt
+// Copyright (C) James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+package cli
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+
+	cliUtil "github.com/purpleidea/mgmt/cli/util"
+	"github.com/purpleidea/mgmt/docs"
+)
+
+// DocsGenerateArgs is the CLI parsing structure and type of the parsed result.
+// This particular one contains all the common flags for the `docs generate`
+// subcommand.
+type DocsGenerateArgs struct {
+	docs.Config // embedded config (can't be a pointer) https://github.com/alexflint/go-arg/issues/240
+
+	DocsGenerate *cliUtil.DocsGenerateArgs `arg:"subcommand:generate" help:"generate documentation"`
+}
+
+// Run executes the correct subcommand. It errors if there's ever an error. It
+// returns true if we did activate one of the subcommands. It returns false if
+// we did not. This information is used so that the top-level parser can return
+// usage or help information if no subcommand activates. This particular Run is
+// the run for the main `docs` subcommand.
+func (obj *DocsGenerateArgs) Run(ctx context.Context, data *cliUtil.Data) (bool, error) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	var name string
+	var args interface{}
+	if cmd := obj.DocsGenerate; cmd != nil {
+		name = cliUtil.LookupSubcommand(obj, cmd) // "generate"
+		args = cmd
+	}
+	_ = name
+
+	Logf := func(format string, v ...interface{}) {
+		// Don't block this globally...
+		//if !data.Flags.Debug {
+		//	return
+		//}
+		data.Flags.Logf("main: "+format, v...)
+	}
+
+	var api docs.API
+
+	if cmd := obj.DocsGenerate; cmd != nil {
+		api = &docs.Generate{
+			DocsGenerateArgs: args.(*cliUtil.DocsGenerateArgs),
+			Config:           obj.Config,
+			Program:          data.Program,
+			Version:          data.Version,
+			Debug:            data.Flags.Debug,
+			Logf:             Logf,
+		}
+	}
+
+	if api == nil {
+		return false, nil // nothing found (display help!)
+	}
+
+	// We don't use these for the setup command in normal operation.
+	if data.Flags.Debug {
+		cliUtil.Hello(data.Program, data.Version, data.Flags) // say hello!
+		defer Logf("goodbye!")
+	}
+
+	// install the exit signal handler
+	wg := &sync.WaitGroup{}
+	defer wg.Wait()
+	exit := make(chan struct{})
+	defer close(exit)
+	wg.Add(1)
+	go func() {
+		defer cancel()
+		defer wg.Done()
+		// must have buffer for max number of signals
+		signals := make(chan os.Signal, 3+1) // 3 * ^C + 1 * SIGTERM
+		signal.Notify(signals, os.Interrupt) // catch ^C
+		//signal.Notify(signals, os.Kill) // catch signals
+		signal.Notify(signals, syscall.SIGTERM)
+		var count uint8
+		for {
+			select {
+			case sig := <-signals: // any signal will do
+				if sig != os.Interrupt {
+					data.Flags.Logf("interrupted by signal")
+					return
+				}
+
+				switch count {
+				case 0:
+					data.Flags.Logf("interrupted by ^C")
+					cancel()
+				case 1:
+					data.Flags.Logf("interrupted by ^C (fast pause)")
+					cancel()
+				case 2:
+					data.Flags.Logf("interrupted by ^C (hard interrupt)")
+					cancel()
+				}
+				count++
+
+			case <-exit:
+				return
+			}
+		}
+	}()
+
+	if err := api.Main(ctx); err != nil {
+		if data.Flags.Debug {
+			data.Flags.Logf("main: %+v", err)
+		}
+		return false, err
+	}
+
+	return true, nil
+}

cli/firstboot.go

@@ -0,0 +1,151 @@
+// Mgmt
+// Copyright (C) James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+package cli
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+
+	cliUtil "github.com/purpleidea/mgmt/cli/util"
+	"github.com/purpleidea/mgmt/firstboot"
+)
+
+// FirstbootArgs is the CLI parsing structure and type of the parsed result.
+// This particular one contains all the common flags for the `firstboot`
+// subcommand.
+type FirstbootArgs struct {
+	firstboot.Config // embedded config (can't be a pointer) https://github.com/alexflint/go-arg/issues/240
+
+	FirstbootStart *cliUtil.FirstbootStartArgs `arg:"subcommand:start" help:"start firstboot service"`
+}
+
+// Run executes the correct subcommand. It errors if there's ever an error. It
+// returns true if we did activate one of the subcommands. It returns false if
+// we did not. This information is used so that the top-level parser can return
+// usage or help information if no subcommand activates. This particular Run is
+// the run for the main `firstboot` subcommand. The firstboot command as a
+// service that lets you run commands once on the first boot of a system.
+func (obj *FirstbootArgs) Run(ctx context.Context, data *cliUtil.Data) (bool, error) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	var name string
+	var args interface{}
+	if cmd := obj.FirstbootStart; cmd != nil {
+		name = cliUtil.LookupSubcommand(obj, cmd) // "pkg"
+		args = cmd
+	}
+	_ = name
+
+	Logf := func(format string, v ...interface{}) {
+		// Don't block this globally...
+		//if !data.Flags.Debug {
+		//	return
+		//}
+		data.Flags.Logf("main: "+format, v...)
+	}
+
+	var api firstboot.API
+
+	if cmd := obj.FirstbootStart; cmd != nil {
+		api = &firstboot.Start{
+			FirstbootStartArgs: args.(*cliUtil.FirstbootStartArgs),
+			Config:             obj.Config,
+			Program:            data.Program,
+			Version:            data.Version,
+			Debug:              data.Flags.Debug,
+			Logf:               Logf,
+		}
+	}
+
+	if api == nil {
+		return false, nil // nothing found (display help!)
+	}
+
+	// We don't use these for the setup command in normal operation.
+	if data.Flags.Debug {
+		cliUtil.Hello(data.Program, data.Version, data.Flags) // say hello!
+		defer Logf("goodbye!")
+	}
+
+	// install the exit signal handler
+	wg := &sync.WaitGroup{}
+	defer wg.Wait()
+	exit := make(chan struct{})
+	defer close(exit)
+	wg.Add(1)
+	go func() {
+		defer cancel()
+		defer wg.Done()
+		// must have buffer for max number of signals
+		signals := make(chan os.Signal, 3+1) // 3 * ^C + 1 * SIGTERM
+		signal.Notify(signals, os.Interrupt) // catch ^C
+		//signal.Notify(signals, os.Kill) // catch signals
+		signal.Notify(signals, syscall.SIGTERM)
+		var count uint8
+		for {
+			select {
+			case sig := <-signals: // any signal will do
+				if sig != os.Interrupt {
+					data.Flags.Logf("interrupted by signal")
+					return
+				}
+
+				switch count {
+				case 0:
+					data.Flags.Logf("interrupted by ^C")
+					cancel()
+				case 1:
+					data.Flags.Logf("interrupted by ^C (fast pause)")
+					cancel()
+				case 2:
+					data.Flags.Logf("interrupted by ^C (hard interrupt)")
+					cancel()
+				}
+				count++
+
+			case <-exit:
+				return
+			}
+		}
+	}()
+
+	if err := api.Main(ctx); err != nil {
+		if data.Flags.Debug {
+			data.Flags.Logf("main: %+v", err)
+		}
+		return false, err
+	}
+
+	return true, nil
+}

cli/run.go

@@ -0,0 +1,243 @@
+// Mgmt
+// Copyright (C) James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+package cli
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+
+	cliUtil "github.com/purpleidea/mgmt/cli/util"
+	"github.com/purpleidea/mgmt/gapi"
+	"github.com/purpleidea/mgmt/lib"
+	"github.com/purpleidea/mgmt/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
+
+	"github.com/spf13/afero"
+)
+
+// RunArgs is the CLI parsing structure and type of the parsed result. This
+// particular one contains all the common flags for the `run` subcommand which
+// all frontends can use.
+type RunArgs struct {
+	lib.Config // embedded config (can't be a pointer) https://github.com/alexflint/go-arg/issues/240
+
+	RunEmpty      *cliUtil.EmptyArgs      `arg:"subcommand:empty" help:"run empty payload"`
+	RunLang       *cliUtil.LangArgs       `arg:"subcommand:lang" help:"run lang (mcl) payload"`
+	RunYaml       *cliUtil.YamlArgs       `arg:"subcommand:yaml" help:"run yaml graph payload"`
+	RunPuppet     *cliUtil.PuppetArgs     `arg:"subcommand:puppet" help:"run puppet graph payload"`
+	RunLangPuppet *cliUtil.LangPuppetArgs `arg:"subcommand:langpuppet" help:"run a combined lang/puppet graph payload"`
+}
+
+// Run executes the correct subcommand. It errors if there's ever an error. It
+// returns true if we did activate one of the subcommands. It returns false if
+// we did not. This information is used so that the top-level parser can return
+// usage or help information if no subcommand activates. This particular Run is
+// the run for the main `run` subcommand. This always requires a frontend to
+// start the engine, but if you don't want a graph, you can use the `empty`
+// frontend. The engine backend is agnostic to which frontend is running, in
+// fact, you can deploy with multiple different frontends, one after another, on
+// the same engine.
+func (obj *RunArgs) Run(ctx context.Context, data *cliUtil.Data) (bool, error) {
+	var name string
+	var args interface{}
+	if cmd := obj.RunEmpty; cmd != nil {
+		name = cliUtil.LookupSubcommand(obj, cmd) // "empty"
+		args = cmd
+	}
+	if cmd := obj.RunLang; cmd != nil {
+		name = cliUtil.LookupSubcommand(obj, cmd) // "lang"
+		args = cmd
+	}
+	if cmd := obj.RunYaml; cmd != nil {
+		name = cliUtil.LookupSubcommand(obj, cmd) // "yaml"
+		args = cmd
+	}
+	if cmd := obj.RunPuppet; cmd != nil {
+		name = cliUtil.LookupSubcommand(obj, cmd) // "puppet"
+		args = cmd
+	}
+	if cmd := obj.RunLangPuppet; cmd != nil {
+		name = cliUtil.LookupSubcommand(obj, cmd) // "langpuppet"
+		args = cmd
+	}
+
+	// XXX: workaround https://github.com/alexflint/go-arg/issues/239
+	lists := [][]string{
+		obj.Seeds,
+		obj.ClientURLs,
+		obj.ServerURLs,
+		obj.AdvertiseClientURLs,
+		obj.AdvertiseServerURLs,
+	}
+	gapiNames := gapi.Names() // list of registered names
+	for _, list := range lists {
+		if l := len(list); name == "" && l > 1 {
+			elem := list[l-2] // second to last element
+			if util.StrInList(elem, gapiNames) {
+				return false, cliUtil.CliParseError(cliUtil.MissingEquals) // consistent errors
+			}
+		}
+	}
+
+	fn, exists := gapi.RegisteredGAPIs[name]
+	if !exists {
+		return false, nil // did not activate
+	}
+	gapiObj := fn()
+
+	main := &lib.Main{}
+	main.Config = &obj.Config // pass in all the parsed data
+
+	main.Program, main.Version = data.Program, data.Version
+	main.Debug, main.Logf = data.Flags.Debug, data.Flags.Logf // no prefix
+	Logf := func(format string, v ...interface{}) {
+		data.Flags.Logf("main: "+format, v...)
+	}
+
+	cliUtil.Hello(main.Program, main.Version, data.Flags) // say hello!
+	defer Logf("goodbye!")
+
+	// create a memory backed temporary filesystem for storing runtime data
+	mmFs := afero.NewMemMapFs()
+	afs := &afero.Afero{Fs: mmFs} // wrap so that we're implementing ioutil
+	standaloneFs := &util.AferoFs{Afero: afs}
+	main.DeployFs = standaloneFs
+
+	info := &gapi.Info{
+		Args: args,
+		Flags: &gapi.Flags{
+			Hostname: obj.Hostname,
+			Noop:     obj.Noop,
+			Sema:     obj.Sema,
+			//Update: obj.Update,
+		},
+
+		Fs:    standaloneFs,
+		Debug: data.Flags.Debug,
+		Logf: func(format string, v ...interface{}) {
+			data.Flags.Logf("cli: "+format, v...)
+		},
+	}
+
+	deploy, err := gapiObj.Cli(info)
+	if err != nil {
+		return false, cliUtil.CliParseError(err) // consistent errors
+	}
+
+	if cmd := obj.RunLang; cmd != nil && cmd.OnlyUnify && deploy == nil {
+		return true, nil // we end early
+	}
+	if cmd := obj.RunLang; cmd != nil && cmd.OnlyDownload && deploy == nil {
+		return true, nil // we end early
+	}
+	main.Deploy = deploy
+	if main.Deploy == nil {
+		// nobody activated, but we'll still watch the etcd deploy chan,
+		// and if there is deployed code that's ready to run, we'll run!
+		data.Flags.Logf("main: no frontend selected (no GAPI activated)")
+	}
+
+	if err := main.Validate(); err != nil {
+		return false, err
+	}
+
+	if err := main.Init(); err != nil {
+		return false, err
+	}
+
+	// install the exit signal handler
+	wg := &sync.WaitGroup{}
+	defer wg.Wait()
+	exit := make(chan struct{})
+	defer close(exit)
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		// must have buffer for max number of signals
+		signals := make(chan os.Signal, 3+1) // 3 * ^C + 1 * SIGTERM
+		signal.Notify(signals, os.Interrupt) // catch ^C
+		//signal.Notify(signals, os.Kill) // catch signals
+		signal.Notify(signals, syscall.SIGTERM)
+		var count uint8
+		for {
+			select {
+			case sig := <-signals: // any signal will do
+				if sig != os.Interrupt {
+					data.Flags.Logf("interrupted by signal")
+					main.Interrupt(fmt.Errorf("killed by %v", sig))
+					return
+				}
+
+				switch count {
+				case 0:
+					data.Flags.Logf("interrupted by ^C")
+					main.Exit(nil)
+				case 1:
+					data.Flags.Logf("interrupted by ^C (fast pause)")
+					main.FastExit(nil)
+				case 2:
+					data.Flags.Logf("interrupted by ^C (hard interrupt)")
+					main.Interrupt(nil)
+				}
+				count++
+
+			case <-exit:
+				return
+			}
+		}
+	}()
+
+	reterr := main.Run()
+	if reterr != nil {
+		// log the error message returned
+		if data.Flags.Debug {
+			data.Flags.Logf("main: %+v", reterr)
+		}
+	}
+
+	if err := main.Close(); err != nil {
+		if data.Flags.Debug {
+			data.Flags.Logf("main: Close: %+v", err)
+		}
+		if reterr == nil {
+			return false, err
+		}
+		reterr = errwrap.Append(reterr, err)
+	}
+
+	if reterr != nil {
+		return false, reterr
+	}
+	return true, nil
+}

cli/setup.go

@@ -0,0 +1,180 @@
+// Mgmt
+// Copyright (C) James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+package cli
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+
+	cliUtil "github.com/purpleidea/mgmt/cli/util"
+	"github.com/purpleidea/mgmt/setup"
+)
+
+// SetupArgs is the CLI parsing structure and type of the parsed result. This
+// particular one contains all the common flags for the `setup` subcommand.
+type SetupArgs struct {
+	setup.Config // embedded config (can't be a pointer) https://github.com/alexflint/go-arg/issues/240
+
+	SetupPkg       *cliUtil.SetupPkgArgs       `arg:"subcommand:pkg" help:"setup packages"`
+	SetupSvc       *cliUtil.SetupSvcArgs       `arg:"subcommand:svc" help:"setup services"`
+	SetupFirstboot *cliUtil.SetupFirstbootArgs `arg:"subcommand:firstboot" help:"setup firstboot"`
+}
+
+// Run executes the correct subcommand. It errors if there's ever an error. It
+// returns true if we did activate one of the subcommands. It returns false if
+// we did not. This information is used so that the top-level parser can return
+// usage or help information if no subcommand activates. This particular Run is
+// the run for the main `setup` subcommand. The setup command does some
+// bootstrap work to help get things going.
+func (obj *SetupArgs) Run(ctx context.Context, data *cliUtil.Data) (bool, error) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	var name string
+	var args interface{}
+	if cmd := obj.SetupPkg; cmd != nil {
+		name = cliUtil.LookupSubcommand(obj, cmd) // "pkg"
+		args = cmd
+	}
+	if cmd := obj.SetupSvc; cmd != nil {
+		name = cliUtil.LookupSubcommand(obj, cmd) // "svc"
+		args = cmd
+	}
+	if cmd := obj.SetupFirstboot; cmd != nil {
+		name = cliUtil.LookupSubcommand(obj, cmd) // "firstboot"
+		args = cmd
+	}
+	_ = name
+
+	Logf := func(format string, v ...interface{}) {
+		// Don't block this globally...
+		//if !data.Flags.Debug {
+		//	return
+		//}
+		data.Flags.Logf("main: "+format, v...)
+	}
+
+	var api setup.API
+
+	if cmd := obj.SetupPkg; cmd != nil {
+		api = &setup.Pkg{
+			SetupPkgArgs: args.(*cliUtil.SetupPkgArgs),
+			Config:       obj.Config,
+			Program:      data.Program,
+			Version:      data.Version,
+			Debug:        data.Flags.Debug,
+			Logf:         Logf,
+		}
+	}
+	if cmd := obj.SetupSvc; cmd != nil {
+		api = &setup.Svc{
+			SetupSvcArgs: args.(*cliUtil.SetupSvcArgs),
+			Config:       obj.Config,
+			Program:      data.Program,
+			Version:      data.Version,
+			Debug:        data.Flags.Debug,
+			Logf:         Logf,
+		}
+	}
+	if cmd := obj.SetupFirstboot; cmd != nil {
+		api = &setup.Firstboot{
+			SetupFirstbootArgs: args.(*cliUtil.SetupFirstbootArgs),
+			Config:             obj.Config,
+			Program:            data.Program,
+			Version:            data.Version,
+			Debug:              data.Flags.Debug,
+			Logf:               Logf,
+		}
+	}
+
+	if api == nil {
+		return false, nil // nothing found (display help!)
+	}
+
+	// We don't use these for the setup command in normal operation.
+	if data.Flags.Debug {
+		cliUtil.Hello(data.Program, data.Version, data.Flags) // say hello!
+		defer Logf("goodbye!")
+	}
+
+	// install the exit signal handler
+	wg := &sync.WaitGroup{}
+	defer wg.Wait()
+	exit := make(chan struct{})
+	defer close(exit)
+	wg.Add(1)
+	go func() {
+		defer cancel()
+		defer wg.Done()
+		// must have buffer for max number of signals
+		signals := make(chan os.Signal, 3+1) // 3 * ^C + 1 * SIGTERM
+		signal.Notify(signals, os.Interrupt) // catch ^C
+		//signal.Notify(signals, os.Kill) // catch signals
+		signal.Notify(signals, syscall.SIGTERM)
+		var count uint8
+		for {
+			select {
+			case sig := <-signals: // any signal will do
+				if sig != os.Interrupt {
+					data.Flags.Logf("interrupted by signal")
+					return
+				}
+
+				switch count {
+				case 0:
+					data.Flags.Logf("interrupted by ^C")
+					cancel()
+				case 1:
+					data.Flags.Logf("interrupted by ^C (fast pause)")
+					cancel()
+				case 2:
+					data.Flags.Logf("interrupted by ^C (hard interrupt)")
+					cancel()
+				}
+				count++
+
+			case <-exit:
+				return
+			}
+		}
+	}()
+
+	if err := api.Main(ctx); err != nil {
+		if data.Flags.Debug {
+			data.Flags.Logf("main: %+v", err)
+		}
+		return false, err
+	}
+
+	return true, nil
+}

cli/util/args.go

@@ -0,0 +1,198 @@
+// Mgmt
+// Copyright (C) James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+package util
+
+import (
+	"reflect"
+	"strings"
+)
+
+// LookupSubcommand returns the name of the subcommand in the obj, of a struct.
+// This is useful for determining the name of the subcommand that was activated.
+// It returns an empty string if a specific name was not found.
+func LookupSubcommand(obj interface{}, st interface{}) string {
+	val := reflect.ValueOf(obj)
+	if val.Kind() == reflect.Ptr { // max one de-referencing
+		val = val.Elem()
+	}
+
+	v := reflect.ValueOf(st) // value of the struct
+	typ := val.Type()
+	for i := 0; i < typ.NumField(); i++ {
+		f := val.Field(i) // value of the field
+		if f.Interface() != v.Interface() {
+			continue
+		}
+
+		field := typ.Field(i)
+		alias, ok := field.Tag.Lookup("arg")
+		if !ok {
+			continue
+		}
+
+		// XXX: `arg` needs a split by comma first or fancier parsing
+		prefix := "subcommand"
+		split := strings.Split(alias, ":")
+		if len(split) != 2 || split[0] != prefix {
+			continue
+		}
+
+		return split[1] // found
+	}
+	return "" // not found
+}
+
+// EmptyArgs is the empty CLI parsing structure and type of the parsed result.
+type EmptyArgs struct{}
+
+// LangArgs is the lang CLI parsing structure and type of the parsed result.
+type LangArgs struct {
+	// Input is the input mcl code or file path or any input specification.
+	Input string `arg:"positional,required"`
+
+	// TODO: removed (temporarily?)
+	//Stdin bool `arg:"--stdin" help:"use passthrough stdin"`
+
+	Download     bool `arg:"--download" help:"download any missing imports"`
+	OnlyDownload bool `arg:"--only-download" help:"stop after downloading any missing imports"`
+	Update       bool `arg:"--update" help:"update all dependencies to the latest versions"`
+
+	OnlyUnify          bool     `arg:"--only-unify" help:"stop after type unification"`
+	SkipUnify          bool     `arg:"--skip-unify" help:"skip type unification"`
+	UnifySolver        *string  `arg:"--unify-name" help:"pick a specific unification solver"`
+	UnifyOptimizations []string `arg:"--unify-optimizations" help:"list of unification optimizations to request (experts only)"`
+
+	Depth int `arg:"--depth" default:"-1" help:"max recursion depth limit (-1 is unlimited)"`
+
+	// The default of 0 means any error is a failure by default.
+	Retry int `arg:"--depth" help:"max number of retries (-1 is unlimited)"`
+
+	ModulePath string `arg:"--module-path,env:MGMT_MODULE_PATH" help:"choose the modules path (absolute)"`
+}
+
+// YamlArgs is the yaml CLI parsing structure and type of the parsed result.
+type YamlArgs struct {
+	// Input is the input yaml code or file path or any input specification.
+	Input string `arg:"positional,required"`
+}
+
+// PuppetArgs is the puppet CLI parsing structure and type of the parsed result.
+type PuppetArgs struct {
+	// Input is the input puppet code or file path or just "agent".
+	Input string `arg:"positional,required"`
+
+	// PuppetConf is the optional path to a puppet.conf config file.
+	PuppetConf string `arg:"--puppet-conf" help:"full path to the puppet.conf file to use"`
+}
+
+// LangPuppetArgs is the langpuppet CLI parsing structure and type of the parsed
+// result.
+type LangPuppetArgs struct {
+	// LangInput is the input mcl code or file path or any input specification.
+	LangInput string `arg:"--lang,required" help:"the input parameter for the lang module"`
+
+	// PuppetInput is the input puppet code or file path or just "agent".
+	PuppetInput string `arg:"--puppet,required" help:"the input parameter for the puppet module"`
+
+	// copy-pasted from PuppetArgs
+
+	// PuppetConf is the optional path to a puppet.conf config file.
+	PuppetConf string `arg:"--puppet-conf" help:"full path to the puppet.conf file to use"`
+
+	// end PuppetArgs
+
+	// copy-pasted from LangArgs
+
+	// TODO: removed (temporarily?)
+	//Stdin bool `arg:"--stdin" help:"use passthrough stdin"`
+
+	Download     bool `arg:"--download" help:"download any missing imports"`
+	OnlyDownload bool `arg:"--only-download" help:"stop after downloading any missing imports"`
+	Update       bool `arg:"--update" help:"update all dependencies to the latest versions"`
+
+	OnlyUnify bool `arg:"--only-unify" help:"stop after type unification"`
+	SkipUnify bool `arg:"--skip-unify" help:"skip type unification"`
+
+	Depth int `arg:"--depth" default:"-1" help:"max recursion depth limit (-1 is unlimited)"`
+
+	// The default of 0 means any error is a failure by default.
+	Retry int `arg:"--depth" help:"max number of retries (-1 is unlimited)"`
+
+	ModulePath string `arg:"--module-path,env:MGMT_MODULE_PATH" help:"choose the modules path (absolute)"`
+
+	// end LangArgs
+}
+
+// SetupPkgArgs is the setup service CLI parsing structure and type of the
+// parsed result.
+type SetupPkgArgs struct {
+	Distro string `arg:"--distro" help:"build for this distro"`
+	Sudo   bool   `arg:"--sudo" help:"include sudo in the command"`
+	Exec   bool   `arg:"--exec" help:"actually run these commands"`
+}
+
+// SetupSvcArgs is the setup service CLI parsing structure and type of the
+// parsed result.
+type SetupSvcArgs struct {
+	BinaryPath string `arg:"--binary-path" help:"path to the binary"`
+	Install    bool   `arg:"--install" help:"install the systemd mgmt service"`
+	Start      bool   `arg:"--start" help:"start the mgmt service"`
+	Enable     bool   `arg:"--enable" help:"enable the mgmt service"`
+}
+
+// SetupFirstbootArgs is the setup service CLI parsing structure and type of the
+// parsed result.
+type SetupFirstbootArgs struct {
+	BinaryPath string `arg:"--binary-path" help:"path to the binary"`
+	Mkdir      bool   `arg:"--mkdir" help:"make the necessary firstboot dirs"`
+	Install    bool   `arg:"--install" help:"install the systemd firstboot service"`
+	Start      bool   `arg:"--start" help:"start the firstboot service (typically not used)"`
+	Enable     bool   `arg:"--enable" help:"enable the firstboot service"`
+
+	FirstbootStartArgs // Include these options if we want to specify them.
+}
+
+// FirstbootStartArgs is the firstboot service CLI parsing structure and type of
+// the parsed result.
+type FirstbootStartArgs struct {
+	LockFilePath string `arg:"--lock-file-path" help:"path to the lock file"`
+	ScriptsDir   string `arg:"--scripts-dir" help:"path to the scripts dir"`
+	DoneDir      string `arg:"--done-dir" help:"dir to move done scripts to"`
+	LoggingDir   string `arg:"--logging-dir" help:"directory to store logs in"`
+}
+
+// DocsGenerateArgs is the docgen utility CLI parsing structure and type of the
+// parsed result.
+type DocsGenerateArgs struct {
+	Output      string `arg:"--output" help:"output path to write to"`
+	RootDir     string `arg:"--root-dir" help:"path to mgmt source dir"`
+	NoResources bool   `arg:"--no-resources" help:"skip resource doc generation"`
+	NoFunctions bool   `arg:"--no-functions" help:"skip function doc generation"`
+}

cli/util/hello.go

@@ -0,0 +1,47 @@
+// Mgmt
+// Copyright (C) James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+package util
+
+import (
+	"fmt"
+	"time"
+)
+
+// Hello is a simple helper function to print a hello message and time.
+func Hello(program, version string, flags Flags) {
+	var start = time.Now().UnixNano()
+	if program == "" {
+		program = "<unknown>"
+	}
+	fmt.Println(fmt.Sprintf("This is: %s, version: %s", program, version))
+	fmt.Println("Copyright (C) James Shubin and the project contributors")
+	fmt.Println("Written by James Shubin <james@shubin.ca> and the project contributors")
+	flags.Logf("main: start: %v", start)
+}

cli/util/util.go

@@ -0,0 +1,98 @@
+// Mgmt
+// Copyright (C) James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+// Package util has some CLI related utility code.
+package util
+
+import (
+	"log"
+	"os"
+	"strings"
+
+	"github.com/purpleidea/mgmt/util/errwrap"
+)
+
+// Error is a constant error type that implements error.
+type Error string
+
+// Error fulfills the error interface of this type.
+func (e Error) Error() string { return string(e) }
+
+const (
+	// MissingEquals means we probably hit the parsing bug.
+	// XXX: see: https://github.com/alexflint/go-arg/issues/239
+	MissingEquals = Error("missing equals sign for list element")
+)
+
+// CliParseError returns a consistent error if we have a CLI parsing issue.
+func CliParseError(err error) error {
+	return errwrap.Wrapf(err, "cli parse error")
+}
+
+// Flags are some constant flags which are used throughout the program.
+type Flags struct {
+	Debug bool // add additional log messages
+	Logf  func(format string, v ...interface{})
+}
+
+// Data is a struct of values that we usually pass to the main CLI function.
+type Data struct {
+	Program string
+	Version string
+	Copying string
+	Tagline string
+	Flags   Flags
+	Args    []string // os.Args usually
+}
+
+// SafeProgram returns the correct program string when given a buggy variant.
+func SafeProgram(program string) string {
+	// FIXME: in sub commands, the cli package appends a space and the sub
+	// command name at the end. hack around this by only using the first bit
+	// see: https://github.com/urfave/cli/issues/783 for more details...
+	split := strings.Split(program, " ")
+	program = split[0]
+	//if program == "" {
+	//	program = "<unknown>"
+	//}
+	return program
+}
+
+// LogSetup changes some of the core logger package settings.
+func LogSetup(debug bool) {
+	// TODO: Move these log package initialization steps to the top main.go?
+	logFlags := log.LstdFlags
+	if debug {
+		logFlags = logFlags + log.Lshortfile
+	}
+	logFlags = logFlags - log.Ldate // remove the date for now
+	log.SetFlags(logFlags)
+
+	log.SetOutput(os.Stderr)
+}

converger/converger.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -13,7 +13,19 @@
 // GNU General Public License for more details.
 //
 // You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
 
 // Package converger is a facility for reporting the converged state.
 package converger
@@ -25,139 +37,251 @@ import (
 	"time"
 
 	"github.com/purpleidea/mgmt/util"
-
-	multierr "github.com/hashicorp/go-multierror"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
-// TODO: we could make a new function that masks out the state of certain
-// UID's, but at the moment the new Timer code has obsoleted the need...
+// New builds a new converger coordinator.
+func New(timeout int) *Coordinator {
+	return &Coordinator{
+		timeout: timeout,
 
-// Converger is the general interface for implementing a convergence watcher.
-type Converger interface { // TODO: need a better name
-	Register() UID
-	IsConverged(UID) bool         // is the UID converged ?
-	SetConverged(UID, bool) error // set the converged state of the UID
-	Unregister(UID)
-	Start()
-	Pause()
-	Loop(bool)
-	ConvergedTimer(UID) <-chan time.Time
-	Status() map[uint64]bool
-	Timeout() int                              // returns the timeout that this was created with
-	AddStateFn(string, func(bool) error) error // adds a stateFn with a name
-	RemoveStateFn(string) error                // remove a stateFn with a given name
-}
+		mutex: &sync.RWMutex{},
 
-// UID is the interface resources can use to notify with if converged. You'll
-// need to use part of the Converger interface to Register initially too.
-type UID interface {
-	ID() uint64   // get Id
-	Name() string // get a friendly name
-	SetName(string)
-	IsValid() bool // has Id been initialized ?
-	InvalidateID() // set Id to nil
-	IsConverged() bool
-	SetConverged(bool) error
-	Unregister()
-	ConvergedTimer() <-chan time.Time
-	StartTimer() (func() error, error) // cancellable is the same as StopTimer()
-	ResetTimer() error                 // resets counter to zero
-	StopTimer() error
-}
+		//lastid: 0,
+		status: make(map[*UID]struct{}),
 
-// converger is an implementation of the Converger interface.
-type converger struct {
-	timeout   int           // must be zero (instant) or greater seconds to run
-	converged bool          // did we converge (state changes of this run Fn)
-	channel   chan struct{} // signal here to run an isConverged check
-	control   chan bool     // control channel for start/pause
-	mutex     *sync.RWMutex // used for controlling access to status and lastid
-	lastid    uint64
-	status    map[uint64]bool
-	stateFns  map[string]func(bool) error // run on converged state changes with state bool
-	smutex    *sync.RWMutex               // used for controlling access to stateFns
-}
+		//converged: false, // initial state
 
-// cuid is an implementation of the UID interface.
-type cuid struct {
-	converger Converger
-	id        uint64
-	name      string // user defined, friendly name
-	mutex     *sync.Mutex
-	timer     chan struct{}
-	running   bool // is the above timer running?
-	wg        *sync.WaitGroup
-}
+		pokeChan: make(chan struct{}, 1), // must be buffered
+
+		readyChan: make(chan struct{}), // ready signal
+
+		//paused: false, // starts off as started
+		pauseSignal: make(chan struct{}),
+		//resumeSignal: make(chan struct{}), // happens on pause
+		//pausedAck: util.NewEasyAck(), // happens on pause
 
-// NewConverger builds a new converger struct.
-func NewConverger(timeout int) Converger {
-	return &converger{
-		timeout:  timeout,
-		channel:  make(chan struct{}),
-		control:  make(chan bool),
-		mutex:    &sync.RWMutex{},
-		lastid:   0,
-		status:   make(map[uint64]bool),
 		stateFns: make(map[string]func(bool) error),
 		smutex:   &sync.RWMutex{},
-	}
-}
 
-// Register assigns a UID to the caller.
-func (obj *converger) Register() UID {
-	obj.mutex.Lock()
-	defer obj.mutex.Unlock()
-	obj.lastid++
-	obj.status[obj.lastid] = false // initialize as not converged
-	return &cuid{
-		converger: obj,
-		id:        obj.lastid,
-		name:      fmt.Sprintf("%d", obj.lastid), // some default
-		mutex:     &sync.Mutex{},
-		timer:     nil,
-		running:   false,
+		closeChan: make(chan struct{}),
 		wg:        &sync.WaitGroup{},
 	}
 }
 
-// IsConverged gets the converged status of a uid.
-func (obj *converger) IsConverged(uid UID) bool {
-	if !uid.IsValid() {
-		panic(fmt.Sprintf("the ID of UID(%s) is nil", uid.Name()))
-	}
-	obj.mutex.RLock()
-	isConverged, found := obj.status[uid.ID()] // lookup
-	obj.mutex.RUnlock()
-	if !found {
-		panic("the ID of UID is unregistered")
-	}
-	return isConverged
+// Coordinator is the central converger engine.
+type Coordinator struct {
+	// timeout must be zero (instant) or greater seconds to run. If it's -1
+	// then this is disabled, and we never run stateFns.
+	timeout int
+
+	// mutex is used for controlling access to status and lastid.
+	mutex *sync.RWMutex
+
+	// lastid contains the last uid we used for registration.
+	//lastid    uint64
+	// status contains a reference to each active UID.
+	status map[*UID]struct{}
+
+	// converged stores the last convergence state. When this changes, we
+	// run the stateFns.
+	converged bool
+
+	// pokeChan receives a message every time we might need to re-calculate.
+	pokeChan chan struct{}
+
+	// readyChan closes to notify any interested parties that the main loop
+	// is running.
+	readyChan chan struct{}
+
+	// paused represents if this coordinator is paused or not.
+	paused bool
+	// pauseSignal closes to request a pause of this coordinator.
+	pauseSignal chan struct{}
+	// resumeSignal closes to request a resume of this coordinator.
+	resumeSignal chan struct{}
+	// pausedAck is used to send an ack message saying that we've paused.
+	pausedAck *util.EasyAck
+
+	// stateFns run on converged state changes.
+	stateFns map[string]func(bool) error
+	// smutex is used for controlling access to the stateFns map.
+	smutex *sync.RWMutex
+
+	// closeChan closes when we've been requested to shutdown.
+	closeChan chan struct{}
+	// wg waits for everything to finish.
+	wg *sync.WaitGroup
 }
 
-// SetConverged updates the converger with the converged state of the UID.
-func (obj *converger) SetConverged(uid UID, isConverged bool) error {
-	if !uid.IsValid() {
-		return fmt.Errorf("the ID of UID(%s) is nil", uid.Name())
-	}
+// Register creates a new UID which can be used to report converged state. You
+// must Unregister each UID before Shutdown will be able to finish running.
+func (obj *Coordinator) Register() *UID {
+	obj.wg.Add(1) // additional tracking for each UID
 	obj.mutex.Lock()
-	if _, found := obj.status[uid.ID()]; !found {
-		panic("the ID of UID is unregistered")
+	defer obj.mutex.Unlock()
+	//obj.lastid++
+	uid := &UID{
+		timeout: obj.timeout, // copy the timeout here
+		//id: obj.lastid,
+		//name: fmt.Sprintf("%d", obj.lastid), // some default
+
+		poke: obj.poke,
+
+		// timer
+		mutex:   &sync.Mutex{},
+		timer:   nil,
+		running: false,
+		wg:      &sync.WaitGroup{},
 	}
-	obj.status[uid.ID()] = isConverged // set
-	obj.mutex.Unlock()                 // unlock *before* poke or deadlock!
-	if isConverged != obj.converged {  // only poke if it would be helpful
-		// run in a go routine so that we never block... just queue up!
-		// this allows us to send events, even if we haven't started...
-		go func() { obj.channel <- struct{}{} }()
+	uid.unregister = func() { obj.Unregister(uid) } // add unregister func
+	obj.status[uid] = struct{}{}                    // TODO: add converged state here?
+	return uid
+}
+
+// Unregister removes the UID from the converger coordinator. If you supply an
+// invalid or unregistered uid to this function, it will panic. An unregistered
+// UID is no longer part of the convergence checking.
+func (obj *Coordinator) Unregister(uid *UID) {
+	defer obj.wg.Done() // additional tracking for each UID
+	obj.mutex.Lock()
+	defer obj.mutex.Unlock()
+
+	if _, exists := obj.status[uid]; !exists {
+		panic("uid is not registered")
 	}
+	uid.StopTimer() // ignore any errors
+	delete(obj.status, uid)
+}
+
+// Run starts the main loop for the converger coordinator. It is commonly run
+// from a go routine. It blocks until the Shutdown method is run to close it.
+// NOTE: when we have very short timeouts, if we start before all the resources
+// have joined the map, then it might appear as if we converged before we did!
+func (obj *Coordinator) Run(startPaused bool) {
+	obj.wg.Add(1)
+	wg := &sync.WaitGroup{} // needed for the startPaused
+	defer wg.Wait()         // don't leave any leftover go routines running
+	if startPaused {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			obj.Pause() // ignore any errors
+			close(obj.readyChan)
+		}()
+	} else {
+		close(obj.readyChan) // we must wait till the wg.Add(1) has happened...
+	}
+	defer obj.wg.Done()
+	for {
+		// pause if one was requested...
+		select {
+		case <-obj.pauseSignal: // channel closes
+			obj.pausedAck.Ack() // send ack
+			// we are paused now, and waiting for resume or exit...
+			select {
+			case <-obj.resumeSignal: // channel closes
+				// resumed!
+
+			case <-obj.closeChan: // we can always escape
+				return
+			}
+
+		case _, ok := <-obj.pokeChan: // we got an event (re-calculate)
+			if !ok {
+				return
+			}
+
+			if err := obj.test(); err != nil {
+				// FIXME: what to do on error ?
+			}
+
+		case <-obj.closeChan: // we can always escape
+			return
+		}
+	}
+}
+
+// Ready blocks until the Run loop has started up. This is useful so that we
+// don't run Shutdown before we've even started up properly.
+func (obj *Coordinator) Ready() {
+	select {
+	case <-obj.readyChan:
+	}
+}
+
+// Shutdown sends a signal to the Run loop that it should exit. This blocks
+// until it does.
+func (obj *Coordinator) Shutdown() {
+	close(obj.closeChan)
+	obj.wg.Wait()
+	close(obj.pokeChan) // free memory?
+}
+
+// Pause pauses the coordinator. It should not be called on an already paused
+// coordinator. It will block until the coordinator pauses with an
+// acknowledgment, or until an exit is requested. If the latter happens it will
+// error. It is NOT thread-safe with the Resume() method so only call either one
+// at a time.
+func (obj *Coordinator) Pause() error {
+	if obj.paused {
+		return fmt.Errorf("already paused")
+	}
+
+	obj.pausedAck = util.NewEasyAck()
+	obj.resumeSignal = make(chan struct{}) // build the resume signal
+	close(obj.pauseSignal)
+
+	// wait for ack (or exit signal)
+	select {
+	case <-obj.pausedAck.Wait(): // we got it!
+		// we're paused
+	case <-obj.closeChan:
+		return fmt.Errorf("closing")
+	}
+	obj.paused = true
+
 	return nil
 }
 
-// isConverged returns true if *every* registered uid has converged.
-func (obj *converger) isConverged() bool {
-	obj.mutex.RLock() // take a read lock
-	defer obj.mutex.RUnlock()
-	for _, v := range obj.status {
+// Resume unpauses the coordinator. It can be safely called on a brand-new
+// coordinator that has just started running without incident. It is NOT
+// thread-safe with the Pause() method, so only call either one at a time.
+func (obj *Coordinator) Resume() {
+	// TODO: do we need a mutex around Resume?
+	if !obj.paused { // no need to unpause brand-new resources
+		return
+	}
+
+	obj.pauseSignal = make(chan struct{}) // rebuild for next pause
+	close(obj.resumeSignal)
+	obj.poke() // unblock and notice the resume if necessary
+
+	obj.paused = false
+
+	// no need to wait for it to resume
+	//return // implied
+}
+
+// poke sends a message to the coordinator telling it that it should re-evaluate
+// whether we're converged or not. This does not block. Do not run this in a
+// goroutine. It must not be called after Shutdown has been called.
+func (obj *Coordinator) poke() {
+	// redundant
+	//if len(obj.pokeChan) > 0 {
+	//	return
+	//}
+
+	select {
+	case obj.pokeChan <- struct{}{}:
+	default: // if chan is now full because more than one poke happened...
+	}
+}
+
+// IsConverged returns true if *every* registered uid has converged. If there
+// are no registered UID's, then this will return true.
+func (obj *Coordinator) IsConverged() bool {
+	for _, v := range obj.Status() {
 		if !v { // everyone must be converged for this to be true
 			return false
 		}
@@ -165,145 +289,40 @@ func (obj *converger) isConverged() bool {
 	return true
 }
 
-// Unregister dissociates the ConvergedUID from the converged checking.
-func (obj *converger) Unregister(uid UID) {
-	if !uid.IsValid() {
-		panic(fmt.Sprintf("the ID of UID(%s) is nil", uid.Name()))
+// test evaluates whether we're converged or not and runs the state change. It
+// is NOT thread-safe.
+func (obj *Coordinator) test() error {
+	// TODO: add these checks elsewhere to prevent anything from running?
+	if obj.timeout < 0 {
+		return nil // nothing to do (only run if timeout is valid)
 	}
-	obj.mutex.Lock()
-	uid.StopTimer() // ignore any errors
-	delete(obj.status, uid.ID())
-	obj.mutex.Unlock()
-	uid.InvalidateID()
-}
 
-// Start causes a Converger object to start or resume running.
-func (obj *converger) Start() {
-	obj.control <- true
-}
+	converged := obj.IsConverged()
+	defer func() {
+		obj.converged = converged // set this only at the end...
+	}()
 
-// Pause causes a Converger object to stop running temporarily.
-func (obj *converger) Pause() { // FIXME: add a sync ACK on pause before return
-	obj.control <- false
-}
-
-// Loop is the main loop for a Converger object. It usually runs in a goroutine.
-// TODO: we could eventually have each resource tell us as soon as it converges,
-// and then keep track of the time delays here, to avoid callers needing select.
-// NOTE: when we have very short timeouts, if we start before all the resources
-// have joined the map, then it might appear as if we converged before we did!
-func (obj *converger) Loop(startPaused bool) {
-	if obj.control == nil {
-		panic("converger not initialized correctly")
-	}
-	if startPaused { // start paused without racing
-		select {
-		case e := <-obj.control:
-			if !e {
-				panic("converger expected true")
-			}
+	if !converged {
+		if !obj.converged { // were we previously also not converged?
+			return nil // nothing to do
 		}
-	}
-	for {
-		select {
-		case e := <-obj.control: // expecting "false" which means pause!
-			if e {
-				panic("converger expected false")
-			}
-			// now i'm paused...
-			select {
-			case e := <-obj.control:
-				if !e {
-					panic("converger expected true")
-				}
-				// restart
-				// kick once to refresh the check...
-				go func() { obj.channel <- struct{}{} }()
-				continue
-			}
 
-		case <-obj.channel:
-			if !obj.isConverged() {
-				if obj.converged { // we're doing a state change
-					// call the arbitrary functions (takes a read lock!)
-					if err := obj.runStateFns(false); err != nil {
-						// FIXME: what to do on error ?
-					}
-				}
-				obj.converged = false
-				continue
-			}
-
-			// we have converged!
-			if obj.timeout >= 0 { // only run if timeout is valid
-				if !obj.converged { // we're doing a state change
-					// call the arbitrary functions (takes a read lock!)
-					if err := obj.runStateFns(true); err != nil {
-						// FIXME: what to do on error ?
-					}
-				}
-			}
-			obj.converged = true
-			// loop and wait again...
-		}
+		// we're doing a state change
+		// call the arbitrary functions (takes a read lock!)
+		return obj.runStateFns(false)
 	}
+
+	// we have converged!
+	if obj.converged { // were we previously also converged?
+		return nil // nothing to do
+	}
+
+	// call the arbitrary functions (takes a read lock!)
+	return obj.runStateFns(true)
 }
 
-// ConvergedTimer adds a timeout to a select call and blocks until then.
-// TODO: this means we could eventually have per resource converged timeouts
-func (obj *converger) ConvergedTimer(uid UID) <-chan time.Time {
-	// be clever: if i'm already converged, this timeout should block which
-	// avoids unnecessary new signals being sent! this avoids fast loops if
-	// we have a low timeout, or in particular a timeout == 0
-	if uid.IsConverged() {
-		// blocks the case statement in select forever!
-		return util.TimeAfterOrBlock(-1)
-	}
-	return util.TimeAfterOrBlock(obj.timeout)
-}
-
-// Status returns a map of the converged status of each UID.
-func (obj *converger) Status() map[uint64]bool {
-	status := make(map[uint64]bool)
-	obj.mutex.RLock() // take a read lock
-	defer obj.mutex.RUnlock()
-	for k, v := range obj.status { // make a copy to avoid the mutex
-		status[k] = v
-	}
-	return status
-}
-
-// Timeout returns the timeout in seconds that converger was created with. This
-// is useful to avoid passing in the timeout value separately when you're
-// already passing in the Converger struct.
-func (obj *converger) Timeout() int {
-	return obj.timeout
-}
-
-// AddStateFn adds a state function to be run on change of converged state.
-func (obj *converger) AddStateFn(name string, stateFn func(bool) error) error {
-	obj.smutex.Lock()
-	defer obj.smutex.Unlock()
-	if _, exists := obj.stateFns[name]; exists {
-		return fmt.Errorf("a stateFn with that name already exists")
-	}
-	obj.stateFns[name] = stateFn
-	return nil
-}
-
-// RemoveStateFn adds a state function to be run on change of converged state.
-func (obj *converger) RemoveStateFn(name string) error {
-	obj.smutex.Lock()
-	defer obj.smutex.Unlock()
-	if _, exists := obj.stateFns[name]; !exists {
-		return fmt.Errorf("a stateFn with that name doesn't exist")
-	}
-	delete(obj.stateFns, name)
-	return nil
-}
-
-// runStateFns runs the listed of stored state functions.
-func (obj *converger) runStateFns(converged bool) error {
+// runStateFns runs the list of stored state functions.
+func (obj *Coordinator) runStateFns(converged bool) error {
 	obj.smutex.RLock()
 	defer obj.smutex.RUnlock()
 	var keys []string
@@ -315,77 +334,125 @@ func (obj *converger) runStateFns(converged bool) error {
 	for _, name := range keys { // run in deterministic order
 		fn := obj.stateFns[name]
 		// call an arbitrary function
-		if e := fn(converged); e != nil {
-			err = multierr.Append(err, e) // list of errors
-		}
+		e := fn(converged)
+		err = errwrap.Append(err, e) // list of errors
 	}
 	return err
 }
 
-// ID returns the unique id of this UID object.
-func (obj *cuid) ID() uint64 {
-	return obj.id
+// AddStateFn adds a state function to be run on change of converged state.
+func (obj *Coordinator) AddStateFn(name string, stateFn func(bool) error) error {
+	obj.smutex.Lock()
+	defer obj.smutex.Unlock()
+	if _, exists := obj.stateFns[name]; exists {
+		return fmt.Errorf("a stateFn with that name already exists")
+	}
+	obj.stateFns[name] = stateFn
+	return nil
 }
 
-// Name returns a user defined name for the specific cuid.
-func (obj *cuid) Name() string {
-	return obj.name
+// RemoveStateFn removes a state function from running on change of converged
+// state.
+func (obj *Coordinator) RemoveStateFn(name string) error {
+	obj.smutex.Lock()
+	defer obj.smutex.Unlock()
+	if _, exists := obj.stateFns[name]; !exists {
+		return fmt.Errorf("a stateFn with that name doesn't exist")
+	}
+	delete(obj.stateFns, name)
+	return nil
 }
 
-// SetName sets a user defined name for the specific cuid.
-func (obj *cuid) SetName(name string) {
-	obj.name = name
+// Status returns a map of the converged status of each UID.
+func (obj *Coordinator) Status() map[*UID]bool {
+	status := make(map[*UID]bool)
+	obj.mutex.RLock() // take a read lock
+	defer obj.mutex.RUnlock()
+	for k := range obj.status {
+		status[k] = k.IsConverged()
+	}
+	return status
 }
 
-// IsValid tells us if the id is valid or has already been destroyed.
-func (obj *cuid) IsValid() bool {
-	return obj.id != 0 // an id of 0 is invalid
+// Timeout returns the timeout in seconds that converger was created with. This
+// is useful to avoid passing in the timeout value separately when you're
+// already passing in the Coordinator struct.
+func (obj *Coordinator) Timeout() int {
+	return obj.timeout
 }
 
-// InvalidateID marks the id as no longer valid.
-func (obj *cuid) InvalidateID() {
-	obj.id = 0 // an id of 0 is invalid
+// UID represents one of the probes for the converger coordinator. It is created
+// by calling the Register method of the Coordinator struct. It should be freed
+// after use with Unregister.
+type UID struct {
+	// timeout is a copy of the main timeout. It could eventually be used
+	// for per-UID timeouts too.
+	timeout int
+	// isConverged stores the convergence state of this particular UID.
+	isConverged bool
+
+	// poke stores a reference to the main poke function.
+	poke func()
+	// unregister stores a reference to the unregister function.
+	unregister func()
+
+	// timer
+	mutex   *sync.Mutex
+	timer   chan struct{}
+	running bool // is the timer running?
+	wg      *sync.WaitGroup
 }
 
-// IsConverged is a helper function to the regular IsConverged method.
-func (obj *cuid) IsConverged() bool {
-	return obj.converger.IsConverged(obj)
+// Unregister removes this UID from the converger coordinator. An unregistered
+// UID is no longer part of the convergence checking.
+func (obj *UID) Unregister() {
+	obj.unregister()
 }
 
-// SetConverged is a helper function to the regular SetConverged notification.
-func (obj *cuid) SetConverged(isConverged bool) error {
-	return obj.converger.SetConverged(obj, isConverged)
+// IsConverged reports whether this UID is converged or not.
+func (obj *UID) IsConverged() bool {
+	return obj.isConverged
 }
 
-// Unregister is a helper function to unregister myself.
-func (obj *cuid) Unregister() {
-	obj.converger.Unregister(obj)
+// SetConverged sets the convergence state of this UID. This is used by the
+// running timer if one is started. The timer will overwrite any value set by
+// this method.
+func (obj *UID) SetConverged(isConverged bool) {
+	obj.isConverged = isConverged
+	obj.poke() // notify of change
 }
 
-// ConvergedTimer is a helper around the regular ConvergedTimer method.
-func (obj *cuid) ConvergedTimer() <-chan time.Time {
-	return obj.converger.ConvergedTimer(obj)
+// ConvergedTimer adds a timeout to a select call and blocks until then.
+// TODO: this means we could eventually have per resource converged timeouts
+func (obj *UID) ConvergedTimer() <-chan time.Time {
+	// be clever: if i'm already converged, this timeout should block which
+	// avoids unnecessary new signals being sent! this avoids fast loops if
+	// we have a low timeout, or in particular a timeout == 0
+	if obj.IsConverged() {
+		// blocks the case statement in select forever!
+		return util.TimeAfterOrBlock(-1)
+	}
+	return util.TimeAfterOrBlock(int(obj.timeout))
 }
 
-// StartTimer runs an invisible timer that automatically converges on timeout.
-func (obj *cuid) StartTimer() (func() error, error) {
+// StartTimer runs a timer that sets us as converged on timeout. It also returns
+// a handle to the StopTimer function which should be run before exit.
+func (obj *UID) StartTimer() (func() error, error) {
 	obj.mutex.Lock()
-	if !obj.running {
-		obj.timer = make(chan struct{})
-		obj.running = true
-	} else {
-		obj.mutex.Unlock()
+	defer obj.mutex.Unlock()
+	if obj.running {
 		return obj.StopTimer, fmt.Errorf("timer already started")
 	}
-	obj.mutex.Unlock()
+	obj.timer = make(chan struct{})
+	obj.running = true
 	obj.wg.Add(1)
 	go func() {
 		defer obj.wg.Done()
 		for {
 			select {
 			case _, ok := <-obj.timer: // reset signal channel
-				if !ok { // channel is closed
-					return // false to exit
+				if !ok {
+					return
 				}
 				obj.SetConverged(false)
 
@@ -393,8 +460,8 @@ func (obj *cuid) StartTimer() (func() error, error) {
 				obj.SetConverged(true) // converged!
 				select {
 				case _, ok := <-obj.timer: // reset signal channel
-					if !ok { // channel is closed
-						return // false to exit
+					if !ok {
+						return
 					}
 				}
 			}
@@ -403,8 +470,8 @@ func (obj *cuid) StartTimer() (func() error, error) {
 	return obj.StopTimer, nil
 }
 
-// ResetTimer resets the counter to zero if using a StartTimer internally.
-func (obj *cuid) ResetTimer() error {
+// ResetTimer resets the timer to zero.
+func (obj *UID) ResetTimer() error {
 	obj.mutex.Lock()
 	defer obj.mutex.Unlock()
 	if obj.running {
@@ -414,8 +481,8 @@ func (obj *cuid) ResetTimer() error {
 	return fmt.Errorf("timer hasn't been started")
 }
 
-// StopTimer stops the running timer permanently until a StartTimer is run.
-func (obj *cuid) StopTimer() error {
+// StopTimer stops the running timer.
+func (obj *UID) StopTimer() error {
 	obj.mutex.Lock()
 	defer obj.mutex.Unlock()
 	if !obj.running {

converger/converger_test.go

@@ -0,0 +1,43 @@
+// Mgmt
+// Copyright (C) James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+//go:build !root
+
+package converger
+
+import (
+	"testing"
+)
+
+func TestBufferedChan1(t *testing.T) {
+	ch := make(chan bool, 1)
+	ch <- true
+	close(ch) // closing a channel that's not empty should not block
+	// must be able to exit without blocking anywhere
+}

debian/copyright

@@ -3,7 +3,7 @@ Upstream-Name: mgmt
 Source: <https://github.com/purpleidea/mgmt>
 
 Files: *
-Copyright: Copyright (C) 2013-2018+ James Shubin and the project contributors
+Copyright: Copyright (C) James Shubin and the project contributors
 License: GPL-3.0
 
 License: GPL-3.0
@@ -19,3 +19,15 @@ License: GPL-3.0
 
   You should have received a copy of the GNU General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+  Additional permission under GNU GPL version 3 section 7
+
+  If you modify this program, or any covered work, by linking or combining it
+  with embedded mcl code and modules (and that the embedded mcl code and
+  modules which link with this program, contain a copy of their source code in
+  the authoritative form) containing parts covered by the terms of any other
+  license, the licensors of this program grant you additional permission to
+  convey the resulting work. Furthermore, the licensors of this program grant
+  the original author, James Shubin, additional permission to update this
+  additional permission if he deems it necessary to achieve the goals of this
+  additional permission.

doc.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -13,7 +13,19 @@
 // GNU General Public License for more details.
 //
 // You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
 
 // Package main provides the main entrypoint for using the `mgmt` software.
 package main

docker/Dockerfile

@@ -1,10 +1,10 @@
-FROM golang:1.9
+FROM golang:1.23
 
 MAINTAINER Michał Czeraszkiewicz <contact@czerasz.com>
 
 # Set the reset cache variable
 # Read more here: http://czerasz.com/2014/11/13/docker-tip-and-tricks/#use-refreshedat-variable-for-better-cache-control
-ENV REFRESHED_AT 2017-11-16
+ENV REFRESHED_AT 2020-09-23
 
 # Update the package list to be able to use required packages
 RUN apt-get update

docker/Dockerfile-fedora.build

@@ -0,0 +1,20 @@
+FROM fedora:41
+LABEL org.opencontainers.image.authors="laurent.indermuehle@pm.me"
+
+ENV GOPATH=/root/gopath
+ENV PATH=/root/gopath/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/go/bin:/usr/local/bin
+ENV LD_LIBRARY_PATH=/usr/lib64
+ENV PKG_CONFIG_PATH=/usr/lib64/pkgconfig
+
+# This forces make-deps.sh to install Ragel 6.1 instead of 7.0
+ENV DOCKER=true
+
+RUN dnf -y install wget unzip git make which gcc gcc-c++ ruby golang
+
+RUN mkdir -p $GOPATH/src/github.com/purpleidea
+WORKDIR $GOPATH/src/github.com/purpleidea
+RUN git clone --recursive https://github.com/purpleidea/mgmt mgmt
+WORKDIR $GOPATH/src/github.com/purpleidea/mgmt
+RUN make deps
+RUN make build
+CMD ["/bin/bash"]

docker/Dockerfile.build

@@ -6,7 +6,7 @@ ENV PATH=/opt/rh/rh-ruby22/root/usr/bin:/root/gopath/bin:/usr/local/sbin:/sbin:/
 ENV LD_LIBRARY_PATH=/opt/rh/rh-ruby22/root/usr/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
 ENV PKG_CONFIG_PATH=/opt/rh/rh-ruby22/root/usr/lib64/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}
 
-RUN yum -y install epel-release wget unzip git make which centos-release-scl gcc && sed -i "s/enabled=0/enabled=1/" /etc/yum.repos.d/epel-testing.repo && yum -y install rh-ruby22 && wget -O /opt/go1.9.1.linux-amd64.tar.gz https://storage.googleapis.com/golang/go1.9.1.linux-amd64.tar.gz && tar -C /usr/local -xzf /opt/go1.9.1.linux-amd64.tar.gz
+RUN yum -y install epel-release wget unzip git make which centos-release-scl gcc && sed -i "s/enabled=0/enabled=1/" /etc/yum.repos.d/epel-testing.repo && yum -y install rh-ruby22 && wget -O /opt/go1.23.5.linux-amd64.tar.gz https://storage.googleapis.com/golang/go1.23.5.linux-amd64.tar.gz && tar -C /usr/local -xzf /opt/go1.23.5.linux-amd64.tar.gz
 RUN mkdir -p $GOPATH/src/github.com/purpleidea && cd $GOPATH/src/github.com/purpleidea && git clone --recursive https://github.com/purpleidea/mgmt
 RUN go get -u gopkg.in/alecthomas/gometalinter.v1 && cd $GOPATH/src/github.com/purpleidea/mgmt && make deps && make build
 CMD ["/bin/bash"]

docker/Dockerfile.development

@@ -1,10 +1,10 @@
-FROM golang:1.9
+FROM golang:1.23
 
 MAINTAINER Michał Czeraszkiewicz <contact@czerasz.com>
 
 # Set the reset cache variable
 # Read more here: http://czerasz.com/2014/11/13/docker-tip-and-tricks/#use-refreshedat-variable-for-better-cache-control
-ENV REFRESHED_AT 2017-11-16
+ENV REFRESHED_AT 2019-02-06
 
 RUN apt-get update
 
@@ -27,8 +27,5 @@ WORKDIR /home/$USER_NAME/mgmt
 # Install dependencies
 RUN make deps
 
-# Chown $GOPATH
-RUN chown -R ${USER_ID}:${GROUP_ID} /go
-
 # Change user
 USER ${USER_NAME}

docs/conf.py

@@ -51,7 +51,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'mgmt'
-copyright = u'2013-2018+ James Shubin and the project contributors'
+copyright = u'Copyright (C) James Shubin and the project contributors'
 author = u'James Shubin'
 
 # The version info for the project you're documenting, acts as replacement for

docs/contributing.md

@@ -0,0 +1,96 @@
+# Contributing
+
+What follows is a short guide with information for participants who wish to
+contribute to the project. It hopes to set both some expectations and boundaries
+so that we both benefit.
+
+## Small patches
+
+If you have a small patch which you believe is straightforward, should be easy
+to merge, and isn't overly onerous on your time to write, please feel free to
+send it our way without asking first. Bug fixes are excellent examples of small
+patches. Please make sure to familiarize yourself with the rough coding style of
+the project first, and read through the [style guide](style-guide.md).
+
+## Making an excellent small patch
+
+As a special case: We'd like to avoid minimal effort, one-off, drive-by patches
+by bots and contributors looking to increase their "activity" numbers. As an
+example: a patch which fixes a small linting issue isn't rousing, but a patch
+that adds a linter test _and_ fixes a small linting issue is, because it shows
+you put in more effort.
+
+## Medium patches
+
+Medium sized patches are especially welcome. Good examples of these patches
+can include writing a new `mgmt` resource or function. You'll generally need
+some knowledge of golang interfaces and concurrency to write these patches.
+Before writing one of these, please make sure you understand some basics about
+the project and how the tool works. After this, it is recommended that you join
+our discussion channel to suggest the idea, and ideally include the actual API
+you'd like to propose before writing the code and sending a patch.
+
+## Making an excellent medium patch proposal
+
+The "API" of a resource is the type signature of the resource struct, and the
+"API" of a function is the type signature or signatures that it supports. (Since
+functions can be polymorphic, more than one signature can be possible!) A good
+proposal would likely also comment on the mechanisms the resources or functions
+would use to watch for events, to check state, and to apply changes. If these
+mechanisms need new dependencies, a brief survey of which dependencies are
+available and why you recommend a particular one is encouraged.
+
+## Large patches or structural and core patches
+
+Please do not send us large, core or structurally significant patches without
+first getting our approval and without getting some medium patches in first.
+These patches take a lot of effort to review, and we don't want to skimp on our
+commitment to that if we can't muster it. Instead grow our relationship with you
+on the medium-sized patches first. (A core patch might refer to something that
+touches either the function engine, resource engine, compiler internals, or
+something that is part of one of the internal API's.)
+
+## Expectations and boundaries
+
+When interacting with the project and soliciting feedback (either for design or
+during a code review) please keep in mind that the project (unfortunately!) has
+time constraints and so must prioritize how it handles workloads. If you are
+someone who has successfully sent in small patches, we will be more willing to
+spend time mentoring your medium sized patches and so on. Think of it this way:
+as you show that you're contributing to the project, we'll contribute more to
+you. Put another way: we can't afford to spend large amounts of time discussing
+potential patches with you, just to end up nowhere. Build up your reputation
+with us, and we hope to help grow our symbiosis with you all the while as you
+grow too!
+
+## Energy output
+
+The same goes for users and issue creators. There are times when we simply don't
+have the cycles to discuss or litigate an issue with you. We wish we did have
+more time, but it is finite, and running a project is not free. Therefore,
+please keep in mind that you don't automatically qualify for free support or
+attention.
+
+## Attention seeking behaviours
+
+Some folks spend too much time starting discussions, commenting on issues,
+"planning" and otherwise displaying attention seeking behaviours. Please avoid
+doing this as much as possible, especially if you are not already a major
+contributor to the project. While it may be well intentioned, if it is
+indistinguishable to us from intentional interference, then it's not welcome
+behaviour. Remember that Free Software is not free to write. If you require more
+attention, then either contribute more to the project, or consider paying for a
+[support contract](https://mgmtconfig.com/).
+
+## Consulting
+
+Having said all that, there are some folks who want to do some longer-term
+planning to decide if our core design and architecture is right for them to
+invest in. If that's the case, and you aren't already a well-known project
+contributor, please [contact](https://mgmtconfig.com/) us for a consulting
+quote. We have packages available for both individuals and businesses.
+
+## Respect
+
+Please be mindful and respectful of others when interacting with the project and
+its contributors. If you cannot abide by that, you may no longer be welcome.

docs/development.md

@@ -3,7 +3,120 @@
 This document contains some additional information and help regarding
 developing `mgmt`. Useful tools, conventions, etc.
 
-Be sure to read [quick start guide](docs/quick-start-guide.md) first.
+Be sure to read [quick start guide](quick-start-guide.md) first.
+
+## Vagrant
+
+If you would like to avoid doing the above steps manually, we have prepared a
+[Vagrant](https://www.vagrantup.com/) environment for your convenience. From the
+project directory, run a `vagrant up`, and then a `vagrant status`. From there,
+you can `vagrant ssh` into the `mgmt` machine. The `MOTD` will explain the rest.
+This environment isn't commonly used by the `mgmt` developers, so it might not
+be working properly.
+
+## Using Docker
+
+Alternatively, you can check out the [docker folder](../docker/) in order to
+develop or deploy using docker. This method is not endorsed or supported, so use
+at your own risk, as it might not be working properly.
+
+## Information about dependencies
+
+Software projects have a few different kinds of dependencies. There are _build_
+dependencies, _runtime_ dependencies, and additionally, a few extra dependencies
+required for running the _test_ suite.
+
+### Build
+
+* A modern `golang` version. The version available in the current Fedora
+releases is usually supported. This is also distributed as a binary officially
+by [golang.org](https://golang.org/dl/).
+
+### Runtime
+
+A relatively modern GNU/Linux system should be able to run `mgmt` without any
+problems. Since `mgmt` runs as a single statically compiled binary, all of the
+library dependencies are included. It is expected, that certain advanced
+resources require host specific facilities to work. These requirements are
+listed below:
+
+| Resource | Dependency        | Version                     | Check version with                                        |
+|----------|-------------------|-----------------------------|-----------------------------------------------------------|
+| augeas   | augeas-devel      | `augeas 1.6` or greater     | `dnf info augeas-devel` or `apt-cache show libaugeas-dev` |
+| file     | inotify           | `Linux 2.6.27` or greater   | `uname -a`                                                |
+| hostname | systemd-hostnamed | `systemd 25` or greater     | `systemctl --version`                                     |
+| nspawn   | systemd-nspawn    | `systemd ???` or greater    | `systemctl --version`                                     |
+| pkg      | packagekitd       | `packagekit 1.x` or greater | `pkcon --version`                                         |
+| svc      | systemd           | `systemd ???` or greater    | `systemctl --version`                                     |
+| virt     | libvirt-devel     | `libvirt 1.2.0` or greater  | `dnf info libvirt-devel` or `apt-cache show libvirt-dev`  |
+| virt     | libvirtd          | `libvirt 1.2.0` or greater  | `libvirtd --version`                                      |
+
+For building a visual representation of the graph, `graphviz` is required.
+
+To build `mgmt` without augeas support please run:
+`GOTAGS='noaugeas' make build`
+
+To build `mgmt` without libvirt support please run:
+`GOTAGS='novirt' make build`
+
+To build `mgmt` without docker support please run:
+`GOTAGS='nodocker' make build`
+
+To build `mgmt` without augeas, libvirt or docker support please run:
+`GOTAGS='noaugeas novirt nodocker' make build`
+
+## OSX/macOS/Darwin development
+
+Developing and running `mgmt` on macOS is currently not supported (but not
+discouraged either). Meaning it might work but in the case it doesn't you would
+have to provide your own patches to fix problems (the project maintainer and
+community are glad to assist where needed).
+
+There are currently some issues that make `mgmt` less suitable to run for
+provisioning macOS. But as a client to provision remote servers it should run
+fine.
+
+Since the primary supported systems are Linux and these are the environments
+tested, it is wise to run these suites during macOS development as well. To ease
+this, Docker can be leveraged ([Docker for Mac](https://docs.docker.com/docker-for-mac/)).
+
+Before running any of the commands below create the development Docker image:
+
+```
+docker/scripts/build-development
+```
+
+This image requires updating every time dependencies (`make-deps.sh`) changes.
+
+Then to run the test suite:
+
+```
+docker run --rm -ti \
+	-v $PWD:/go/src/github.com/purpleidea/mgmt/ \
+	-w /go/src/github.com/purpleidea/mgmt/ \
+	purpleidea/mgmt:development \
+	make test
+```
+
+For convenience this command is wrapped in `docker/scripts/exec-development`.
+
+Basically any command can be executed this way. Because the repository source is
+mounted into the Docker container invocation will be quick and allow rapid
+testing, for example:
+
+```
+docker/scripts/exec-development test/test-shell.sh load0.sh
+```
+
+Other examples:
+
+```
+docker/scripts/exec-development make build
+docker/scripts/exec-development ./mgmt run --tmp-prefix lang examples/lang/load0.mcl
+```
+
+Be advised that this method is not supported and it might not be working
+properly.
 
 ## Testing
 
@@ -45,5 +158,6 @@ individual tests to run.
 
 ### IDE/Editor support
 
-- Emacs: see `misc/emacs/`
-- [Textmate](https://github.com/aequitas/mgmt.tmbundle)
+* Emacs: see `misc/emacs/`
+* [Textmate](https://github.com/aequitas/mgmt.tmbundle)
+* [VSCode](https://github.com/aequitas/mgmt.vscode)

docs/docs.go

@@ -0,0 +1,50 @@
+// Mgmt
+// Copyright (C) James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+// Package docs provides a tool that generates documentation from the source.
+//
+// ./mgmt docs generate --output /tmp/docs.json && cat /tmp/docs.json | jq
+package docs
+
+import (
+	"context"
+)
+
+// API is the simple interface we expect for any setup items.
+type API interface {
+	// Main runs everything for this setup item.
+	Main(context.Context) error
+}
+
+// Config is a struct of all the configuration values which are shared by all of
+// the setup utilities. By including this as a separate struct, it can be used
+// as part of the API if we want.
+type Config struct {
+	//Foo string `arg:"--foo,env:MGMT_DOCGEN_FOO" help:"Foo..."` // TODO: foo
+}

docs/documentation.md

@@ -122,6 +122,10 @@ entire set of running mgmt agents will need to all simultaneously converge for
 the group to exit. This is particularly useful for bootstrapping new clusters
 which need to exchange information that is only available at run time.
 
+This existed in earlier versions of mgmt as a `--remote` option, but it has been
+removed and is being ported to a more powerful variant where you can remote
+execute via a `remote` resource.
+
 #### Blog post
 
 You can read the introductory blog post about this topic here:
@@ -129,13 +133,13 @@ You can read the introductory blog post about this topic here:
 
 ### Puppet support
 
-You can supply a Puppet manifest instead of creating the (YAML) graph manually.
+You can supply a puppet manifest instead of creating the (YAML) graph manually.
 Puppet must be installed and in `mgmt`'s search path. You also need the
-[ffrank-mgmtgraph Puppet module](https://forge.puppet.com/ffrank/mgmtgraph).
+[ffrank-mgmtgraph puppet module](https://forge.puppet.com/ffrank/mgmtgraph).
 
 Invoke `mgmt` with the `--puppet` switch, which supports 3 variants:
 
-1. Request the configuration from the Puppet Master (like `puppet agent` does)
+1. Request the configuration from the puppet server (like `puppet agent` does)
 
 	`mgmt run puppet --puppet agent`
 
@@ -147,11 +151,11 @@ Invoke `mgmt` with the `--puppet` switch, which supports 3 variants:
 
 	`mgmt run puppet --puppet 'file { "/etc/ntp.conf": ensure => file }'`
 
-For more details and caveats see [Puppet.md](Puppet.md).
+For more details and caveats see [puppet-guide.md](puppet-guide.md).
 
 #### Blog post
 
-An introductory post on the Puppet support is on
+An introductory post on the puppet support is on
 [Felix's blog](http://ffrank.github.io/features/2016/06/19/puppet-powered-mgmt/).
 
 ## Reference
@@ -250,6 +254,66 @@ integer, then that value is the max size for that semaphore. Valid semaphore
 id's include: `some_id`, `hello:42`, `not:smart:4` and `:13`. It is expected
 that the last bare example be only used by the engine to add a global semaphore.
 
+#### Rewatch
+
+Boolean. Rewatch specifies whether we re-run the Watch worker during a graph
+swap if it has errored. When doing a graph compare to swap the graphs, if this
+is true, and this particular worker has errored, then we'll remove it and add it
+back as a new vertex, thus causing it to run again. This is different from the
+`Retry` metaparam which applies during the normal execution. It is only when
+this is exhausted that we're in permanent worker failure, and only then can we
+rely on this metaparam.
+
+#### Realize
+
+Boolean. Realize ensures that the resource is guaranteed to converge at least
+once before a potential graph swap removes or changes it. This guarantee is
+useful for fast changing graphs, to ensure that the brief creation of a resource
+is seen. This guarantee does not prevent against the engine quitting normally,
+and it can't guarantee it if the resource is blocked because of a failed
+pre-requisite resource.
+*XXX: This is currently not implemented!*
+
+#### Dollar
+
+Boolean. Dollar allows you to have a resource name that starts with a `$` sign.
+This is false by default. This helps you catch cases when you write code like:
+
+```mcl
+$foo = "/tmp/file1"
+file "$foo" {} # incorrect!
+```
+
+The above code would ignore the `$foo` variable and attempt to make a file named
+`$foo` which would obviously not work. To correctly interpolate a variable, you
+need to surround the name with curly braces.
+
+```mcl
+$foo = "/tmp/file1"
+file "${foo}" {} # correct!
+```
+
+This meta param is a safety measure to make your life easier. It works for all
+resources. If someone comes up with a resource which would routinely start with
+a dollar sign, then we can revisit the default for this resource kind.
+
+#### Reverse
+
+Boolean. Reverse is a property that some resources can implement that specifies
+that some "reverse" operation should happen when that resource "disappears". A
+disappearance happens when a resource is defined in one instance of the graph,
+and is gone in the subsequent one. This disappearance can happen if it was
+previously in an if statement that then becomes false.
+
+This is helpful for building robust programs with the engine. The engine adds a
+"reversed" resource to that subsequent graph to accomplish the desired "reverse"
+mechanics. The specifics of what this entails is a property of the particular
+resource that is being "reversed".
+
+It might be wise to combine the use of this meta parameter with the use of the
+`realize` meta parameter to ensure that your reversed resource actually runs at
+least once, if there's a chance that it might be gone for a while.
+
 ### Lang metadata file
 
 Any module *must* have a metadata file in its root. It must be named
@@ -298,10 +362,6 @@ recommended that you use this, since it's preferable to write code in the
 The main interface to the `mgmt` tool is the command line. For the most recent
 documentation, please run `mgmt --help`.
 
-#### `--yaml <graph.yaml>`
-
-Point to a graph file to run.
-
 #### `--converged-timeout <seconds>`
 
 Exit if the machine has converged for approximately this many seconds.
@@ -325,35 +385,13 @@ size of 42, you can expect a semaphore if named: `:42`. It is expected that
 consumers of the semaphore metaparameter always include a prefix to avoid a
 collision with this globally defined semaphore. The size value must be greater
 than zero at this time. The traditional non-parallel execution found in config
-management tools such as `Puppet` can be obtained with `--sema 1`.
-
-#### `--remote <graph.yaml>`
-
-Point to a graph file to run on the remote host specified within. This parameter
-can be used multiple times if you'd like to remotely run on multiple hosts in
-parallel.
-
-#### `--allow-interactive`
-
-Allow interactive prompting for SSH passwords if there is no authentication
-method that works.
+management tools such as `puppet` can be obtained with `--sema 1`.
 
 #### `--ssh-priv-id-rsa`
 
 Specify the path for finding SSH keys. This defaults to `~/.ssh/id_rsa`. To
 never use this method of authentication, set this to the empty string.
 
-#### `--cconns`
-
-The maximum number of concurrent remote ssh connections to run. This defaults
-to `0`, which means unlimited.
-
-#### `--no-caching`
-
-Don't allow remote caching of the remote execution binary. This will require
-the binary to be copied over for every remote execution, but it limits the
-likelihood that there is leftover information from the configuration process.
-
 #### `--prefix <path>`
 
 Specify a path to a custom working directory prefix. This directory will get
@@ -371,8 +409,8 @@ default prefix. This can't be combined with the `--prefix` option.
 If this option is specified, we will attempt to fall back to a temporary prefix
 if the primary prefix couldn't be created. This is useful for avoiding failures
 in environments where the primary prefix may or may not be available, but you'd
-like to try. The canonical example is when running `mgmt` with `--remote` there
-might be a cached copy of the binary in the primary prefix, but in case there's
+like to try. The canonical example is when running `mgmt` with remote execution
+there might be a cached copy of the binary in the primary prefix, but if there's
 no binary available continue working in a temporary directory to avoid failure.
 
 ### Compilation options
@@ -422,7 +460,7 @@ directory in the git source repository. It is available from:
 
 ### Systemd:
 
-See [`misc/mgmt.service`](misc/mgmt.service) for a sample systemd unit file.
+See [`misc/mgmt.service`](../misc/mgmt.service) for a sample systemd unit file.
 This unit file is part of the RPM.
 
 To specify your custom options for `mgmt` on a systemd distro:
@@ -455,7 +493,7 @@ To report any bugs, please file a ticket at: [https://github.com/purpleidea/mgmt
 
 ## Authors
 
-Copyright (C) 2013-2018+ James Shubin and the project contributors
+Copyright (C) James Shubin and the project contributors
 
 Please see the
 [AUTHORS](https://github.com/purpleidea/mgmt/tree/master/AUTHORS) file

docs/faq.md

@@ -9,6 +9,18 @@ I wanted a next generation config management solution that didn't have all of
 the design flaws or limitations that the current generation of tools do, and no
 tool existed!
 
+### Why did you choose `golang` for the project?
+
+When I started working on the project, I needed to choose a language that
+already had an implementation of a distributed consensus algorithm available.
+That meant [Paxos](https://en.wikipedia.org/wiki/Paxos_(computer_science)) or
+[Raft](https://en.wikipedia.org/wiki/Raft_(computer_science)). Golang was one
+language that actually had two different Raft implementations, `etcd`, and
+`consul`. Other design requirements included something that was reasonably fast,
+typed and memory-safe, and suited for systems engineering. After a reasonably
+extensive search, I chose `golang`. I think it was the right decision. There are
+a number of other features of the language which helped influence the decision.
+
 ### How do I contribute to the project if I don't know `golang`?
 
 There are many different ways you can contribute to the project. They can be
@@ -41,10 +53,11 @@ find a number of tutorials online.
 3. Spend between four to six hours with the [golang tour](https://tour.golang.org/).
 Skip over the longer problems, but try and get a solid overview of everything.
 If you forget something, you can always go back and repeat those parts.
-4. Connect to our [#mgmtconfig](https://webchat.freenode.net/?channels=#mgmtconfig)
-IRC channel on the [Freenode](https://freenode.net/) network. You can use any
-IRC client that you'd like, but the [hosted web portal](https://webchat.freenode.net/?channels=#mgmtconfig)
-will suffice if you don't know what else to use.
+4. Connect to our [#mgmtconfig](https://web.libera.chat/?channels=#mgmtconfig)
+IRC channel on the [Libera.Chat](https://libera.chat/) network. You can use any
+IRC client that you'd like, but the [hosted web portal](https://web.libera.chat/?channels=#mgmtconfig)
+will suffice if you don't know what else to use. [Here are a few suggestions for
+alternative clients.](https://libera.chat/guides/clients)
 5. Now it's time to try and starting writing a patch! We have tagged a bunch of
 [open issues as #mgmtlove](https://github.com/purpleidea/mgmt/issues?q=is%3Aissue+is%3Aopen+label%3Amgmtlove)
 for new users to have somewhere to get involved. Look through them to see if
@@ -125,6 +138,58 @@ The downside to this approach is that you won't benefit from the automatic
 elastic nature of the embedded etcd servers, and that you're responsible if you
 accidentally break your etcd cluster, or if you use an unsupported version.
 
+### In `mgmt` you talk about events. What is this referring to?
+
+Mgmt has two main concepts that involve "events":
+1. Events in the [resource primitive](resource-guide.md).
+2. Events in the [reactive language](language-guide.md).
+
+Each resource primitive in mgmt can test (check) and set (apply) the desired
+state that was requested of it. This is familiar to what is common with existing
+tools such as `Puppet`, `Ansible`, `Chef`, `Terraform`, etc... In addition,
+`mgmt` can also **watch** the state and detect changes. As a result, it never
+has to waste time and cpu resources by polling to test and set state, leading to
+a design which is algorithmically much faster than the existing generation of
+tools.
+
+To describe the set of resources to apply, mgmt describes this collection with a
+language. In order to model the time component of infrastructure, we use a
+special kind of language called an [FRP](https://en.wikipedia.org/wiki/Functional_reactive_programming).
+This language has a built-in concept that we call "events", and which means that
+we re-evaluate the relevant portions of the code whenever a value or function
+has an event that tells us that it changed. The `R` in `FRP` stands for
+reactive. This is similar to how a spreadsheet updates dependent cells when a
+pre-requisite value is modified. [This article](https://en.wikipedia.org/wiki/Reactive_programming)
+provides a bit more background.
+
+Whenever any of the streams of values in the language change, the program is
+partially re-evaluated. The output of any mgmt program is a [DAG](https://en.wikipedia.org/wiki/Directed_acyclic_graph)
+of resources, or more precisely, a stream of resource graphs. Since we have
+events per-resource, we can efficiently switch from one desired-state resource
+graph to the next without re-checking their individual states, since we've been
+monitoring them all along.
+
+One side-effect of all this, is that if a rogue systems administrator manually
+changes the state of any managed resource, mgmt will detect this and attempt to
+revert the change. This makes for excellent live demos, but is not the primary
+design goal. It is a consequence of tracking state so that graph changes are
+efficient. We implement the event detection via an intentional per-resource
+[main loop](https://en.wikipedia.org/wiki/Event_loop) which can enable other
+interesting functionality too!
+
+Make sure to get rid of your rogue sysadmin! ;)
+
+### Do I need to run `mgmt` as `root`?
+
+No and yes. It depends. Nothing in mgmt explicitly requires root in the design,
+however mgmt will require root only if the changes to your system that you want
+it to make require root.
+
+For example, if you use it to manage files that require root access to modify,
+then you'll need root. If you only use it to manage files and resources
+elsewhere, then it shouldn't need root. Many resources are perfectly usable
+without root, and virtually all of my live demos are done without root.
+
 ### How can I run `mgmt` on-demand, or in `cron`, instead of continuously?
 
 By default, `mgmt` will run continuously in an attempt to keep your machine in a
@@ -148,43 +213,243 @@ requires a number of seconds as an argument.
 #### Example:
 
 ```
-./mgmt run lang --lang examples/lang/hello0.mcl --converged-timeout=5
+./mgmt run lang examples/lang/hello0.mcl --converged-timeout=5
 ```
 
-### What does the error message about an inconsistent dataDir mean?
+### Can I run `mgmt` for type-checking only?
+
+Yes, you can, add the `--only-unify` option to the lang frontend while using the
+`run` command, and it will exit after type unification.
+
+#### Example:
+
+```
+./mgmt run --tmp-prefix lang --only-unify examples/lang/hello0.mcl
+```
+
+It will also print how long it took on either success or failure. Keep in mind
+that even if you pass type unification, an `mgmt` run can still fail later on
+for other reasons, although these are mostly runtime considerations.
+
+### Why is type unification happening twice with `mgmt run`?
+
+When you use the `run` action, it runs all of the compile time checks (including
+type unification) that are possible, and then packages everything up into a
+deployable object and runs the same mechanism that `mgmt deploy` uses, sending
+the deploy to itself. At this point, mgmt starts up as a server, and receives
+the deploy. It will then need to run type unification again before running the
+code.
+
+You can skip the first type unification check by adding the `--skip-unify`
+option to the lang frontend when using the `run` command.
+
+You can also skip this check when running the `deploy` action, but if your code
+doesn't pass, you might be deploying broken code. This is not recommended.
+
+#### Example:
+
+```
+./mgmt run --tmp-prefix lang --skip-unify examples/lang/hello0.mcl
+```
+
+### Why does my file resource error with `no such file or directory`?
+
+If you create a file resource and only specify the content like this:
+
+```
+file "/tmp/foo" {
+	content => "hello world\n",
+}
+```
+
+Then this will attempt to set the contents of that file to the desired string,
+but *only* if that file already exists. If you'd like to ensure that it also
+gets created in case it is not present, then you must also specify the state:
+
+```
+file "/tmp/foo" {
+	state => $const.res.file.state.exists,
+	content => "hello world\n",
+}
+```
+
+Similar logic applies for situations when you only specify the `mode` parameter.
+
+This all turns out to be more safe and "correct", in that it would error and
+prevent masking an error for a situation when you expected a file to already be
+at that location. It also turns out to simplify the internals significantly, and
+remove an ambiguous scenario with the reversible file resource.
+
+### Package resources error with: "The name is not activatable", what's wrong?
+
+You may see an error like:
+
+`main: error running auto edges: The name is not activatable`
+
+This can happen because the mgmt `pkg` resource uses a library and daemon called
+`PackageKit` to install packages. If it is not installed, then it cannot do its
+work. On Fedora system you may wish to run `dnf install /usr/bin/pkcon` or on a
+Debian system you may wish to run `apt install packagekit-tools`.
+
+PackageKit is excellent because it provides both an API and an event system to
+watch the package database for changes, and it abstracts away the differences
+between the various package managers. If you'd prefer to not need to install
+this tool, then you can contribute a native `pkg:rpm` and `pkg:deb` resource to
+mgmt!
+
+### When running mgmt, it says: "module path error: can't find a module path".
+
+You might get an error along the lines of:
+
+```
+could not set scope: import scope `git://github.com/purpleidea/mgmt/modules/some_module_name/` failed: module path error: can't find a module path
+```
+
+This usually means that you haven't specified the directory that mgmt should use
+when looking for modules. This could happen when using mgmt interactively or
+when it's being run as a service. In such cases you may want the main invocation
+to look something like:
+
+```
+mgmt run lang --module-path '/etc/mgmt/modules/' /etc/mgmt/main.mcl
+```
+
+### I get an error: "cannot open shared object file: No such file or directory".
+
+Mgmt currently uses two libraries that depend on `.so` files being installed on
+the host. Those are for `augeas` and `libvirt`. If those dependencies are not
+present, then mgmt will not run. The complete error might look like:
+
+```
+mgmt: error while loading shared libraries: libvirt-lxc.so.0: cannot open shared object file: No such file or directory
+```
+
+or:
+
+```
+mgmt: error while loading shared libraries: libaugeas.so.0: cannot open shared object file: No such file or directory
+```
+
+or something similar. There are two solutions to this:
+
+1. Use a build that doesn't include one or both of those features. You can build
+that like: `GOTAGS="noaugeas novirt nodocker" make build`.
+
+2. Install those dependencies. On a Fedora machine you might want to run:
+
+```
+dnf install libvirt-devel augeas-devel
+```
+
+On a Debian machine you might want to run:
+
+```
+apt install libvirt-dev libaugeas-dev
+```
+
+### Why do function names inside of templates include underscores?
+
+The golang template library which we use to implement the golang.template() func
+doesn't support the dot notation, so we import all our normal functions, and
+just replace dots with underscores. As an example, the standard `datetime.print`
+function is shown within mcl scripts as datetime_print after being imported.
+
+### On startup `mgmt` hangs after: `etcd: server: starting...`.
 
 If you get an error message similar to:
 
 ```
-Etcd: Connect: CtxError...
-Etcd: CtxError: Reason: CtxDelayErr(5s): No endpoints available yet!
-Etcd: Connect: Endpoints: []
-Etcd: The dataDir (/var/lib/mgmt/etcd) might be inconsistent or corrupt.
+etcd: server: starting...
+etcd: server: start timeout of 1m0s reached
+etcd: server: close timeout of 15s reached
 ```
 
-This happens when there are a series of fatal connect errors in a row. This can
-happen when you start `mgmt` using a dataDir that doesn't correspond to the
-current cluster view. As a result, the embedded etcd server never finishes
-starting up, and as a result, a default endpoint never gets added. The solution
-is to either reconcile the mistake, and if there is no important data saved, you
-can remove the etcd dataDir. This is typically `/var/lib/mgmt/etcd/member/`.
+But nothing happens afterwards, this can be due to a corrupt etcd storage
+directory. Each etcd server embedded in mgmt must have a special directory where
+it stores local state. It must not be shared by more than one individual member.
+This dir is typically `/var/lib/mgmt/etcd/member/`. If you accidentally use it
+(for example during testing) with a different cluster view, then you can corrupt
+it. This can happen if you use it with more than one different hostname.
 
-### Why do resources have both a `Cmp` method and an `IFF` (on the UID) method?
+The solution is to avoid making this mistake, and if there is no important data
+saved, you can remove the etcd member dir and start over.
 
-The `Cmp()` methods are for determining if two resources are effectively the
-same, which is used to make graph change delta's efficient. This is when we want
-to change from the current running graph to a new graph, but preserve the common
-vertices. Since we want to make this process efficient, we only update the parts
-that are different, and leave everything else alone. This `Cmp()` method can
-tell us if two resources are the same. In case it is not obvious, `cmp` is an
-abbrev. for compare.
+### On running `make` to build a new version, it errors with: `Text file busy`.
 
-The `IFF()` method is part of the whole UID system, which is for discerning if a
-resource meets the requirements another expects for an automatic edge. This is
-because the automatic edge system assumes a unified UID pattern to test for
-equality. In the future it might be helpful or sane to merge the two similar
-comparison functions although for now they are separate because they are
-actually answer different questions.
+If you get an error like:
+
+```
+cp: cannot create regular file 'mgmt': Text file busy
+```
+
+This can happen if you ran `make build` (or just `make`) when there was already
+an instance of mgmt running, or if a related file locking issue occurred. To
+solve this, shutdown and running mgmt process, run `rm mgmt` to remove the file,
+and then get a new one by running `make` again.
+
+### Type unification error with string interpolation.
+
+Look carefully at the following code:
+
+```
+$num = 42
+print "hello" {
+	msg => "My favourite number is ${num}",
+}
+```
+
+What's actually happening is that we can't unify `str + int`, because our
+addition operator only supports `str + str`, `int + int`, and `float + float`.
+The string interpolation feature can only combine strings. One solution is to
+instead write:
+
+```
+$num = "42" # now this is a string
+print "hello" {
+	msg => "My favourite number is ${num}",
+}
+```
+
+The first example will usually error with something along the lines of:
+
+`unify error with: topLevel(func() { <built-in:concat> }): type error: int != str`
+
+Now you know why this specific case doesn't work! We may reconsider allowing
+other types to be pulled into interpolation in the future. If you have a good
+case for this, then let us know.
+
+### The run and deploy commands don't parse correctly when used with `--seeds`.
+
+If you're running a command with `--seeds`, `--server-urls`, or `--client-urls`,
+then make sure you are using an equals sign between the flag name and the value.
+For example, if you were to run:
+
+```
+# wrong invocation!
+mgmt deploy --no-git --seeds http://127.0.0.1:2379 lang code/test.mcl
+```
+
+Then the `--seeds` flag would interpret `lang` and `code/test.mcl` as additional
+seeds. This flag as well as the other aforementioned ones all accept multiple
+values. Use an equals sign to guarantee you enter the correct data, eg:
+
+```
+# better invocation! (note the equals sign)
+mgmt deploy --no-git --seeds=http://127.0.0.1:2379 lang code/test.mcl
+```
+
+This is caused by a parsing peculiarity of the CLI library that we are using.
+This is tracked upstream at: [https://github.com/alexflint/go-arg/issues/239](https://github.com/alexflint/go-arg/issues/239).
+We have a workaround in place to mitigate it and attempt to show you a helpful
+error message, but it's also documented here in the meantime. The error you will
+see is: `cli parse error: missing equals sign for list element`.
+
+### The docs speaks of `--remote` but the CLI errors out?
+
+The `--remote` flag existed in an earlier version of mgmt. It was removed and
+will be replaced with a more powerful version, which is a "remote" resource. The
+code is mostly ready but it's not finished. If you'd like to help finish it or
+sponsor the work, please let me know.
 
 ### Does this support Windows? OSX? GNU Hurd?
 
@@ -193,7 +458,7 @@ serious automation workloads. Support for non-Linux operating systems isn't a
 high priority of mine, but we're happy to accept patches for missing features
 or resources that you think would make sense on your favourite platform.
 
-### Why aren't you using `glide` or `godep` for dependency management?
+### Why aren't you using `glide`, `godep` or `go mod` for dependency management?
 
 Vendoring dependencies means that as the git master branch of each dependency
 marches on, you're left behind using an old version. As a result, bug fixes and
@@ -248,6 +513,14 @@ Don't blindly use the tools that others tell you to. Learn what they do, think
 for yourself, and become a power user today! That process led us to using
 `git submodules`. Hopefully you'll come to the same conclusions that we did.
 
+**UPDATE:**
+
+After golang made it virtually impossible to build without `go.mod` stuff, we've
+switched to it since golang 1.16. I still think the above approach was better,
+and that the `go mod` tooling should have been a layer on top of git submodules
+so that we don't grow yet another lock file format, and existing folks who are
+comfortable with `git` can use those tools directly.
+
 ### Did you know that there is a band named `MGMT`?
 
 I didn't realize this when naming the project, and it is accidental. After much
@@ -261,10 +534,9 @@ which definitely existed before the band did.
 
 ### You didn't answer my question, or I have a question!
 
-It's best to ask on [IRC](https://webchat.freenode.net/?channels=#mgmtconfig)
-to see if someone can help you. Once we get a big enough community going, we'll
-add a mailing list. If you don't get any response from the above, you can
-contact me through my [technical blog](https://purpleidea.com/contact/)
-and I'll do my best to help. If you have a good question, please add it as a
-patch to this documentation. I'll merge your question, and add a patch with the
-answer!
+It's best to ask on [IRC](https://web.libera.chat/?channels=#mgmtconfig)
+to see if someone can help you. If you don't get a response from IRC, you can
+contact me through my [technical blog](https://purpleidea.com/contact/) and I'll
+do my best to help. If you have a good question, please add it as a patch to
+this documentation. I'll merge your question, and add a patch with the answer!
+For news and updates, subscribe to the [mailing list](https://www.redhat.com/mailman/listinfo/mgmtconfig-list).

docs/function-guide.md

@@ -37,10 +37,12 @@ available types and values in the mgmt language. It is very easy to use, and
 should be fairly intuitive. Most of what you'll need to know can be inferred
 from looking at example code.
 
-To implement a function, you'll need to create a file in
-[`lang/funcs/simple/`](https://github.com/purpleidea/mgmt/tree/master/lang/funcs/simple/).
-The function should be implemented as a `FuncValue` in our type system. It is
-then registered with the engine during `init()`. An example explains it best:
+To implement a function, you'll need to create a file that imports the
+[`lang/funcs/simple/`](https://github.com/purpleidea/mgmt/tree/master/lang/funcs/simple/)
+module. It should probably get created in the correct directory inside of:
+[`lang/core/`](https://github.com/purpleidea/mgmt/tree/master/lang/core/). The
+function should be implemented as a `simple.Scaffold` in our API. It is then
+registered with the engine during `init()`. An example explains it best:
 
 ### Example
 
@@ -48,17 +50,20 @@ then registered with the engine during `init()`. An example explains it best:
 package simple
 
 import (
+	"context"
 	"fmt"
 
+	"github.com/purpleidea/mgmt/lang/funcs/simple"
 	"github.com/purpleidea/mgmt/lang/types"
 )
 
 // you must register your functions in init when the program starts up
 func init() {
 	// Example function that squares an int and prints out answer as an str.
-	Register("talkingsquare", &types.FuncValue{
-		T: types.NewType("func(a int) str"), // declare the signature
-		V: func(input []types.Value) (types.Value, error) {
+
+	simple.ModuleRegister(ModuleName, "talkingsquare", &simple.Scaffold{
+		T: types.NewType("func(int) str"), // declare the signature
+		F: func(ctx context.Context, input []types.Value) (types.Value, error) {
 			i := input[0].Int() // get first arg as an int64
 			// must return the above specified value
 			return &types.StrValue{
@@ -84,102 +89,41 @@ mgmt engine to shutdown. It should be seen as the equivalent to calling a
 Ideally, your functions should never need to error. You should never cause a
 real `panic()`, since this could have negative consequences to the system.
 
-## Simple Polymorphic Function API
-
-Most functions should be implemented using the simple function API. If they need
-to have multiple polymorphic forms under the same name, then you can use this
-API. This is useful for situations when it would be unhelpful to name the
-functions differently, or when the number of possible signatures for the
-function would be infinite.
-
-The canonical example of this is the `len` function which returns the number of
-elements in either a `list` or a `map`. Since lists and maps are two different
-types, you can see that polymorphism is more convenient than requiring a
-`listlen` and `maplen` function. Nevertheless, it is also required because a
-`list of int` is a different type than a `list of str`, which is a different
-type than a `list of list of str` and so on. As you can see the number of
-possible input types for such a `len` function is infinite.
-
-Another downside to implementing your functions with this API is that they will
-*not* be made available for use inside templates. This is a limitation of the
-`golang` template library. In the future if this limitation proves to be
-significantly annoying, we might consider writing our own template library.
-
-As with the simple, non-polymorphic API, you can only implement [pure](https://en.wikipedia.org/wiki/Pure_function)
-functions, without writing too much boilerplate code. They will be automatically
-re-evaluated as needed when their input values change.
-
-To implement a function, you'll need to create a file in
-[`lang/funcs/simplepoly/`](https://github.com/purpleidea/mgmt/tree/master/lang/funcs/simplepoly/).
-The function should be implemented as a list of `FuncValue`'s in our type
-system. It is then registered with the engine during `init()`. You may also use
-the `variant` type in your type definitions. This special type will never be
-seen inside a running program, and will get converted to a concrete type if a
-suitable match to this signature can be found. Be warned that signatures which
-contain too many variants, or which are very general, might be hard for the
-compiler to match, and ambiguous type graphs make for user compiler errors.
-
-An example explains it best:
-
 ### Example
 
 ```golang
+package simple
+
 import (
+	"context"
 	"fmt"
 
+	"github.com/purpleidea/mgmt/lang/funcs/simple"
 	"github.com/purpleidea/mgmt/lang/types"
-	"github.com/purpleidea/mgmt/lang/funcs/simplepoly"
 )
 
 func init() {
-	simplepoly.Register("len", []*types.FuncValue{
-		{
-			T: types.NewType("func([]variant) int"),
-			V: Len,
-		},
-		{
-			T: types.NewType("func({variant: variant}) int"),
-			V: Len,
-		},
+	// This is the actual definition of the `len` function.
+	simple.Register("len", &simple.Scaffold{
+		T: types.NewType("func(?1) int"), // contains a unification var
+		C: simple.TypeMatch([]string{     // match on any of these sigs
+			"func(str) int",
+			"func([]?1) int",
+			"func(map{?1: ?2}) int",
+		}),
+		// The implementation is left as an exercise for the reader.
+		F: Len,
 	})
 }
-
-// Len returns the number of elements in a list or the number of key pairs in a
-// map. It can operate on either of these types.
-func Len(input []types.Value) (types.Value, error) {
-	var length int
-	switch k := input[0].Type().Kind; k {
-	case types.KindList:
-		length = len(input[0].List())
-	case types.KindMap:
-		length = len(input[0].Map())
-
-	default:
-		return nil, fmt.Errorf("unsupported kind: %+v", k)
-	}
-
-	return &types.IntValue{
-		V: int64(length),
-	}, nil
-}
 ```
 
-This simple polymorphic function can accept an infinite number of signatures, of
-which there are two basic forms. Both forms return an `int` as is seen above.
-The first form takes a `[]variant` which means a `list` of `variant`'s, which
-means that it can be a list of any type, since `variant` itself is not a
-concrete type. The second form accepts a `{variant: variant}`, which means that
-it accepts any form of `map` as input.
+## Simple Polymorphic Function API
 
-The implementation for both of these forms is the same: it is handled by the
-same `Len` function which is clever enough to be able to deal with any of the
-type signatures possible from those two patterns.
-
-At compile time, if your `mcl` code type checks correctly, a concrete type will
-be known for each and every usage of the `len` function, and specific values
-will be passed in for this code to compute the length of. As usual, make sure to
-only write safe code that will not panic! A panic is a bug. If you really cannot
-continue, then you must return an error.
+Most functions should be implemented using the simple function API. If they need
+to have multiple polymorphic forms under the same name, with each resultant type
+match needing to be paired to a different implementation, then you can use this
+API. This is useful for situations when the functions differ in output type
+only.
 
 ## Function API
 
@@ -190,7 +134,7 @@ if it meets your needs. Most functions will be able to use that API. If you
 really need something more powerful, then you can use the regular function API.
 What follows are each of the method signatures and a description of each.
 
-### Default
+### Info
 
 ```golang
 Info() *interfaces.Info
@@ -219,7 +163,7 @@ Init(init *interfaces.Init) error
 
 This is called to initialize the function. If something goes wrong, it should
 return an error. It is passed a struct that contains all the important
-information and poiinters that it might need to work with throughout its
+information and pointers that it might need to work with throughout its
 lifetime. As a result, it will need to save a copy to that pointer for future
 use in the other methods.
 
@@ -229,27 +173,6 @@ use in the other methods.
 // Init runs some startup code for this function.
 func (obj *FooFunc) Init(init *interfaces.Init) error {
 	obj.init = init
-	obj.closeChan = make(chan struct{}) // shutdown signal
-	return nil
-}
-```
-
-### Close
-
-```golang
-Close() error
-```
-
-This is called to cleanup the function. It usually causes the stream to
-shutdown. Even if `Stream()` decided to shutdown early, it might still get
-called. It is usually called by the engine to tell the function to shutdown.
-
-#### Example
-
-```golang
-// Close runs some shutdown code for this function and turns off the stream.
-func (obj *FooFunc) Close() error {
-	close(obj.closeChan) // send a signal to tell the stream to close
 	return nil
 }
 ```
@@ -257,23 +180,24 @@ func (obj *FooFunc) Close() error {
 ### Stream
 
 ```golang
-Stream() error
+Stream(context.Context) error
 ```
 
 `Stream` is where the real _work_ is done. This method is started by the
 language function engine. It will run this function while simultaneously sending
-it values on the `input` channel. It will only send a complete set of input
+it values on the `Input` channel. It will only send a complete set of input
 values. You should send a value to the output channel when you have decided that
 one should be produced. Make sure to only use input values of the expected type
 as declared in the `Info` struct, and send values of the similarly declared
 appropriate return type. Failure to do so will may result in a panic and
-sadness.
+sadness. You must shutdown if the input context cancels. You must close the
+`Output` channel if you are done generating new values and/or when you shutdown.
 
 #### Example
 
 ```golang
 // Stream returns the single value that was generated and then closes.
-func (obj *FooFunc) Stream() error {
+func (obj *FooFunc) Stream(ctx context.Context) error {
 	defer close(obj.init.Output) // the sender closes
 	var result string
 	for {
@@ -290,7 +214,7 @@ func (obj *FooFunc) Stream() error {
 
 			result = fmt.Sprintf("the input is: %d", ix)
 
-		case <-obj.closeChan:
+		case <-ctx.Done():
 			return nil
 		}
 
@@ -299,7 +223,7 @@ func (obj *FooFunc) Stream() error {
 			V: result,
 		}:
 
-		case <-obj.closeChan:
+		case <-ctx.Done():
 			return nil
 		}
 	}
@@ -330,8 +254,6 @@ type FooFunc struct {
 	init *interfaces.Init
 
 	// this space can be used if needed
-
-	closeChan chan struct{} // shutdown signal
 }
 ```
 
@@ -354,7 +276,9 @@ method instead.
 
 ```golang
 // moduleName is already set to "math" by the math package. Do this in `init`.
-funcs.ModuleRegister(moduleName, "cos", func() interfaces.Func { return &CosFunc{} })
+funcs.ModuleRegister(moduleName, "cos", func() interfaces.Func {
+	return &CosFunc{}
+})
 ```
 
 ### Composite functions
@@ -368,23 +292,6 @@ We don't expect this functionality to be particularly useful or common, as it's
 probably easier and preferable to simply import common golang library code into
 multiple different functions instead.
 
-## Polymorphic Function API
-
-The polymorphic function API is an API that lets you implement functions which
-do not necessarily have a single static function signature. After compile time,
-all functions must have a static function signature. We also know that there
-might be different ways you would want to call `printf`, such as:
-`printf("the %s is %d", "answer", 42)` or `printf("3 * 2 = %d", 3 * 2)`. Since
-you couldn't implement the infinite number of possible signatures, this API lets
-you write code which can be coerced into different forms. This makes
-implementing what would appear to be generic or polymorphic, instead something
-that is actually static and that still has the static type safety properties
-that were guaranteed by the mgmt language.
-
-Since this is an advanced topic, it is not described in full at this time. For
-more information please have a look at the source code comments, some of the
-existing implementations, and ask around in the community.
-
 ## Frequently asked questions
 
 (Send your questions as a patch to this FAQ! I'll review it, merge it, and
@@ -435,6 +342,11 @@ generator to build your `FuncValue` implementations, and pass in the unique
 signature to each one as you are building them. Using a generator is a common
 technique which was mentioned previously.
 
+One obvious situation where this might occur is if your function doesn't take
+any inputs! An example `math.fortytwo()` function was implemented that
+demonstrates the use of function generators to pass the type signatures into the
+implementations.
+
 ### Where can I find more information about mgmt?
 
 Additional blog posts, videos and other material [is available!](https://github.com/purpleidea/mgmt/blob/master/docs/on-the-web.md).

docs/generate.go

@@ -0,0 +1,795 @@
+// Mgmt
+// Copyright (C) James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+package docs
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	cliUtil "github.com/purpleidea/mgmt/cli/util"
+	docsUtil "github.com/purpleidea/mgmt/docs/util"
+	"github.com/purpleidea/mgmt/engine"
+	engineUtil "github.com/purpleidea/mgmt/engine/util"
+	"github.com/purpleidea/mgmt/lang/funcs"
+	"github.com/purpleidea/mgmt/lang/interfaces"
+	"github.com/purpleidea/mgmt/util"
+)
+
+const (
+	// JSONSuffix is the output extension for the generated documentation.
+	JSONSuffix = ".json"
+)
+
+// Generate is the main entrypoint for this command. It generates everything.
+type Generate struct {
+	*cliUtil.DocsGenerateArgs // embedded config
+	Config                    // embedded Config
+
+	// Program is the name of this program, usually set at compile time.
+	Program string
+
+	// Version is the version of this program, usually set at compile time.
+	Version string
+
+	// Debug represents if we're running in debug mode or not.
+	Debug bool
+
+	// Logf is a logger which should be used.
+	Logf func(format string, v ...interface{})
+}
+
+// Main runs everything for this setup item.
+func (obj *Generate) Main(ctx context.Context) error {
+	if err := obj.Validate(); err != nil {
+		return err
+	}
+
+	if err := obj.Run(ctx); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Validate verifies that the structure has acceptable data stored within.
+func (obj *Generate) Validate() error {
+	if obj == nil {
+		return fmt.Errorf("data is nil")
+	}
+	if obj.Program == "" {
+		return fmt.Errorf("program is empty")
+	}
+	if obj.Version == "" {
+		return fmt.Errorf("version is empty")
+	}
+
+	return nil
+}
+
+// Run performs the desired actions to generate the documentation.
+func (obj *Generate) Run(ctx context.Context) error {
+
+	outputFile := obj.DocsGenerateArgs.Output
+	if outputFile == "" || !strings.HasSuffix(outputFile, JSONSuffix) {
+		return fmt.Errorf("must specify output")
+	}
+	// support relative paths too!
+	if !strings.HasPrefix(outputFile, "/") {
+		wd, err := os.Getwd()
+		if err != nil {
+			return err
+		}
+		outputFile = filepath.Join(wd, outputFile)
+	}
+
+	if obj.Debug {
+		obj.Logf("output: %s", outputFile)
+	}
+
+	// Ensure the directory exists.
+	//d := filepath.Dir(outputFile)
+	//if err := os.MkdirAll(d, 0750); err != nil {
+	//	return fmt.Errorf("could not make output dir at: %s", d)
+	//}
+
+	resources, err := obj.genResources()
+	if err != nil {
+		return err
+	}
+
+	functions, err := obj.genFunctions()
+	if err != nil {
+		return err
+	}
+
+	data := &Output{
+		Version:   safeVersion(obj.Version),
+		Resources: resources,
+		Functions: functions,
+	}
+
+	b, err := json.Marshal(data)
+	if err != nil {
+		return err
+	}
+	b = append(b, '\n') // needs a trailing newline
+
+	if err := os.WriteFile(outputFile, b, 0600); err != nil {
+		return err
+	}
+	obj.Logf("wrote: %s", outputFile)
+
+	return nil
+}
+
+func (obj *Generate) getResourceInfo(kind, filename, structName string) (*ResourceInfo, error) {
+	rootDir := obj.DocsGenerateArgs.RootDir
+	if rootDir == "" {
+		wd, err := os.Getwd()
+		if err != nil {
+			return nil, err
+		}
+		rootDir = wd + "/" // add a trailing slash
+	}
+	if !strings.HasPrefix(rootDir, "/") || !strings.HasSuffix(rootDir, "/") {
+		return nil, fmt.Errorf("bad root dir: %s", rootDir)
+	}
+
+	// filename might be "noop.go" for example
+	p := filepath.Join(rootDir, engine.ResourcesRelDir, filename)
+
+	fset := token.NewFileSet()
+
+	// f is a: https://golang.org/pkg/go/ast/#File
+	f, err := parser.ParseFile(fset, p, nil, parser.ParseComments)
+	if err != nil {
+		return nil, err
+	}
+
+	// mcl field name to golang field name
+	mapping, err := engineUtil.LangFieldNameToStructFieldName(kind)
+	if err != nil {
+		return nil, err
+	}
+	// golang field name to mcl field name
+	nameMap, err := util.MapSwap(mapping)
+	if err != nil {
+		return nil, err
+	}
+	// mcl field name to mcl type
+	typMap, err := engineUtil.LangFieldNameToStructType(kind)
+	if err != nil {
+		return nil, err
+	}
+
+	ri := &ResourceInfo{}
+	// Populate the fields, even if they don't have a comment.
+	ri.Name = structName // golang name
+	ri.Kind = kind       // duplicate data
+	ri.File = filename
+	ri.Fields = make(map[string]*ResourceFieldInfo)
+	for mclFieldName, fieldName := range mapping {
+		typ, exists := typMap[mclFieldName]
+		if !exists {
+			continue
+		}
+
+		ri.Fields[mclFieldName] = &ResourceFieldInfo{
+			Name: fieldName,
+			Type: typ.String(),
+			Desc: "", // empty for now
+		}
+	}
+
+	var previousComment *ast.CommentGroup
+
+	// Walk through the AST...
+	ast.Inspect(f, func(node ast.Node) bool {
+
+		// Comments above the struct appear as a node right _before_ we
+		// find the struct, so if we see one, save it for later...
+		if cg, ok := node.(*ast.CommentGroup); ok {
+			previousComment = cg
+			return true
+		}
+
+		typeSpec, ok := node.(*ast.TypeSpec)
+		if !ok {
+			return true
+		}
+		name := typeSpec.Name.Name // name is now known!
+
+		// If the struct isn't what we're expecting, then move on...
+		if name != structName {
+			return true
+		}
+
+		// Check if the TypeSpec is a named struct type that we want...
+		st, ok := typeSpec.Type.(*ast.StructType)
+		if !ok {
+			return true
+		}
+
+		// At this point, we have the struct we want...
+
+		var comment *ast.CommentGroup
+		if typeSpec.Doc != nil {
+			// I don't know how to even get here...
+			comment = typeSpec.Doc // found!
+
+		} else if previousComment != nil {
+			comment = previousComment // found!
+			previousComment = nil
+		}
+
+		ri.Desc = commentCleaner(comment)
+
+		// Iterate over the fields of the struct
+		for _, field := range st.Fields.List {
+			// Check if the field has a comment associated with it
+			if field.Doc == nil {
+				continue
+			}
+
+			if len(field.Names) < 1 { // XXX: why does this happen?
+				continue
+			}
+
+			fieldName := field.Names[0].Name
+			if fieldName == "" { // Can this happen?
+				continue
+			}
+			if isPrivate(fieldName) {
+				continue
+			}
+
+			mclFieldName, exists := nameMap[fieldName]
+			if !exists {
+				continue
+			}
+
+			ri.Fields[mclFieldName].Desc = commentCleaner(field.Doc)
+		}
+
+		return true
+	})
+
+	return ri, nil
+}
+
+func (obj *Generate) genResources() (map[string]*ResourceInfo, error) {
+	resources := make(map[string]*ResourceInfo)
+	if obj.DocsGenerateArgs.NoResources {
+		return resources, nil
+	}
+
+	r := engine.RegisteredResourcesNames()
+	sort.Strings(r)
+	for _, kind := range r {
+		metadata, err := docsUtil.LookupResource(kind)
+		if err != nil {
+			return nil, err
+		}
+
+		if strings.HasPrefix(kind, "_") {
+			// TODO: Should we display these somehow?
+			// built-in resource
+			continue
+		}
+
+		ri, err := obj.getResourceInfo(kind, metadata.Filename, metadata.Typename)
+		if err != nil {
+			return nil, err
+		}
+
+		if ri.Name == "" {
+			return nil, fmt.Errorf("empty resource name: %s", kind)
+		}
+		if ri.File == "" {
+			return nil, fmt.Errorf("empty resource file: %s", kind)
+		}
+		if ri.Desc == "" {
+			obj.Logf("empty resource desc: %s", kind)
+		}
+		fields := []string{}
+		for field := range ri.Fields {
+			fields = append(fields, field)
+		}
+		sort.Strings(fields)
+		for _, field := range fields {
+			if ri.Fields[field].Desc == "" {
+				obj.Logf("empty resource (%s) field desc: %s", kind, field)
+			}
+		}
+
+		resources[kind] = ri
+	}
+
+	return resources, nil
+}
+
+func (obj *Generate) getFunctionInfo(pkg, name string, metadata *docsUtil.Metadata) (*FunctionInfo, error) {
+	rootDir := obj.DocsGenerateArgs.RootDir
+	if rootDir == "" {
+		wd, err := os.Getwd()
+		if err != nil {
+			return nil, err
+		}
+		rootDir = wd + "/" // add a trailing slash
+	}
+	if !strings.HasPrefix(rootDir, "/") || !strings.HasSuffix(rootDir, "/") {
+		return nil, fmt.Errorf("bad root dir: %s", rootDir)
+	}
+	if metadata.Filename == "" {
+		return nil, fmt.Errorf("empty filename for: %s.%s", pkg, name)
+	}
+
+	// filename might be "pow.go" for example and contain a rel dir
+	p := filepath.Join(rootDir, funcs.FunctionsRelDir, metadata.Filename)
+
+	fset := token.NewFileSet()
+
+	// f is a: https://golang.org/pkg/go/ast/#File
+	f, err := parser.ParseFile(fset, p, nil, parser.ParseComments)
+	if err != nil {
+		return nil, err
+	}
+
+	fi := &FunctionInfo{}
+	fi.Name = metadata.Typename
+	fi.File = metadata.Filename
+
+	var previousComment *ast.CommentGroup
+	found := false
+
+	rawFunc := func(node ast.Node) (*ast.CommentGroup, string) {
+		fd, ok := node.(*ast.FuncDecl)
+		if !ok {
+			return nil, ""
+		}
+		return fd.Doc, fd.Name.Name // name is now known!
+	}
+
+	rawStruct := func(node ast.Node) (*ast.CommentGroup, string) {
+		typeSpec, ok := node.(*ast.TypeSpec)
+		if !ok {
+			return nil, ""
+		}
+
+		// Check if the TypeSpec is a named struct type that we want...
+		if _, ok := typeSpec.Type.(*ast.StructType); !ok {
+			return nil, ""
+		}
+
+		return typeSpec.Doc, typeSpec.Name.Name // name is now known!
+	}
+
+	// Walk through the AST...
+	ast.Inspect(f, func(node ast.Node) bool {
+
+		// Comments above the struct appear as a node right _before_ we
+		// find the struct, so if we see one, save it for later...
+		if cg, ok := node.(*ast.CommentGroup); ok {
+			previousComment = cg
+			return true
+		}
+
+		doc, name := rawFunc(node) // First see if it's a raw func.
+		if name == "" {
+			doc, name = rawStruct(node) // Otherwise it's a struct.
+		}
+
+		// If the func isn't what we're expecting, then move on...
+		if name != metadata.Typename {
+			return true
+		}
+
+		var comment *ast.CommentGroup
+		if doc != nil {
+			// I don't know how to even get here...
+			comment = doc // found!
+
+		} else if previousComment != nil {
+			comment = previousComment // found!
+			previousComment = nil
+		}
+
+		fi.Desc = commentCleaner(comment)
+		found = true
+
+		return true
+	})
+
+	if !found {
+		//return nil, nil
+	}
+
+	return fi, nil
+}
+
+func (obj *Generate) genFunctions() (map[string]*FunctionInfo, error) {
+	functions := make(map[string]*FunctionInfo)
+	if obj.DocsGenerateArgs.NoFunctions {
+		return functions, nil
+	}
+
+	m := funcs.Map() // map[string]func() interfaces.Func
+	names := []string{}
+	for name := range m {
+		names = append(names, name)
+	}
+	sort.Slice(names, func(i, j int) bool {
+		a := names[i]
+		b := names[j]
+		// TODO: do a sorted-by-package order.
+		return a < b
+	})
+
+	for _, name := range names {
+		//v := m[name]
+		//fn := v()
+		fn := m[name]()
+
+		// eg: golang/strings.has_suffix
+		sp := strings.Split(name, ".")
+		if len(sp) == 0 {
+			return nil, fmt.Errorf("unexpected empty function")
+		}
+		if len(sp) > 2 {
+			return nil, fmt.Errorf("unexpected function name: %s", name)
+		}
+		n := sp[0]
+		p := sp[0]
+		if len(sp) == 1 { // built-in
+			p = "" // no package!
+		}
+		if len(sp) == 2 { // normal import
+			n = sp[1]
+		}
+
+		if strings.HasPrefix(n, "_") {
+			// TODO: Should we display these somehow?
+			// built-in function
+			continue
+		}
+
+		var sig *string
+		//iface := ""
+		if x := fn.Info().Sig; x != nil {
+			s := x.String()
+			sig = &s
+			//iface = "simple"
+		}
+
+		metadata := &docsUtil.Metadata{}
+
+		// XXX: maybe we need a better way to get this?
+		mdFunc, ok := fn.(interfaces.MetadataFunc)
+		if !ok {
+			// Function doesn't tell us what the data is, let's try
+			// to get it automatically...
+			metadata.Typename = funcs.GetFunctionName(fn) // works!
+			metadata.Filename = ""                        // XXX: How can we get this?
+
+			// XXX: We only need this back-channel metadata store
+			// because we don't know how to get the filename without
+			// manually writing code in each function. Alternatively
+			// we could add a New() method to each struct and then
+			// we could modify the struct instead of having it be
+			// behind a copy which is needed to get new copies!
+			var err error
+			metadata, err = docsUtil.LookupFunction(name)
+			if err != nil {
+				return nil, err
+			}
+
+		} else if mdFunc == nil {
+			// programming error
+			return nil, fmt.Errorf("unexpected empty metadata for function: %s", name)
+
+		} else {
+			metadata = mdFunc.GetMetadata()
+		}
+
+		if metadata == nil {
+			return nil, fmt.Errorf("unexpected nil metadata for function: %s", name)
+		}
+
+		// This may be an empty func name if the function did not know
+		// how to get it. (This is normal for automatic regular funcs.)
+		if metadata.Typename == "" {
+			metadata.Typename = funcs.GetFunctionName(fn) // works!
+		}
+
+		fi, err := obj.getFunctionInfo(p, n, metadata)
+		if err != nil {
+			return nil, err
+		}
+		// We may not get any fields added if we can't find anything...
+		fi.Name = metadata.Typename
+		fi.Package = p
+		fi.Func = n
+		fi.File = metadata.Filename
+		//fi.Desc = desc
+		fi.Signature = sig
+
+		// Hack for golang generated functions!
+		if strings.HasPrefix(fi.Package, "golang/") && fi.File == "generated_funcs.go" {
+			pkg := fi.Package[len("golang/"):]
+			frag := strings.TrimPrefix(fi.Name, strings.Title(strings.Join(strings.Split(pkg, "/"), ""))) // yuck
+			fi.File = fmt.Sprintf("https://pkg.go.dev/%s#%s", pkg, frag)
+		}
+
+		if fi.Func == "" {
+			return nil, fmt.Errorf("empty function name: %s", name)
+		}
+		if fi.File == "" {
+			return nil, fmt.Errorf("empty function file: %s", name)
+		}
+		if fi.Desc == "" {
+			obj.Logf("empty function desc: %s", name)
+		}
+		if fi.Signature == nil {
+			obj.Logf("empty function sig: %s", name)
+		}
+
+		functions[name] = fi
+	}
+
+	return functions, nil
+}
+
+// Output is the type of the final data that will be for the json output.
+type Output struct {
+	// Version is the sha1 or ref name of this specific version. This is
+	// used if we want to generate documentation with links matching the
+	// correct version. If unspecified then this assumes git master.
+	Version string `json:"version"`
+
+	// Resources contains the collection of every available resource!
+	// FIXME: should this be a list instead?
+	Resources map[string]*ResourceInfo `json:"resources"`
+
+	// Functions contains the collection of every available function!
+	// FIXME: should this be a list instead?
+	Functions map[string]*FunctionInfo `json:"functions"`
+}
+
+// ResourceInfo stores some information about each resource.
+type ResourceInfo struct {
+	// Name is the golang name of this resource.
+	Name string `json:"name"`
+
+	// Kind is the kind of this resource.
+	Kind string `json:"kind"`
+
+	// File is the file name where this resource exists.
+	File string `json:"file"`
+
+	// Desc explains what this resource does.
+	Desc string `json:"description"`
+
+	// Fields is a collection of each resource field and corresponding info.
+	Fields map[string]*ResourceFieldInfo `json:"fields"`
+}
+
+// ResourceFieldInfo stores some information about each field in each resource.
+type ResourceFieldInfo struct {
+	// Name is what this field is called in golang format.
+	Name string `json:"name"`
+
+	// Type is the mcl type for this field.
+	Type string `json:"type"`
+
+	// Desc explains what this field does.
+	Desc string `json:"description"`
+}
+
+// FunctionInfo stores some information about each function.
+type FunctionInfo struct {
+	// Name is the golang name of this function. This may be an actual
+	// function if used by the simple API, or the name of a struct.
+	Name string `json:"name"`
+
+	// Package is the import name to use to get to this function.
+	Package string `json:"package"`
+
+	// Func is the name of the function in that package.
+	Func string `json:"func"`
+
+	// File is the file name where this function exists.
+	File string `json:"file"`
+
+	// Desc explains what this function does.
+	Desc string `json:"description"`
+
+	// Signature is the type signature of this function. If empty then the
+	// signature is not known statically and it may be polymorphic.
+	Signature *string `json:"signature,omitempty"`
+}
+
+// commentCleaner takes a comment group and returns it as a clean string. It
+// removes the spurious newlines and programmer-focused comments. If there are
+// blank lines, it replaces them with a single newline. The idea is that the
+// webpage formatter would replace the newline with a <br /> or similar. This
+// code is a modified alternative of the ast.CommentGroup.Text() function.
+func commentCleaner(g *ast.CommentGroup) string {
+	if g == nil {
+		return ""
+	}
+	comments := make([]string, len(g.List))
+	for i, c := range g.List {
+		comments[i] = c.Text
+	}
+
+	lines := make([]string, 0, 10) // most comments are less than 10 lines
+	for _, c := range comments {
+		// Remove comment markers.
+		// The parser has given us exactly the comment text.
+		switch c[1] {
+		case '/':
+			//-style comment (no newline at the end)
+			c = c[2:]
+			if len(c) == 0 {
+				// empty line
+				break
+			}
+			if isDevComment(c[1:]) { // get rid of one space
+				continue
+			}
+			if c[0] == ' ' {
+				// strip first space - required for Example tests
+				c = c[1:]
+				break
+			}
+			//if isDirective(c) {
+			//	// Ignore //go:noinline, //line, and so on.
+			//	continue
+			//}
+		case '*':
+			/*-style comment */
+			c = c[2 : len(c)-2]
+		}
+
+		// Split on newlines.
+		cl := strings.Split(c, "\n")
+
+		// Walk lines, stripping trailing white space and adding to list.
+		for _, l := range cl {
+			lines = append(lines, stripTrailingWhitespace(l))
+		}
+	}
+
+	// Remove leading blank lines; convert runs of interior blank lines to a
+	// single blank line.
+	n := 0
+	for _, line := range lines {
+		if line != "" || n > 0 && lines[n-1] != "" {
+			lines[n] = line
+			n++
+		}
+	}
+	lines = lines[0:n]
+
+	// Concatenate all of these together. Blank lines should be a newline.
+	s := ""
+	for i, line := range lines {
+		if line == "" {
+			continue
+		}
+		s += line
+		if i < len(lines)-1 { // Is there another line?
+			if lines[i+1] == "" {
+				s += "\n" // Will eventually be a line break.
+			} else {
+				s += " "
+			}
+		}
+	}
+
+	return s
+}
+
+// TODO: should we use unicode.IsSpace instead?
+func isWhitespace(ch byte) bool { return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' }
+
+// TODO: should we replace with a strings package stdlib function?
+func stripTrailingWhitespace(s string) string {
+	i := len(s)
+	for i > 0 && isWhitespace(s[i-1]) {
+		i--
+	}
+	return s[0:i]
+}
+
+// isPrivate specifies if a field name is "private" or not.
+func isPrivate(fieldName string) bool {
+	if fieldName == "" {
+		panic("invalid field name")
+	}
+	x := fieldName[0:1]
+
+	if strings.ToLower(x) == x {
+		return true // it was already private
+	}
+
+	return false
+}
+
+// isDevComment tells us that the comment is for developers only!
+func isDevComment(comment string) bool {
+	if strings.HasPrefix(comment, "TODO:") {
+		return true
+	}
+	if strings.HasPrefix(comment, "FIXME:") {
+		return true
+	}
+	if strings.HasPrefix(comment, "XXX:") {
+		return true
+	}
+	return false
+}
+
+// safeVersion parses the main version string and returns a short hash for us.
+// For example, we might get a string of 0.0.26-176-gabcdef012-dirty as input,
+// and we'd want to return abcdef012.
+func safeVersion(version string) string {
+	const dirty = "-dirty"
+
+	s := version
+	if strings.HasSuffix(s, dirty) { // helpful dirty remover
+		s = s[0 : len(s)-len(dirty)]
+	}
+
+	ix := strings.LastIndex(s, "-")
+	if ix == -1 { // assume we have a standalone version (future proofing?)
+		return s
+	}
+	s = s[ix+1:]
+
+	// From the `git describe` man page: The "g" prefix stands for "git" and
+	// is used to allow describing the version of a software depending on
+	// the SCM the software is managed with. This is useful in an
+	// environment where people may use different SCMs.
+	const g = "g"
+	if strings.HasPrefix(s, g) {
+		s = s[len(g):]
+	}
+
+	return s
+}

docs/language-guide.md

@@ -54,7 +54,7 @@ can be impossible to infer the item's type.
 
 An unordered set of unique keys of the same type and corresponding value pairs
 of another type, eg:
-`{"boiling" => 100, "freezing" => 0, "room" => "25", "house" => 22, "canada" => -30,}`.
+`{"boiling" => 100, "freezing" => 0, "room" => 25, "house" => 22, "canada" => -30,}`.
 That is to say, all of the keys must have the same type, and all of the values
 must have the same type. You can use any type for either, although it is
 probably advisable to avoid using very complex types as map keys.
@@ -206,7 +206,7 @@ value to use if that boolean is true. You can do this with the resource-specific
 $b = true # change me to false and then try editing the file manually
 file "/tmp/mgmt-elvis" {
 	content => $b ?: "hello world\n",
-	state => "exists",
+	state => $const.res.file.state.exists,
 }
 ```
 
@@ -283,6 +283,14 @@ one of many ways you can perform iterative tasks that you might have
 traditionally used a `for` loop for instead. This is preferred, because flow
 control is error-prone and can make for less readable code.
 
+The single `str` variation, may only be used when it is possible for the
+compiler to determine statically that the value is of that type. Otherwise, it
+will assume it to be a list of strings. Programmers should explicitly wrap their
+variables in a string by interpolation to force this static `str` determination,
+or in square brackets to force a list. The former is generally preferable
+because it generates a smaller function graph since it doesn't need to build a
+list.
+
 ##### Internal edges
 
 Resources may also declare edges internally. The edges may point to or from
@@ -293,7 +301,7 @@ send notifications. You may have multiples of these per resource, including
 multiple `Depend` lines if necessary. Each of these properties also supports the
 conditional inclusion `elvis` operator as well.
 
-For example, you may write is:
+For example, you may write:
 
 ```mcl
 $b = true # for example purposes
@@ -337,6 +345,28 @@ to express a relationship between three resources. The first character in the
 resource kind must be capitalized so that the parser can't ascertain
 unambiguously that we are referring to a dependency relationship.
 
+##### Edge naming
+
+Each edge must have a unique name of type `str` that is used to uniquely
+identify that edge, and can be used in the functioning of the edge at its
+discretion.
+
+Alternatively, the name value may be a list of strings `[]str` to build a list
+of edges, each with a name from that list.
+
+Using this construct is a veiled form of looping (iteration). This technique is
+one of many ways you can perform iterative tasks that you might have
+traditionally used a `for` loop for instead. This is preferred, because flow
+control is error-prone and can make for less readable code.
+
+The single `str` variation, may only be used when it is possible for the
+compiler to determine statically that the value is of that type. Otherwise, it
+will assume it to be a list of strings. Programmers should explicitly wrap their
+variables in a string by interpolation to force this static `str` determination,
+or in square brackets to force a list. The former is generally preferable
+because it generates a smaller function graph since it doesn't need to build a
+list.
+
 #### Class
 
 A class is a grouping structure that bind's a list of statements to a name in
@@ -402,6 +432,38 @@ time.
 Recursive classes are not currently supported and it is not clear if they will
 be in the future. Discussion about this topic is welcome on the mailing list.
 
+A class names can contain colons to indicate it is nested inside of the class in
+the same scope which is named with the prefix indicated by colon separation.
+Instead of needing to repeatedly indent the child classes, we can instead prefix
+them at the definition site (where created with the class keyword) with the name
+of the parent class, followed by a colon, to get the desired embedded sugar.
+
+For example, instead of writing:
+
+```mcl
+class base() {
+	class inner() {
+		class deepest() {
+		}
+	}
+}
+```
+
+You can instead write:
+
+```mcl
+class base() {
+}
+class base:inner() {
+}
+class base:inner:deepest() {
+}
+```
+
+Of course, you can only access any of the inner classes by first including
+(with the include keyword) a parent class, and then subsequently including the
+inner one.
+
 #### Include
 
 The `include` statement causes the previously defined class to produce the
@@ -414,6 +476,57 @@ parameters, then the same class can even be called with different signatures.
 Whether the output is useful and whether there is a unique type unification
 solution is dependent on your code.
 
+Classes can be included under a new scoped prefix by using the `as` field and an
+identifier. When used in this manner, the captured scope of the class at its
+definition site are made available in the scope of the include. Variables,
+functions, and child classes are all exported.
+
+Variables are available in the include scope:
+
+```mcl
+import "fmt"
+
+class c1 {
+	test "t1" {}	# gets pulled out
+	$x = "hello"	# gets exported
+}
+include c1 as i1
+
+test "print0" {
+	anotherstr => fmt.printf("%s", $i1.x),	# hello
+	onlyshow => ["AnotherStr",], # displays nicer
+}
+```
+
+Classes are also available in the new scope:
+
+```mcl
+import "fmt"
+class c1 {
+	test "t1" {}	# gets pulled out
+	$x = "hello"	# gets exported
+
+	class c0 {
+		test "t2" {}
+		$x = "goodbye"
+	}
+}
+include c1 as i1
+include i1.c0 as i0
+
+test "print0" {
+	anotherstr => fmt.printf("%s", $i1.x),	# hello
+	onlyshow => ["AnotherStr",], # displays nicer
+}
+test "print1" {
+	anotherstr => fmt.printf("%s", $i0.x),	# goodbye
+	onlyshow => ["AnotherStr",], # displays nicer
+}
+```
+
+Of course classes can be parameterized too, and those variables specified during
+the `include`.
+
 #### Import
 
 The `import` statement imports a scope into the specified namespace. A scope can
@@ -478,7 +591,7 @@ Lexing is done using [nex](https://github.com/blynn/nex). It is a pure-golang
 implementation which is similar to _Lex_ or _Flex_, but which produces golang
 code instead of C. It integrates reasonably well with golang's _yacc_ which is
 used for parsing. The token definitions are in:
-[lang/lexer.nex](https://github.com/purpleidea/mgmt/tree/master/lang/lexer.nex).
+[lang/lexer.nex](https://github.com/purpleidea/mgmt/tree/master/lang/parser/lexer.nex).
 Lexing and parsing run together by calling the `LexParse` method.
 
 #### Parsing
@@ -490,7 +603,7 @@ and trial and error. One small advantage yacc has over standard yacc is that it
 can produce error messages from examples. The best documentation is to examine
 the source. There is a short write up available [here](https://research.swtch.com/yyerror).
 The yacc file exists at:
-[lang/parser.y](https://github.com/purpleidea/mgmt/tree/master/lang/parser.y).
+[lang/parser.y](https://github.com/purpleidea/mgmt/tree/master/lang/parser/parser.y).
 Lexing and parsing run together by calling the `LexParse` method.
 
 #### Interpolation
@@ -502,6 +615,10 @@ and can be used for other scenarios in which one statement or expression would
 be better represented by a larger AST. Most nodes in the AST simply return their
 own node address, and do not modify the AST.
 
+This stage also implements the class nesting when it finds class names with
+colons that should be nested inside of a base class. Currently this does modify
+the AST for efficiency and simplicity.
+
 #### Scope propagation
 
 Scope propagation passes the parent scope (starting with the top-level, built-in
@@ -511,6 +628,9 @@ without making any changes. The `ExprVar` node naturally consumes scope's and
 the `StmtProg` node cleverly passes the scope through in the order expected for
 the out-of-order bind logic to work.
 
+This step typically calls the ordering algorithm to determine the correct order
+of statements in a program.
+
 #### Type unification
 
 Each expression must have a known type. The unpleasant option is to force the
@@ -519,23 +639,27 @@ so that each `Expr` node in the AST knows what to expect. Type annotation is
 allowed in situations when you want to explicitly specify a type, or when the
 compiler cannot deduce it, however, most of it can usually be inferred.
 
-For type inferrence to work, each node in the AST implements a `Unify` method
-which is able to return a list of invariants that must hold true. This starts at
-the top most AST node, and gets called through to it's children to assemble a
-giant list of invariants. The invariants can take different forms. They can
-specify that a particular expression must have a particular type, or they can
-specify that two expressions must have the same types. More complex invariants
-allow you to specify relationships between different types and expressions.
-Furthermore, invariants can allow you to specify that only one invariant out of
-a set must hold true.
+For type inference to work, each `Stmt` node in the AST implements a `TypeCheck`
+method which is able to return a list of invariants that must hold true. This
+starts at the top most AST node, and gets called through to it's children to
+assemble a giant list of invariants. The invariants all have the same form. They
+specify that a particular expression corresponds to two particular types which
+may both contain unification variables.
+
+Each `Expr` node in the AST implements an `Infer` and `Check` method. The
+`Infer` method returns the type of that node along with a list of invariants as
+described above. Unification variables can of course be used throughout. The
+`Check` method always uses a generic check implementation and generally doesn't
+need to be implemented by the user.
 
 Once the list of invariants has been collected, they are run through an
 invariant solver. The solver can return either return successfully or with an
-error. If the solver returns successfully, it means that it has found a trivial
+error. If the solver returns successfully, it means that it has found a single
 mapping between every expression and it's corresponding type. At this point it
 is a simple task to run `SetType` on every expression so that the types are
-known. If the solver returns in error, it is usually due to one of two
-possibilities:
+known. During this stage, each SetType method verifies that it's a compatible
+type that it can use. If either that method or if the solver returns in error,
+it is usually due to one of two possibilities:
 
 1. Ambiguity
 
@@ -555,16 +679,16 @@ possibilities:
 	always happens if the user has made a type error in their program.
 
 Only one solver currently exists, but it is possible to easily plug in an
-alternate implementation if someone more skilled in the art of solver design
-would like to propose a more logical or performant variant.
+alternate implementation if someone wants to experiment with the art of solver
+design and would like to propose a more logical or performant variant.
 
 #### Function graph generation
 
-At this point we have a fully type AST. The AST must now be transformed into a
+At this point we have a fully typed AST. The AST must now be transformed into a
 directed, acyclic graph (DAG) data structure that represents the flow of data as
 necessary for everything to be reactive. Note that this graph is *different*
 from the resource graph which is produced and sent to the engine. It is just a
-coincidence that both happen to be DAG's. (You don't freak out when you see a
+coincidence that both happen to be DAG's. (You aren't surprised when you see a
 list data structure show up in more than one place, do you?)
 
 To produce this graph, each node has a `Graph` method which it can call. This
@@ -572,9 +696,8 @@ starts at the top most node, and is called down through the AST. The edges in
 the graphs must represent the individual expression values which are passed
 from node to node. The names of the edges must match the function type argument
 names which are used in the definition of the corresponding function. These
-corresponding functions must exist for each expression node and are produced by
-calling that expression's `Func` method. These are usually called by the
-function engine during function creation and validation.
+corresponding functions must exist for each expression node and are produced as
+the vertices of this returned graph. This is built for the function engine.
 
 #### Function engine creation and validation
 
@@ -598,8 +721,9 @@ If you'd like to create a built-in, core function, you'll need to implement the
 function API interface named `Func`. It can be found in
 [lang/interfaces/func.go](https://github.com/purpleidea/mgmt/tree/master/lang/interfaces/func.go).
 Your function must have a specific type. For example, a simple math function
-might have a signature of `func(x int, y int) int`. As you can see, all the
-types are known _before_ compile time.
+might have a signature of `func(x int, y int) int`. The simple functions have
+their types known _before_ compile time. You may also include unification
+variables in the function signature as long as the top-level type is a function.
 
 A separate discussion on this matter can be found in the [function guide](function-guide.md).
 
@@ -627,6 +751,12 @@ added in the future. This method is usually called before any other, and should
 not depend on any other method being called first. Other methods must not depend
 on this method being called first.
 
+If you use any unification variables in the function signature, then your
+function will *not* be made available for use inside templates. This is a
+limitation of the `golang` templating library. In the future if this limitation
+proves to be significantly annoying, we might consider writing our own template
+library.
+
 #### Example
 
 ```golang
@@ -637,6 +767,18 @@ func (obj *FooFunc) Info() *interfaces.Info {
 }
 ```
 
+#### Example
+
+This example contains unification variables.
+
+```golang
+func (obj *FooFunc) Info() *interfaces.Info {
+	return &interfaces.Info{
+		Sig: types.NewType("func(a ?1, b ?2, foo [?3]) ?1"),
+	}
+}
+```
+
 ### Init
 
 ```golang
@@ -674,13 +816,13 @@ one value must be produced.
 
 ```golang
 Please see the example functions in
-[lang/funcs/core/](https://github.com/purpleidea/mgmt/tree/master/lang/funcs/core/).
+[lang/core/](https://github.com/purpleidea/mgmt/tree/master/lang/core/).
 ```
 
 ### Stream
 
 ```golang
-Stream() error
+Stream(context.Context) error
 ```
 
 Stream is called by the function engine when it is ready for your function to
@@ -689,74 +831,77 @@ value. Failure to produce at least one value will probably cause the function
 engine to hang waiting for your output. This function must close the `Output`
 channel when it has no more values to send. The engine will close the `Input`
 channel when it has no more values to send. This may or may not influence
-whether or not you close the `Output` channel.
+whether or not you close the `Output` channel. You must shutdown if the input
+context cancels.
 
 #### Example
 
 ```golang
 Please see the example functions in
-[lang/funcs/core/](https://github.com/purpleidea/mgmt/tree/master/lang/funcs/core/).
+[lang/core/](https://github.com/purpleidea/mgmt/tree/master/lang/core/).
 ```
 
-### Close
+### BuildableFunc Function API
 
-```golang
-Close() error
-```
-
-Close asks the particular function to shutdown its `Stream()` function and
-return.
-
-#### Example
-
-```golang
-Please see the example functions in
-[lang/funcs/core/](https://github.com/purpleidea/mgmt/tree/master/lang/funcs/core/).
-```
-
-### Polymorphic Function API
-
-For some functions, it might be helpful to be able to implement a function once,
-but to have multiple polymorphic variants that can be chosen at compile time.
-For this more advanced topic, you will need to use the
-[Polymorphic Function API](#polymorphic-function-api). This will help with code
-reuse when you have a small, finite number of possible type signatures, and also
-for more complicated cases where you might have an infinite number of possible
-type signatures. (eg: `[]str`, or `[][]str`, or `[][][]str`, etc...)
+For some functions, it might be helpful to have a function which needs a "build"
+step which is run after type unification. This step can be used to build the
+function using the determined type, but it may also just be used for checking
+that unification picked a valid solution.
 
 Suppose you want to implement a function which can assume different type
 signatures. The mgmt language does not support polymorphic types-- you must use
 static types throughout the language, however, it is legal to implement a
 function which can take different specific type signatures based on how it is
 used. For example, you might wish to add a math function which could take the
-form of `func(x int, x int) int` or `func(x float, x float) float` depending on
-the input values. You might also want to implement a function which takes an
-arbitrary number of input arguments (the number must be statically fixed at the
-compile time of your program though) and which returns a string.
+form of `func(x int, y int) int` or `func(x float, y float) float` depending on
+the input values. For this case you could use a signature containing unification
+variables, eg: `func(x ?1, y ?1) ?1`. At the end the buildable function would
+need to check that it received a `?1` type of either `int` or `float`, since
+this function might not support doing math on strings. Remember that type
+unification can only return zero or one solutions, it's not possible to return
+more than one, which is why this secondary validation step is a brilliant way to
+filter out invalid solutions without needing to encode them as algebraic
+conditions during the solver state, which would otherwise make it exponential.
 
-The `PolyFunc` interface adds additional methods which you must implement to
-satisfy such a function implementation. If you'd like to implement such a
-function, then please notify the project authors, and they will expand this
-section with a longer description of the process.
+### InferableFunc Function API
 
-#### Examples
+You might also want to implement a function which takes an arbitrary number of
+input arguments (the number must be statically fixed at the compile time of your
+program though) and which returns a string or something else.
 
-What follows are a few examples that might help you understand some of the
-language details.
+The `InferableFunc` interface adds ad additional `FuncInfer` method which you
+must implement to satisfy such a function implementation. This lets you
+dynamically generate a type signature (including unification variables) and a
+list of invariants before running the type unification solver. It takes as input
+a list of the statically known input types and input values (if any) and as well
+the number of input arguments specified. This is usually enough information to
+generate a fixed type signature of a fixed size.
 
-##### Example Foo
-
-TODO: please add an example here!
-
-##### Example Bar
-
-TODO: please add an example here!
+Using this API should generally be pretty rare, but it is how certain special
+functions such as `fmt.printf` are built. If you'd like to implement such a
+function, then please notify the project authors as we're curious about your
+use case.
 
 ## Frequently asked questions
 
 (Send your questions as a patch to this FAQ! I'll review it, merge it, and
 respond by commit with the answer.)
 
+### Why am I getting a deploy.readfile error when the file actually exists?
+
+You may be seeing an error like:
+
+`readfile`: open /*/files/foo: file does not exist can't read file `/files/foo`?
+
+If you look, the `foo` file is indeed in the `files/` directory. The problem is
+that the `files/` directory won't be seen if you didn't specify to include it as
+part of your deploy. To do so, chances are that all you need to do is add a
+`metadata.yaml` file into the parent directory to that files folder. This will
+be used as the entrypoint instead of the naked `main.mcl` file that you have
+there, and with that metadata entrypoint, you get a default `files/` directory
+added. You can of course change the `files/` path by setting a key in the
+`metadata.yaml` file, but we recommend you leave it as the default.
+
 ### What is the difference between `ExprIf` and `StmtIf`?
 
 The language contains both an `if` expression, and and `if` statement. An `if`

docs/on-the-web.md

@@ -21,15 +21,15 @@ if we missed something that you think is relevant!
 | Felix Frank | blog | [Puppet Powered Mgmt (puppet to mgmt tl;dr)](https://ffrank.github.io/features/2016/06/19/puppet-powered-mgmt/) |
 | James Shubin | blog | [Automatic clustering in mgmt](https://purpleidea.com/blog/2016/06/20/automatic-clustering-in-mgmt/) |
 | James Shubin | video | [Recording from CoreOSFest 2016](https://www.youtube.com/watch?v=KVmDCUA42wc&html5=1) |
-| James Shubin | video | [Recording from DebConf16](http://meetings-archive.debian.net/pub/debian-meetings/2016/debconf16/Next_Generation_Config_Mgmt.webm) ([Slides](https://annex.debconf.org//debconf-share/debconf16/slides/15-next-generation-config-mgmt.pdf)) |
+| James Shubin | video | [Recording from DebConf16](http://meetings-archive.debian.net/pub/debian-meetings/2016/debconf16/Next_Generation_Config_Mgmt.webm) |
 | Felix Frank | blog | [Edging It All In (puppet and mgmt edges)](https://ffrank.github.io/features/2016/07/12/edging-it-all-in/) |
 | Felix Frank | blog | [Translating All The Things (puppet to mgmt translation warnings)](https://ffrank.github.io/features/2016/08/19/translating-all-the-things/) |
-| James Shubin | video | [Recording from systemd.conf 2016](https://www.youtube.com/watch?v=jB992Zb3nH0&html5=1) |
+| James Shubin | video | [Recording from systemd.conf 2016](https://www.youtube.com/watch?v=_TowsFAWWRA) |
 | James Shubin | blog | [Remote execution in mgmt](https://purpleidea.com/blog/2016/10/07/remote-execution-in-mgmt/) |
-| James Shubin | video | [Recording from High Load Strategy 2016](https://vimeo.com/191493409) |
-| James Shubin | video | [Recording from NLUUG 2016](https://www.youtube.com/watch?v=MmpwOQAb_SE&html5=1) |
+| James Shubin | video | [Recording from High Load Strategy 2016](https://www.youtube.com/watch?v=-4g14KUVPVk) |
+| James Shubin | video | [Recording from NLUUG 2016](https://www.youtube.com/watch?v=0vO93ni1zos) |
 | James Shubin | blog | [Send/Recv in mgmt](https://purpleidea.com/blog/2016/12/07/sendrecv-in-mgmt/) |
-| Julien Pivotto | blog | [Augeas resource for mgmt](https://roidelapluie.be/blog/2017/02/14/mgmt-augeas/) |
+| Julien Pivotto | blog | [Augeas resource for mgmt](https://purpleidea.com/cached/mgmt-augeas.html) (Cached from: https://roidelapluie.be/blog/2017/02/14/mgmt-augeas/) |
 | James Shubin | blog | [Metaparameters in mgmt](https://purpleidea.com/blog/2017/03/01/metaparameters-in-mgmt/) |
 | James Shubin | video | [Recording from Incontro DevOps 2017](https://vimeo.com/212241877) |
 | Yves Brissaud | blog | [mgmt aux HumanTalks Grenoble (french)](http://log.winsos.net/2017/04/12/mgmt-aux-human-talks-grenoble.html) |
@@ -44,3 +44,20 @@ if we missed something that you think is relevant!
 | James Shubin | blog | [Mgmt Configuration Language](https://purpleidea.com/blog/2018/02/05/mgmt-configuration-language/) |
 | James Shubin | video | [Recording from CfgMgmtCamp.eu 2018](https://www.youtube.com/watch?v=NxObmwZDyrI) |
 | Jonathan Gold | blog | [Go Netlink and Select](https://jonathangold.ca/blog/go-netlink-and-select/) |
+| James Shubin | video | [Recording from DevOpsDays Montreal 2018](https://www.youtube.com/watch?v=1i38c5cooHo) |
+| James Shubin | video | [Recording from FOSDEM Minimalistic Languages Devroom 2019](https://video.fosdem.org/2019/K.4.201/mgmtconfig.webm) |
+| James Shubin | video | [Recording from FOSDEM Infra Management Devroom 2019](https://video.fosdem.org/2019/UB2.252A/mgmt.webm) |
+| James Shubin | video | [Recording from FOSDEM Graph Processing Devroom 2019](https://video.fosdem.org/2019/H.1308/graph_mgmt_config.webm) |
+| James Shubin | video | [Recording from FOSDEM Virtualization Devroom 2019](https://video.fosdem.org/2019/H.2213/vai_real_time_virtualization_automation.webm) |
+| James Shubin | video | [Recording from FOSDEM Containers Devroom 2019](https://video.fosdem.org/2019/UA2.114/containers_mgmt.webm) |
+| James Shubin | video | [Recording from FOSDEM Monitoring Devroom 2019](https://video.fosdem.org/2019/UB2.252A/real_time_merging_of_config_management_and_monitoring.webm) |
+| James Shubin | blog | [Mgmt Configuration Language: Class and Include](https://purpleidea.com/blog/2019/07/26/class-and-include-in-mgmt/) |
+| James Shubin | video | [Recording from FOSDEM 2020, Main Track (History)](https://video.fosdem.org/2020/Janson/automation.webm) |
+| James Shubin | video | [Recording from FOSDEM 2020, Infra Management Devroom](https://video.fosdem.org/2020/UA2.120/mgmt.webm) |
+| James Shubin | video | [Recording from FOSDEM 2020, Minimalistic Languages Devroom](https://video.fosdem.org/2020/AW1.125/mgmtconfigmore.webm) |
+| James Shubin | video | [Recording from CfgMgmtCamp.eu 2020](https://www.youtube.com/watch?v=Kd7FAORFtsc) |
+| James Shubin | video | [Recording from CfgMgmtCamp.eu 2023](https://www.youtube.com/watch?v=FeRGRj8w0BU) |
+| James Shubin | video | [Recording from FOSDEM 2024, Golang Devroom](https://video.fosdem.org/2024/ud2218a/fosdem-2024-2575-single-binary-full-stack-provisioning.mp4) |
+| James Shubin | video | [Recording from CfgMgmtCamp.eu 2024](https://www.youtube.com/watch?v=vBt9lpGD4bc) |
+| James Shubin | blog | [Mgmt Configuration Language: Functions](https://purpleidea.com/blog/2024/11/22/functions-in-mgmt/) |
+| James Shubin | blog | [Modules and imports in mgmt](https://purpleidea.com/blog/2024/12/03/modules-and-imports-in-mgmt/) |

docs/puppet-guide.md

@@ -69,7 +69,7 @@ be avoided.
 ### Unsupported resources
 
 Puppet has a fairly large number of
-[built-in types](https://docs.puppet.com/puppet/latest/reference/type.html),
+[built-in types](https://www.puppet.com/docs/puppet/8/cheatsheet_core_types.html),
 and countless more are available through
 [modules](https://forge.puppet.com/). It's unlikely that all of them will
 eventually receive native counterparts in `mgmt`.
@@ -95,7 +95,8 @@ For example, translating a plain `file` resource will lead to a warning message:
 
 ```
 $ puppet mgmtgraph print --code 'file { "/tmp/mgmt-test": }'
-Warning: File[/tmp/mgmt-test] uses the 'puppet' file bucket, which mgmt cannot do. There will be no backup copies!
+Warning: File[/tmp/mgmt-test] uses the 'puppet' file bucket, which mgmt cannot
+do. There will be no backup copies!
 ```
 
 The reason is that per default, Puppet assumes the following parameter value
@@ -114,7 +115,7 @@ puppet mgmtgraph print --code 'file { "/tmp/mgmt-test": backup => false }'
 ```
 
 This is tedious in a more complex manifest. A good simplification is the
-following [resource default](https://docs.puppet.com/puppet/latest/reference/lang_defaults.html)
+following [resource default](https://www.puppet.com/docs/puppet/8/lang_defaults)
 anywhere on the top scope of your manifest:
 
 ```puppet
@@ -173,7 +174,7 @@ useful when you are in the process of replacing Puppet with mgmt. You
 can translate your custom modules into mgmt's language one by one,
 and let mgmt run the current mix.
 
-Instead of the usual `--puppet`, `--puppet-conf`, and `--lang` for mcl,
+Instead of the usual `--puppet-conf` flag and argv for `puppet` and `mcl` input,
 you need to use alternative flags to make this work:
 
 * `--lp-lang` to specify the mcl input

docs/quick-start-guide.md

@@ -2,65 +2,122 @@
 
 ## Introduction
 
-This guide is intended for developers. Once `mgmt` is minimally viable, we'll
-publish a quick start guide for users too. If you're brand new to `mgmt`, it's
-probably a good idea to start by reading the
-[introductory article](https://purpleidea.com/blog/2016/01/18/next-generation-configuration-mgmt/)
-or to watch an [introductory video](https://www.youtube.com/watch?v=LkEtBVLfygE&html5=1).
-Once you're familiar with the general idea, please start hacking...
+This guide is intended for users and developers. If you're brand new to `mgmt`,
+it's probably a good idea to start by reading an
+[introductory article about the engine](https://purpleidea.com/blog/2016/01/18/next-generation-configuration-mgmt/)
+and an [introductory article about the language](https://purpleidea.com/blog/2018/02/05/mgmt-configuration-language/).
+[There are other articles and videos available](on-the-web.md) if you'd like to
+learn more or prefer different formats. Once you're familiar with the general
+idea, or if you prefer a hands-on approach, please start hacking...
 
-## Quick start
+## Getting mgmt
 
-### Installing golang
+You can either build `mgmt` from source, or you can download a pre-built
+release. There are also some distro repositories available, but they may not be
+up to date. A pre-built release is the fastest option if there's one that's
+available for your platform. If you are developing or testing a new patch to
+`mgmt`, or there is not a release available for your platform, then you'll have
+to build your own.
 
-* You need golang version 1.10 or greater installed.
+### Downloading a pre-built release:
+
+The latest releases can be found [here](https://github.com/purpleidea/mgmt/releases/).
+An alternate mirror is available [here](https://dl.fedoraproject.org/pub/alt/purpleidea/mgmt/releases/).
+
+Make sure to verify the signatures of all packages before you use them. The
+signing key can be downloaded from [https://purpleidea.com/contact/#pgp-key](https://purpleidea.com/contact/#pgp-key)
+to verify the release.
+
+If you've decided to install a pre-build release, you can skip to the
+[Running mgmt](#running-mgmt) section below!
+
+### Building a release:
+
+You'll need some dependencies, including `golang`, and some associated tools.
+
+#### Installing golang
+
+* You need a modern golang version installed.
 	* To install on rpm style systems: `sudo dnf install golang`
 	* To install on apt style systems: `sudo apt install golang`
 	* To install on macOS systems install [Homebrew](https://brew.sh)
 	and run: `brew install go`
 * You can run `go version` to check the golang version.
-* If your distro is tool old, you may need to [download](https://golang.org/dl/)
+* If your distro is too old, you may need to [download](https://golang.org/dl/)
 a newer golang version.
 
-### Setting up golang
+#### Setting up golang
 
-* If you do not have a GOPATH yet, create one and export it:
+* You can skip this step, as your installation will default to using `~/go/`,
+but if you do not have a `GOPATH` yet and want one in a custom location, create
+one and export it:
 
-```
+```shell
 mkdir $HOME/gopath
 export GOPATH=$HOME/gopath
 ```
 
 * You might also want to add the GOPATH to your `~/.bashrc` or `~/.profile`.
-* For more information you can read the [GOPATH documentation](https://golang.org/cmd/go/#hdr-GOPATH_environment_variable).
+* For more information you can read the
+[GOPATH documentation](https://golang.org/cmd/go/#hdr-GOPATH_environment_variable).
 
-### Getting the mgmt code and dependencies
+#### Getting the mgmt code and associated dependencies
 
-* Download the `mgmt` code into the GOPATH, and switch to that directory:
+* Download the `mgmt` code and switch to that directory:
 
-```
-mkdir -p $GOPATH/src/github.com/purpleidea/
-cd $GOPATH/src/github.com/purpleidea/
-git clone --recursive https://github.com/purpleidea/mgmt/
-cd $GOPATH/src/github.com/purpleidea/mgmt
+```shell
+git clone --recursive https://github.com/purpleidea/mgmt/ ~/mgmt/
+cd ~/mgmt/
 ```
 
-* Add $GOPATH/bin to $PATH
+* Add `$GOPATH/bin` to `$PATH`
 
-```
+```shell
 export PATH=$PATH:$GOPATH/bin
 ```
 
 * Run `make deps` to install system and golang dependencies. Take a look at
-`misc/make-deps.sh` for details.
-* Run `make build` to get a freshly built `mgmt` binary.
+`misc/make-deps.sh` if you want to see the details of what it does.
 
-### Running mgmt
+#### Building mgmt
 
-* Run `time ./mgmt run --tmp-prefix lang --lang examples/lang/hello0.mcl` to try
-out a very simple example!
+* Now run `make` to get a freshly built `mgmt` binary. If this succeeds, you can
+proceed to the [Running mgmt](#running-mgmt) section below!
+
+### Installing a distro release
+
+Installation of `mgmt` from distribution packages currently needs improvement.
+They are not always up-to-date with git master and as such are not recommended.
+At the moment we have:
+* [COPR](https://copr.fedoraproject.org/coprs/purpleidea/mgmt/) (currently dead)
+* [Arch](https://aur.archlinux.org/packages/mgmt/) (currently stale)
+
+Please contribute more and help improve these! We'd especially like to see a
+Debian package!
+
+### Building from a container:
+
+This method avoids polluting your workstation with the dependencies for the
+build. Here is an example using Fedora, Podman and Buildah:
+
+```shell
+git clone --recursive https://github.com/purpleidea/mgmt/
+cd mgmt
+docker build -t mgmt -f docker/Dockerfile .
+docker run --rm --entrypoint cat mgmt mgmt > mgmt
+chmod +x mgmt
+./mgmt --version
+# you could now copy the mgmt binary somewhere into your $PATH
+# e.g., /usr/local/bin/ to make it accessible from anywhere
+```
+
+## Running mgmt
+
+* Run `mgmt run --tmp-prefix lang examples/lang/hello0.mcl` to try out a very
+simple example! If you built it from source, you'll need to use `./mgmt` from
+the project directory.
 * Look in that example file that you ran to see if you can figure out what it
-did!
+did! You can press `^C` to exit `mgmt`.
 * Have fun hacking on our future technology and get involved to shape the
 project!
 
@@ -68,118 +125,3 @@ project!
 
 Please look in the [examples/lang/](../examples/lang/) folder for some more
 examples!
-
-## Vagrant
-
-If you would like to avoid doing the above steps manually, we have prepared a
-[Vagrant](https://www.vagrantup.com/) environment for your convenience. From the
-project directory, run a `vagrant up`, and then a `vagrant status`. From there,
-you can `vagrant ssh` into the `mgmt` machine. The MOTD will explain the rest.
-
-## Using Docker
-
-Alternatively, you can check out the [docker-guide](docs/docker-guide.md) in
-order to develop or deploy using docker.
-
-## Information about dependencies
-
-Software projects have a few different kinds of dependencies. There are _build_
-dependencies, _runtime_ dependencies, and additionally, a few extra dependencies
-required for running the _test_ suite.
-
-### Build
-
-* `golang` 1.10 or higher (required, available in some distros and distributed
-as a binary officially by [golang.org](https://golang.org/dl/))
-
-### Runtime
-
-A relatively modern GNU/Linux system should be able to run `mgmt` without any
-problems. Since `mgmt` runs as a single statically compiled binary, all of the
-library dependencies are included. It is expected, that certain advanced
-resources require host specific facilities to work. These requirements are
-listed below:
-
-| Resource | Dependency        | Version                     | Check version with                                        |
-|----------|-------------------|-----------------------------|-----------------------------------------------------------|
-| augeas   | augeas-devel      | `augeas 1.6` or greater     | `dnf info augeas-devel` or `apt-cache show libaugeas-dev` |
-| file     | inotify           | `Linux 2.6.27` or greater   | `uname -a`                                                |
-| hostname | systemd-hostnamed | `systemd 25` or greater     | `systemctl --version`                                     |
-| nspawn   | systemd-nspawn    | `systemd ???` or greater    | `systemctl --version`                                     |
-| pkg      | packagekitd       | `packagekit 1.x` or greater | `pkcon --version`                                         |
-| svc      | systemd           | `systemd ???` or greater    | `systemctl --version`                                     |
-| virt     | libvirt-devel     | `libvirt 1.2.0` or greater  | `dnf info libvirt-devel` or `apt-cache show libvirt-dev`  |
-| virt     | libvirtd          | `libvirt 1.2.0` or greater  | `libvirtd --version`                                      |
-
-For building a visual representation of the graph, `graphviz` is required.
-
-To build `mgmt` without augeas support please run:
-`GOTAGS='noaugeas' make build`
-
-To build `mgmt` without libvirt support please run:
-`GOTAGS='novirt' make build`
-
-To build `mgmt` without docker support please run:
-`GOTAGS='nodocker' make build`
-
-To build `mgmt` without augeas, libvirt or docker support please run:
-`GOTAGS='noaugeas novirt nodocker' make build`
-
-## Binary Package Installation
-
-Installation of `mgmt` from distribution packages currently needs improvement.
-They are not always up-to-date with git master and as such are not recommended.
-At the moment we have:
-* [COPR](https://copr.fedoraproject.org/coprs/purpleidea/mgmt/)
-* [Arch](https://aur.archlinux.org/packages/mgmt/)
-
-Please contribute more! We'd especially like to see a Debian package!
-
-## OSX/macOS/Darwin development
-
-Developing and running `mgmt` on macOS is currently not supported (but not
-discouraged either). Meaning it might work but in the case it doesn't you would
-have to provide your own patches to fix problems (the project maintainer and
-community are glad to assist where needed).
-
-There are currently some issues that make `mgmt` less suitable to run for provisioning
-macOS. But as a client to provision remote servers it should run fine.
-
-Since the primary supported systems are Linux and these are the environments
-tested for it is wise to run these suites during macOS development as well. To
-ease this Docker can be leveraged ([Docker for Mac](https://docs.docker.com/docker-for-mac/)).
-
-Before running any of the commands below create the development Docker image:
-
-```
-docker/scripts/build-development
-```
-
-This image requires updating every time dependencies (`make-deps.sh`) change.
-
-Then to run the test suite:
-
-```
-docker run --rm -ti \
-	-v $PWD:/go/src/github.com/purpleidea/mgmt/ \
-	-w /go/src/github.com/purpleidea/mgmt/ \
-	purpleidea/mgmt:development \
-	make test
-```
-
-For convenience this command is wrapped in `docker/scripts/exec-development`.
-
-Basically any command can be executed this way. Because the repository source is
-mounted into the Docker container invocation will be quick and allow rapid
-testing, example:
-
-```
-docker/scripts/exec-development test/test-shell.sh load0.sh
-```
-
-Other examples:
-
-```
-docker/scripts/exec-development make build
-docker/scripts/exec-development ./mgmt run --tmp-prefix lang --lang examples/lang/load0.mcl
-```

docs/release-notes/0.0.10

@@ -0,0 +1,51 @@
+I've just released version 0.0.10 of mgmt!
+
+NEWS
+
+57 files changed, 1991 insertions(+), 752 deletions(-)
+
+* There's a new resource called `KV`. Short examples exist, but I
+haven't yet published a whole integration showing the usefulness.
+
+* A major race was fixed. The issue of what to do with BackPokes during
+start/pause was never previously solved. I had this as an open issue on
+my whiteboard for a while, and I finally got some time to work through
+it. The answer wasn't that difficult, but I think it was shrouded in
+some tunnel vision. See the commit messages and source comments for the
+details.
+
+* The GAPI grew four new associated World API methods: StrWatch,
+StrGet, StrSet, StrDel, and the associated etcd backed implementations.
+These are quite useful when combined with the KV resource.
+
+* There are now P/V style counting semaphores available as metaparams.
+This is particularly cool because the implementation is (AFAIK,
+assuming no bugs) dead-lock free! This is mentioned in my recent blog
+post.
+
+* See the git log for more NEWS, and sorry for anything notable I left
+out!
+
+BUGS
+
+* There's a `concurrent map write` bug in the semaphore implementation
+which is fixed in git master. Since it was a race, it was only caught
+after this release was made. I should also figure out if the sema check
+should go after the BackPoke or not.
+
+MISC
+
+We're still looking for new contributors, and there are easy, medium
+and hard issues available! You're also welcome to suggest your own!
+Please join us in #mgmtconfig on Freenode IRC, or ping this list if
+you'd like help getting started!
+
+THANKS
+
+Thanks (alphabetically) to everyone who contributed to the latest
+release:
+James Shubin, Julien Pivotto, Michael Borden.
+We had 3 unique committers since 0.0.9, and have had 30 overall.
+
+Happy hacking,
+James

docs/release-notes/0.0.11

@@ -0,0 +1,59 @@
+I've just released version 0.0.11 of mgmt!
+
+NEWS
+
+ 20 files changed, 579 insertions(+), 126 deletions(-)
+
+* Added a missing mutex around the semaphore map which prevents
+occasional panics
+
+* Removed exec pollint param which is not needed because of the poll
+metaparam
+
+* Fixed a state rechecking bug in exec resource (things are faster now)
+
+* Fixed the major annoyance of exec resources receiving main's signals.
+If we would ^C the main mgmt, the child processes would receive this
+too which was incorrect.
+
+* Fixed the deadlock on resource errors. This meant that previously if
+a resource failed, we would deadlock the graph from shutting down. This
+was bad and I'm glad it's now fixed. Sorry about that!
+
+* Improved the backpoke logic to not require semaphores since we used
+to take the lock even when we were going to backpoke which was
+unnecessary.
+
+* Added fast pausing to the graph. This means that a ^C or a pause
+transition used to wait for the whole graph to roll through, but it now
+finishes after the currently running resources finish executing. Read
+the commit messages for more background here including the discussion
+about a possible Interrupt() addition to the resource API.
+
+* Prometheus support has been updated!
+
+* See the git log for more NEWS, and sorry for anything notable I left
+out!
+
+BUGS
+
+* We're in pretty good shape now. There are some small issues, but
+nothing major that I don't know about.
+
+MISC
+
+We're still looking for new contributors, and there are easy, medium
+and hard issues available! You're also welcome to suggest your own!
+Please join us in #mgmtconfig on Freenode IRC, or ping this list if
+you'd like help getting started!
+
+THANKS
+
+Thanks (alphabetically) to everyone who contributed to the latest
+release:
+James Shubin, Julien Pivotto
+We had 2 unique committers since 0.0.10, and have had 30 overall.
+
+Happy hacking,
+James
+@purpleidea

docs/release-notes/0.0.12

@@ -0,0 +1,87 @@
+I've just released version 0.0.12 of mgmt!
+
+Sorry if I've been more quiet than usual, I've had to focus a lot of my
+time on GlusterFS related features ($dayjob official directives) and
+work there.
+
+One goal is to try and use libmgmt to either wholly or partially re-
+implement glusterd. As a result, a lot of my upstream focus has been
+re-prioritized to features needed for that effort.
+
+I wrote a PoC called gd3: https://github.com/purpleidea/gd3
+(It has now bit-rotted compared to upstream mgmt, but is easy to fix.)
+The initial scope looks like it will be much smaller, but hopefully
+this is interesting to you too. Ping me if you'd like to help.
+
+We desperately need your contributions if we're going to get mgmt
+standalone into a MVP. To motivate you, there's some great new stuff
+that landed since 0.0.11, including:
+
+* a great new YAML parser from contributor Mildred
+
+* a huge pgraph refactor (to remove internal deps and cycles)
+
+* a great amount of new tests and testing
+
+and so much more...
+
+NEWS
+
+ 76 files changed, 7549 insertions(+), 4269 deletions(-)
+
+* The svc resource now also supports user services
+
+* There's a fabulous new yaml parser that fixes a longstanding issue in
+my original code. Try it with --yaml2. I'll remove the old one
+eventually. Thanks to contributor Mildred for this great work!
+
+* Refactored the lib/ etcd usage into the GAPI's for a cleaner main
+
+* World API grew some new methods for libmgmt users
+
+* pgraph refactor and cleanup, now it's a fairly clean standalone pkg
+
+* pgraph functions to flatten/merge in subgraphs along with examples
+
+* Giant resource refactor to hopefully make things more logical, and to
+simplify the resource API. This also introduces the mgraph struct to
+add the higher level graph knowledge outside of pgraph.
+
+* A partial implementation of a "Graph" (recursive subgraph?) resource!
+See the code for details, as help is wanted to finish this. This should
+help elucidate what the most elegant design for the mgmt core should
+be.
+
+* Send/Recv support for the exec resource as output, stdout, and stderr
+
+* GAPI improvements to support exit requests and fast pausing
+
+* AutoEdge API improvements including a fix+test for a regression
+
+* A possible fix for the possible etcd server startup race
+
+* A fun amount of new tests all over including for gometalinter
+
+* See the git log for more NEWS, and for anything notable I left out!
+
+BUGS
+
+* We fixed a bunch of stuff, and added more tests!
+
+MISC
+
+We're still looking for new contributors, and there are easy, medium
+and hard issues available! You're also welcome to suggest your own!
+Please join us in #mgmtconfig on Freenode IRC, or ping this list if
+you'd like help getting started!
+
+THANKS
+
+Thanks (alphabetically) to everyone who contributed to the latest
+release:
+James Shubin, Julien Pivotto, Mildred Ki'Lya
+We had 3 unique committers since 0.0.11, and have had 30 overall.
+
+Happy hacking,
+James
+@purpleidea

docs/release-notes/0.0.13

@@ -0,0 +1,89 @@
+I've just released version 0.0.13 of mgmt!
+
+I guess this is an appropriate number for a scary October release.
+
+We recently re-licensed to the more permissive GPL. I put a lot of
+thought into it, and also wrote up a short blog post about some of the
+reasoning. It's here:
+
+https://ttboj.wordpress.com/2017/10/17/copyleft-is-dead-long-live-copyl
+eft/
+
+If you read to the end you'll also see that Red Hat is not funding my
+mgmt work anymore. This is too bad, and means I'll only have small
+amounts of personal time available to work on this. If you'd like to
+help fund my work, or know someone who'd like to, please let me know!
+
+Having said that, there's some great new stuff that landed since
+0.0.12, including:
+
+* new resources from new contributor Jonathan Gold (aws, group, user)
+
+* an HCL frontend from new contributor Chris McKenzie
+
+* polish in a number of places including in the nspawn resource
+
+and so much more... If you'd prefer to have releases more often, then
+please let me know! Lastly, the language and a number of associated
+parts are on its way. I hope to push this monster patch to git master
+before February.
+
+It's also worth mentioning that we have 17 resources now! wow.
+
+NEWS
+
+ 140 files changed, 3921 insertions(+), 848 deletions(-)
+
+* Many improvements to tests, testing and small fixes to avoid false-
+failures on travis.
+
+* golint now reports 0 issues.
+
+* An HCL frontend if you'd prefer that to the YAML. Also a great
+example of how to plug in a new frontend.
+
+* An update to golang 1.8 as the minimum version required.
+
+* Bump the etcd version to 3.2.6+ -- Looking forward to a 3.3 release
+which should probably include some patches we upstreamed.
+
+* Addition of new user and group resources. These also include a bunch
+of automatic edges.
+
+* Addition of an AWS resource! I've wanted this for a while, as it
+demonstrates nicely how event based cloud resources can fit nicely into
+our design. There's still a lot to do, and we have some suggestions for
+Amazon too. If you have a contact there, please put me in touch!
+
+* Our nspawn resource is more polished now.
+
+* See the git log for more NEWS, and for anything notable I left out!
+
+MISC
+
+We're still looking for new contributors, and there are easy, medium
+and hard issues available! You're also welcome to suggest your own!
+Please join us in #mgmtconfig on Freenode IRC, or ping this list if
+you'd like help getting started!
+
+MENTORING
+
+We offer mentoring for new golang/mgmt hackers who want to get
+involved. This is free and friendly. You get to improve your skills,
+and we get some patches in return. Ping me off-list for details.
+
+THANKS
+
+Thanks (alphabetically) to everyone who contributed to the latest
+release:
+AdnanLFC, Arthur Mello, ChrisMcKenzie, Dennis Kliban, Ismael Puerto,
+James Shubin, Jonathan Gold, Juan Luis de Sousa-Valadas Castaño, Juan-
+Luis de Sousa-Valadas Castaño (although I suspect the last two are the
+same ;))
+
+We had 9 unique committers since 0.0.12, and have had 38 overall.
+Run 'git log 0.0.12..0.0.13' to see what has changed since 0.0.12
+
+Happy hacking,
+James
+@purpleidea

docs/release-notes/0.0.14

@@ -0,0 +1,81 @@
+I've just released version 0.0.14 of mgmt!
+
+> 118 files changed, 2688 insertions(+), 974 deletions(-)
+
+There's some great new stuff that landed since
+0.0.13, including:
+
+* amazon AWS EC2 resource is now in git master.
+
+* more automatic edges from new contributor Guillaume Herail (xiu)
+
+* a move to golang 1.8 or higher
+
+and so much more... This will probably be the last release before the
+language lands in git master. It's a pretty giant patch coming :/
+
+NEWS
+
+* We're > 1k stars on GitHub now. It's a silly metric, but ¯\_(ツ)_/¯
+
+* Jonathan Gold has done a lot of hard work on the AWS EC2 resource,
+and it's now in git master. There are still many things we'd like to
+do, but it's a great start on what is a *monster* resource, and
+hopefully it will inspire others to get involved.
+
+In particular, it was a great learning experience (for me in
+particular!) about how bad the EC2 golang API is. There are some
+notable design bugs we found, and if anyone from Amazon engineering
+would like to reach out to us, we'd love to provide some constructive
+ideas.
+
+* Guillaume Herail (xiu) wrote some really nice patches, and picked up
+on the autoedges API very quickly. Hopefully he'll have time to work on
+even more!
+
+* Paul Morgan sent us some nice shell fixups-- many more exist in an
+open PR that didn't make it into this release. Hopefully we'll get
+those merged by 0.0.15
+
+* Felix Frank did a few Puppet compiler fix ups. I think he's been
+refreshing his work with new resources recently...
+
+* We're looking for help writing Google, DigitalOcean, Rackspace, etc
+resources if anyone is interested, reach out to us. Particularly if
+there is support from those organizations as well.
+
+* See the git log for more NEWS, and for anything notable I left out!
+
+EVENTS
+
+There are a bunch of upcoming mgmt talks and events! Stay tuned for
+details in the coming email, but TL;DR: Linux Conf Australia, FOSDEM, &
+CfgMgmtCamp.eu -- from three different speakers, and including a
+hackathon too!
+
+MISC
+
+We're still looking for new contributors, and there are easy, medium
+and hard issues available! You're also welcome to suggest your own!
+Please join us in #mgmtconfig on Freenode IRC, or ping this list if
+you'd like help getting started! Expect many new tagged #mgmtlove
+issues within the month.
+
+MENTORING
+
+We offer mentoring for new golang/mgmt hackers who want to get
+involved. This is free and friendly. You get to improve your skills,
+and we get some patches in return. Ping me off-list for details.
+
+THANKS
+
+Thanks (alphabetically) to everyone who contributed to the latest
+release:
+Felix Frank, Guillaume Herail, James Shubin, Jonathan Gold,
+jonathangold, Julien Pivotto, Paul Morgan, Toshaan Bharvani
+We had 8 unique committers since 0.0.13, and have had 41 overall.
+run 'git log 0.0.13..0.0.14' to see what has changed since 0.0.13
+
+Happy hacking,
+James
+@purpleidea

docs/release-notes/0.0.15

@@ -0,0 +1,116 @@
+I've just released version 0.0.15 of mgmt!
+
+> 328 files changed, 29869 insertions(+), 943 deletions(-)
+
+(Yeah, that's almost 30k+ LOC)
+
+There's some great new stuff that landed since 0.0.14, including:
+
+* THE LANGUAGE (mcl)
+
+* "Deploys": a distributed way to push code into your cluster
+
+* Scheduling (as a reactive function)
+
+* Better testing
+
+* a move to etcd 3.3+ and golang 1.9+
+
+and so much more... This is a monster release. Please try out the
+language and all the other new features today :)
+
+NEWS
+
+* We released the language. Please play around with it :) It's time to
+get used to this cool new paradigm. Learn more from the...
+
+Blog post:
+https://purpleidea.com/blog/2018/02/05/mgmt-configuration-language/
+
+Video:
+https://www.youtube.com/watch?v=NxObmwZDyrI
+
+Docs:
+https://github.com/purpleidea/mgmt/blob/master/docs/language-guide.md
+
+Function guide:
+https://github.com/purpleidea/mgmt/blob/master/docs/function-guide.md
+
+And tons of code all over git master. Check the lang/ folder.
+
+* There is a reactive scheduler in the language. Use your imagination,
+or play around with:
+https://github.com/purpleidea/mgmt/blob/3ad7097c8aa7eab7f895aab9af22338
+c0cf82986/lang/funcs/core/schedule_polyfunc.go#L18
+
+* There is a "deploys" feature. It's not documented yet. You should
+poke around if you're curious. Consider this an early soft release.
+
+* There is a FS implementation to store files in a POSIX-like layer on
+top of etcd. It's used by deploys. It needs more tests though :)
+
+* The language grew two "simple" API's for implementing functions, so
+that new functionality can be exposed in the mgmt language.
+
+* The language grew two ways to specify edges between resources: either
+internal to the resource, or externally as standalone edges.
+
+* The language now supports optional resource parameters via the
+"elvis" operator. This keeps things type safe and avoids needing an
+"undef" or "nil" in the language. This operator also works for edge
+declarations.
+
+* New contributor Johan Bloemberg has been on fire sending patches!
+He has already made some great improvements to our Makefile for
+testing, and the addition of the env* functions in the language, with
+much more code pending in open PR's.
+
+* Initial debian packaging has been added. It now needs a maintainer to
+build, upload, and love it :)
+
+* We have an early emacs major mode for "mcl", our language.
+
+* Lots of new documentation has been added. Particularly for developers
+wanting to contribute to the project.
+
+* We're looking for help writing Google, DigitalOcean, Rackspace, etc
+resources if anyone is interested, reach out to us. Particularly if
+there is support from those organizations as well.
+
+* See the git log for more NEWS, and for anything notable I left out!
+
+MISC
+
+We're still looking for new contributors, and there are easy, medium
+and hard issues available! You're also welcome to suggest your own!
+Please join us in #mgmtconfig on Freenode IRC, or ping this list if
+you'd like help getting started! For details please see:
+
+https://github.com/purpleidea/mgmt/blob/master/docs/faq.md#how-do-i-con
+tribute-to-the-project-if-i-dont-know-golang
+
+Many new tagged #mgmtlove issues were tagged:
+https://github.com/purpleidea/mgmt/issues?q=is%3Aissue+is%3Aopen+label%
+3Amgmtlove
+
+MENTORING
+
+We offer mentoring for new golang/mgmt hackers who want to get
+involved.
+This is free and friendly. You get to improve your skills,
+and we get
+some patches in return. Ping me off-list for details.
+
+THANKS
+
+Thanks (alphabetically) to everyone who contributed to the latest
+release:
+Carsten Thiel, dsx, James Shubin, Joe Julian, Johan Bloemberg, Jonathan
+Gold, jonathangold, karimb, Oliver Frommel, Peter Oliver, Toshaan
+Bharvani, Wim
+We had 12 unique committers since 0.0.14, and have had 48 overall.
+run 'git log 0.0.14..0.0.15' to see what has changed since 0.0.14
+
+Happy hacking,
+James
+@purpleidea

docs/release-notes/0.0.16

@@ -0,0 +1,104 @@
+I've just released version 0.0.16 of mgmt!
+
+> 220 files changed, 14243 insertions(+), 9491 deletions(-)
+
+Woo...
+
+There's some great new stuff that landed since 0.0.15, including:
+
+* A giant engine re-write! (Makes resource writing more elegant too.)
+
+* New resources!!
+
+* New language features!!!
+
+and so much more... This is a monster release. Please try out the new
+features today :)
+
+NEWS
+
+* New resources include: net, mount and docker:container. Jonathan was
+responsible for all of these. Please take them for a spin! He's looking
+for a job too, and would probably be happy to get paid to work on mgmt.
+
+* We're > 1.5k stars on GitHub now. It's a silly metric, but ¯\_(ツ)_/¯
+
+* A giant engine refactoring/re-write was done. This cleaned up the
+code significantly, and made it more elegant to write resources.
+Unfortunately there is one small bug that I missed and that I haven't
+fixed yet. It rarely happens except during some of our tests during
+shutdown, which causes intermittent failures. It shouldn't block you
+playing with mgmt.
+
+* The language "class" and "include" statements have been added. These
+are important pieces for writing reusable modules which are coming
+soon. Try them out! (This comes with a bunch of tests too.)
+
+* We have an integration testing framework. It's pretty cool, it spins
+up a full mgmt cluster and runs stuff. Try it out or add some tests.
+
+* I had fun fixing a big bug: 06ee05026b0c743d19c7d62675f8ddeabdc8dd4f
+
+* I removed the remote execution functionality from core. I realized it
+could be re-written as a resource, and it was also in the way from some
+other cleanups that were more important. Half the new code is done,
+ping me if this is a priority for you or you want to help.
+
+* I also removed the HCL front-end, because mcl is usable enough to be
+more fun to play with, and I wanted to refactor some code. If someone
+really wants it back, let me know.
+
+* We have some release building scripts in git master, so you can now
+download pre-built (with fpm) RPM, DEB, or PACMAN packages! They're
+signed too. https://github.com/purpleidea/mgmt/releases/tag/0.0.16
+
+* We're looking for help writing Google, DigitalOcean, Rackspace, etc
+resources if anyone is interested, reach out to us. Particularly if
+there is support from those organizations as well.
+
+* Many other bug fixes, changes, etc...
+
+* See the git log for more NEWS, and for anything notable I left out!
+
+MISC
+
+I took a bit of a break recently to catch up on some life stuff, but I
+think I'm back on track. While git master hasn't been especially busy,
+there's an active feature branch at feat/import which contains some fun
+stuff, with a very WIP giant patch still sitting on my machine. I hope
+to finish it up soon and then do another release. That branch contains
+one of the last big features before I'll really be ready to run mgmt on
+my personal servers!
+
+We're still looking for new contributors, and there are easy, medium
+and hard issues available! You're also welcome to suggest your own!
+Please join us in #mgmtconfig on Freenode IRC, or ping this list if
+you'd like help getting started! For details please see:
+
+https://github.com/purpleidea/mgmt/blob/master/docs/faq.md#how-do-i-con
+tribute-to-the-project-if-i-dont-know-golang
+
+Many new tagged #mgmtlove issues were tagged:
+https://github.com/purpleidea/mgmt/issues?q=is%3Aissue+is%3Aopen+label%
+3Amgmtlove
+
+MENTORING
+
+We offer mentoring for new golang/mgmt hackers who want to get
+involved.
+This is free and friendly. You get to improve your skills,
+and we get
+some patches in return. Ping me off-list for details.
+
+THANKS
+
+Thanks (alphabetically) to everyone who contributed to the latest
+release:
+Alan Jenkins, James Shubin, jesus m. rodriguez, Jonathan Gold,
+jonathangold, Lauri Ojansivu, phaer
+We had 7 unique committers since 0.0.15, and have had 52 overall.
+run 'git log 0.0.15..0.0.16' to see what has changed since 0.0.15
+
+Happy hacking,
+James
+@purpleidea

docs/release-notes/0.0.17

@@ -0,0 +1,111 @@
+I've just released version 0.0.17 of mgmt!
+
+> 269 files changed, 13281 insertions(+), 1633 deletions(-)
+
+There's some very useful stuff that landed since 0.0.16, including:
+
+* Modules and import system now exists!
+* A lot of tests, fixes and a huge new test infra was added
+* Merging puppet with mcl code is now possible (madness!)
+* We made a small change to the cli UI
+* Bump to golang 1.10 (please update your environments)
+
+And much more...
+
+DOWNLOAD
+
+Prebuilt binaries are available here:
+https://github.com/purpleidea/mgmt/releases/tag/0.0.17
+
+NEWS
+
+* One of the biggest missing features was the lack of a module/import
+system. After some initial exploration on what turned out to be a dead-
+end, I found what I think is a very elegant approach, which is now in
+this release. Please try it out, there are docs available. I hope to
+write a blog post about it soon.
+
+(There's one additional "kind" of import that I'm considering, similar
+to a macro "#include", that I might add. To be determined. Let me know
+if you find anything missing as of today.)
+
+* Felix added the first version of his mcl+puppet frontend. This should
+allow you to more cleverly merge legacy puppet environments with mcl
+code. It's fantastic, take a look.
+
+* Jonathan added a systemd-timer resource. This is a great replacement
+for cron.
+
+* We changed the CLI ui to improve the determinism of the frontend
+selection. Basically we changed from: `mgmt run --lang code.mcl` to:
+`mgmt run lang --lang code.mcl`. Remember to put --tmp-prefix after
+`run` directly where it is used.
+
+* We made a whole bunch of cleanups to the test infra, added new test
+infra for testing complex mcl modules and the import/module system, and
+of course added new tests.
+
+* You can pass a list of strings as the resource name to build that
+many resources. (Looping/iteration!)
+
+* You can specify all the metaparams and auto-* properties in mcl now.
+
+* Native mcl code can be used to write imported core code.
+
+* There was a bug that snuck into the pkg res. This has now been fixed.
+
+* A small, long-time copy+pasta error bug was fixed in Exec.
+
+* Virtually all the imports of the "log" package are at the top-level
+now. This will make moving to a new logger easier in the future. I have
+an innovative logger idea that I have a design for that I'll eventually
+get to.
+
+* A few workarounds for occasional test failures were added. Some
+legacy code needs a cleanup, and it's not done yet. Fortunately, none
+of these issues seem to occur in real-life as far as I can tell, and
+are caused by closing down mgmt at weird times.
+
+* Found a bug (now fixed) in the upstream lexer Yikes! See:
+57ce3fa587897d74634c1216af67dd42252c64e5
+
+* We're looking for help writing Amazon, Google, DigitalOcean, etc
+resources if anyone is interested, reach out to us. Particularly if
+there is support from those organizations as well.
+
+* Many other bug fixes, changes, etc...
+
+* See the git log for more NEWS, and for anything notable I left out!
+
+MISC
+
+We're still looking for new contributors, and there are easy, medium
+and hard issues available! You're also welcome to suggest your own!
+Please join us in #mgmtconfig on Freenode IRC, or ping this list if
+you'd like help getting started! For details please see:
+
+https://github.com/purpleidea/mgmt/blob/master/docs/faq.md#how-do-i-con
+tribute-to-the-project-if-i-dont-know-golang
+
+Many new tagged #mgmtlove issues were tagged:
+https://github.com/purpleidea/mgmt/issues?q=is%3Aissue+is%3Aopen+label%
+3Amgmtlove
+
+MENTORING
+
+We offer mentoring for new golang/mgmt hackers who want to get
+involved. This is fun and friendly! You get to improve your skills,
+and we get some patches in return. Ping me off-list for details.
+
+THANKS
+
+Thanks (alphabetically) to everyone who contributed to the latest
+release:
+Felix Frank, James Shubin, Jonathan Gold, Kevin Kuehler, Michael Lesko-
+Krleza, Tom Payne, Vincent Membré
+We had 7 unique committers since 0.0.16, and have had 56 overall.
+run 'git log 0.0.16..0.0.17' to see what has changed since 0.0.16
+
+Happy hacking,
+James
+@purpleidea

docs/release-notes/0.0.18

@@ -0,0 +1,106 @@
+I've just released version 0.0.18 of mgmt!
+
+> 202 files changed, 5606 insertions(+), 1880 deletions(-)
+
+There's some great stuff that landed since 0.0.17, including:
+
+* A re-write of the core engine algorithm
+* Tests are very stable
+* At least three old bugs were killed
+* An infra to write tests for individual resources was added
+
+And much more...
+
+This adds a significant amount of polish and bug fixes to mgmt. We're
+on the home stretch to MVP!
+
+DOWNLOAD
+
+Prebuilt binaries are available here:
+https://github.com/purpleidea/mgmt/releases/tag/0.0.18
+
+NEWS
+
+* There was a rare race that would panic the engine on shutdown. This
+only seemed to happen in CPU/system call starved environments like
+travis-ci. This was due to some ignorance when writing this early part
+of the code base. The algorithm has been re-written, and this also
+removed the use of at least one mutex. Things should be stable now, and
+also much faster. (Although perf was never an issue.)
+
+* A hidden race/deadlock in the exec resource was found and killed.
+Woo! Some new tools to help find these and other problems are in misc/
+
+* The early converger code was re-written. I was not as knowledgeable
+about golang in the very beginning, and this code needed refreshing. It
+contained a rare deadlock which needed to be killed.
+
+* Toshaan added an uptime() function.
+
+* Julien added a method for generating some simple functions for the
+language.
+
+* Lander added two new functions.
+
+* James added a new readfile() function, and other examples.
+
+* The template function now allows you to use imported functions. They
+use underscores instead of periods for namespace separation due to a
+limitation in the template library.
+
+* Kevin and I killed a tricky race in the SocketSet code! Woo :) Kevin
+also added a great cpucount() fact!
+
+* James gave a number of presentations at FOSDEM. Some recordings are
+available: https://purpleidea.com/talks/
+
+* We're looking for help writing Amazon, Google, DigitalOcean, etc
+resources if anyone is interested, reach out to us. Particularly if
+there is support from those organizations as well.
+
+* Many other bug fixes, changes, etc...
+
+* See the git log for more NEWS, and for anything notable I left out!
+
+BUGS
+
+* An unfortunate bug in the type unification code was found. This can
+cause small code bases to take a lot of ram/cpu to run. This will be
+prioritized in an upcoming release. Until then you'll have to avoid
+fancy type unification. (Specify types you know when it has to guess.)
+
+If efficient type unification algorithms are your specialty, please let
+us know, we'd like your help!
+
+MISC
+
+We're still looking for new contributors, and there are easy, medium
+and hard issues available! You're also welcome to suggest your own!
+Please join us in #mgmtconfig on Freenode IRC, or ping this list if
+you'd like help getting started! For details please see:
+
+https://github.com/purpleidea/mgmt/blob/master/docs/faq.md#how-do-i-con
+tribute-to-the-project-if-i-dont-know-golang
+
+Many tagged #mgmtlove issues exist:
+https://github.com/purpleidea/mgmt/issues?q=is%3Aissue+is%3Aopen+label%
+3Amgmtlove
+
+MENTORING
+
+We offer mentoring for new golang/mgmt hackers who want to get
+involved. This is fun and friendly! You get to improve your skills,
+and we get some patches in return. Ping me off-list for details.
+
+THANKS
+
+Thanks (alphabetically) to everyone who contributed to the latest
+release:
+Felix Frank, James Shubin, Jeff Waugh, Johan Bloemberg, Julien Pivotto,
+Kevin Kuehler, Lander Van den Bulcke, Toshaan Bharvani
+We had 8 unique committers since 0.0.17, and have had 58 overall.
+run 'git log 0.0.17..0.0.18' to see what has changed since 0.0.17
+
+Happy hacking,
+James
+@purpleidea

docs/release-notes/0.0.19

@@ -0,0 +1,131 @@
+I've just released version 0.0.19 of mgmt!
+
+> 361 files changed, 10451 insertions(+), 3919 deletions(-)
+
+This is a very important (and huge) release and has some important
+fixes that landed since 0.0.18, including:
+
+* A huge re-write of the elastic etcd clustering code base
+* A significant improvement in the type unification algorithm
+* An important import/class scoping bug was discovered and fixed
+* New mcl functions and resource improvements
+
+And much more...
+
+What comes next is just polish, new features and small bug fixes!
+
+DOWNLOAD
+
+Prebuilt binaries are available here:
+https://github.com/purpleidea/mgmt/releases/tag/0.0.19
+
+NEWS
+
+* A giant etcd re-write was completed and merged. The elastic
+clustering algorithm is not perfect, however it should suffice for most
+use cases, and it's always possible to point mgmt at an external etcd
+cluster if you don't understand the limitations of the automatic
+clustering algorithm. The important part is that the core code is much
+cleaner now, so hopefully races and bugs of ignorance are gone now. :)
+
+* I found an unfortunate bug in the type unification algorithm that
+severely impacted performance for some types of code bases. This is now
+fixed, and I hope we should not experience problems again! Special
+thanks to Sam for talking me through the problem and understanding the
+space better! Woo \o/
+
+* An important import/class scoping bug was fixed. Thanks to Nicolas
+for the bug report. We also added tests for this too!
+
+* Nicolas also added our first os detection function. os.is_debian and
+os.is_redhat are now in core. Get your favourite os added today!
+
+* The polymorphic len function can also check str length.
+
+* The exec resource got a big cleanup. It also learned the interrupt
+trait so that long running commands can be forcefully killed if need
+be.
+
+* A fancy new test infra for testing functions over time was added.
+Anytime we want to check our individual FRP functions are working as
+expected, this is an easy way to add a test. This way, if we ever find
+a bug, we can drop in a test with the fix. This actually helped find a
+very subtle bug in readfile that nobody had experienced yet!
+
+* File res with state => exists but no content now performs as
+expected.
+
+* Improved send/recv, since it was neglected a bit. Hopefully it ends
+up being a useful primitive.
+
+* Added a new synchronization primitive that I'm called
+SubscribedSignal. I found it very useful for building some of my code,
+and I hope you'll find it useful too. I'd offer it upstream to the sync
+package if Google didn't force their crappy CLA nonsense on everyone
+who wanted to send a patch. :/ Death by 1000 paper cuts, I guess.
+
+* Added a match function in the new regexp core package. Try it out and
+add some more functions!
+
+* Wouter has been testing mgmt and filling all sorts of useful bug
+reports. We fixed at least one related to a report, and more are
+planned for the next release. Wouter also sent in one cleanup patch to
+remove some dead code. Welcome to the project!
+
+* We're looking for help writing Amazon, Google, DigitalOcean, etc
+resources if anyone is interested, reach out to us. Particularly if
+there is support from those organizations as well.
+
+* Many other bug fixes, changes, etc...
+
+* See the git log for more NEWS, and for anything notable I left out!
+
+BUGS
+
+* There are a few known issues, in particular with some over eager
+checking done in the Validate portion of two resources, that should
+actually be runtime checks in CheckApply. As a result, if you intend to
+change some state during the graph execution, the resource won't see
+it. These should be very easy to fix if someone is interested in
+writing the patch!
+
+TALKS
+
+* James will be presenting at this year's OSDC in Berlin. There will be
+a workshop: https://osdc.de/events/mgmt-config-workshop/
+and a talk:
+https://osdc.de/events/mgmt-config-the-future-of-your-autonomous-datacentre/
+Sign up soon if you want to guarantee a spot, as they're limited!
+
+MISC
+
+We're still looking for new contributors, and there are easy, medium
+and hard issues available! You're also welcome to suggest your own!
+Please join us in #mgmtconfig on Freenode IRC, or ping this list if
+you'd like help getting started! For details please see:
+
+https://github.com/purpleidea/mgmt/blob/master/docs/faq.md#how-do-i-con
+tribute-to-the-project-if-i-dont-know-golang
+
+Many tagged #mgmtlove issues exist:
+https://github.com/purpleidea/mgmt/issues?q=is%3Aissue+is%3Aopen+label%
+3Amgmtlove
+
+MENTORING
+
+We offer mentoring for new golang/mgmt hackers who want to get
+involved. This is fun and friendly! You get to improve your skills,
+and we get some patches in return. Ping me off-list for details.
+
+THANKS
+
+Thanks (alphabetically) to everyone who contributed to the latest
+release:
+Adam Sigal, Felix Frank, James Shubin, Jonathan Gold, Michael Schubert,
+Mitch Fossen, Nicolas Charles, Wouter Dullaert
+We had 8 unique committers since 0.0.18, and have had 63 overall.
+run 'git log 0.0.18..0.0.19' to see what has changed since 0.0.18
+
+Happy hacking,
+James
+@purpleidea

docs/release-notes/0.0.20

@@ -0,0 +1,145 @@
+I've just released version 0.0.20 of mgmt!
+
+> 295 files changed, 8585 insertions(+), 1413 deletions(-)
+
+This was a very challenging release but it includes many useful changes
+since 0.0.19, including:
+
+* Function values / lambdas exist and are first-class
+* Over 70 new tests have been added!
+* Core functions, classes, and globals can now be written in mcl code
+* A new compiler "Ordering" step was added
+
+And much more...
+
+DOWNLOAD
+
+Prebuilt binaries are available here:
+https://github.com/purpleidea/mgmt/releases/tag/0.0.20
+
+NEWS
+
+* After a long and challenging road, I finally got function values and
+lambdas merged. If you find any bugs, please let me know! You can now
+treat functions as first-class values, and even use them as closures by
+capturing variable state.
+
+* Over 70 new tests were added, most of which test the behaviour of the
+new functions.
+
+* Core packages always allowed you to write new functions in pure
+golang, but now you can implement new functions, classes, and even
+variables in native mcl code! It's still desirable and perhaps
+efficient in some cases to want native golang implementations, but all
+good programs self-host some of their stdlib in their own language
+eventually.
+
+* A new compiler step called "Ordering" was added. It's hidden inside
+the SetScope step, but now lets us generate code ordering graphs and
+determine exactly what to run first.
+
+* Light copying of Node's allows more correct and efficient function
+graphs that can share common vertices and edges. For example, if two
+different closures capture a variable $x, they'll both use the same
+copy when running the function, since the compiler can prove if they're
+identical.
+
+* Improved the type system slightly to allow advanced type comparisons.
+
+* The type unification algorithm was improved. Hopefully it should
+solve all scenarios without needing the recursive solver which was very
+slow. If you find a case that isn't speedy, please let us know!
+
+* Added subtest listing by using -short and -v in a test. This lets you
+know what's available and how to run individual sub tests easily.
+
+* Support for the systemd STATE_DIRECTORY and xdg cache dir was added
+by new contributor John! Thanks!
+
+* New contributor Adam added a pgraph test.
+
+* A bug in the systemd mount resource was fixed. Hopefully it should
+work correctly now.
+
+* A bug that prevented us from allowing nested system imports was
+fixed. Nest away! This will pave the way for us to automatically import
+most of the golang standard library by doing: import "golang/regexp" or
+golang/whatever".
+
+* Added an example showing that unicode is allowed in strings.
+
+* Fixed a rare race in the engine.
+
+* Added some new core functions including math.mod and datetime
+improvements.
+
+* Changed the API to remove the use of --lang. This avoids the
+stuttering.
+
+* Moved to golang 1.11 and etcd 3.3.13. The later includes a fix for an
+un-catchable error scenario which we fixed in etcd.
+
+* Improved the pgraph library significantly so that we can generate
+better graphs with accurate vertices based on the vertex pointers.
+
+* Added ArchLinux OS family detection.
+
+* We're looking for help writing Amazon, Google, DigitalOcean, etc
+resources if anyone is interested, reach out to us. Particularly if
+there is support from those organizations as well.
+
+* Many other bug fixes, changes, etc...
+
+* See the git log for more NEWS, and for anything notable I left out!
+
+BUGS
+
+* There are a few known issues with some disabled code paths in the new
+function value code. These issues don't *need* to be fixed, but if they
+are, then we should see a slight performance increase. Happy to have
+someone dig into these, and they shouldn't bother anyone.
+
+* Some of the pre-built binaries might not work on your system. We need
+to start building them with the right dependencies so that `ldd`
+related things are happy. For now, please try building yourself if the
+build doesn't work for you, or help improve our build system.
+
+TALKS
+
+I'll be giving a talk at an upcoming mini-conference in Montreal. If
+you're interested in attending, please let me know.
+If you'd like to give an mgmt talk somewhere, please let me know!
+
+MISC
+
+We're still looking for new contributors, and there are easy, medium
+and hard issues available! You're also welcome to suggest your own!
+Please join us in #mgmtconfig on Freenode IRC, or ping this list if
+you'd like help getting started! For details please see:
+
+https://github.com/purpleidea/mgmt/blob/master/docs/faq.md#how-do-i-con
+tribute-to-the-project-if-i-dont-know-golang
+
+Many tagged #mgmtlove issues exist:
+https://github.com/purpleidea/mgmt/issues?q=is%3Aissue+is%3Aopen+label%
+3Amgmtlove
+
+MENTORING
+
+We offer mentoring for new golang/mgmt hackers who want to get
+involved. This is fun and friendly! You get to improve your skills,
+and we get some patches in return. Ping me off-list for details.
+
+THANKS
+
+Thanks (alphabetically) to everyone who contributed to the latest
+release:
+Adam Sigal, Christian Rebischke, Felix Frank, James Shubin, Jan
+Martens, Johan Bloemberg, John Hooks, Ward Vandewege
+We had 8 unique committers since 0.0.19, and have had 67 overall.
+run 'git log 0.0.19..0.0.20' to see what has changed since 0.0.19
+
+
+Happy hacking,
+James
+@purpleidea

docs/release-notes/0.0.21

@@ -0,0 +1,133 @@
+I've just released version 0.0.21 of mgmt!
+
+> 178 files changed, 4351 insertions(+), 829 deletions(-)
+
+This was a very lonely release but it includes some very useful
+additions since 0.0.20, including:
+
+* The first mgmt meme!
+* Working distro packages for Fedora, Debian, Ubuntu and Arch!
+* Reversible resources!
+* A deploy package to let you read files from the active deploy
+* Improved file resource behaviours
+
+And much more...
+
+DOWNLOAD
+
+Prebuilt binaries are available here:
+https://github.com/purpleidea/mgmt/releases/tag/0.0.21
+
+NEWS
+
+* Someone made a cool mgmt meme. Look in art/mgmt_*_meme.jpg
+
+* Distro packages are now built properly, so they should all work now.
+We have builds for Fedora-30, Fedora-29, Debian-10, Ubuntu-Bionic, and
+Archlinux. If you'd like a build for a different distro/version please
+let me know.
+
+* We finally got rid of the old `Compare(...) bool` API, and moved to
+`Cmp(...) error`. We'll now get more useful information from Res
+compares when they differ. Thanks to new contributor Donald Bakong for
+working on this. He's ramping up his golang contributions, so we expect
+to see more from him in the future!
+
+* We now have reversible resources. Basically if you create a resource
+and specify the reverse metaparam, eg: `Meta:reverse => true`, then if
+a resource is removed (either because a new version of code doesn't
+have it anymore OR more importantly if it was inside an `if` block that
+became false) then the engine will perform some "reverse" action for
+it. For a file, if it was added, we'll remove it. If it was edited,
+we'll undo the edit. If we added ugo+w, we'll remove that. And so on.
+The engine bits are done, and as well so have the file resource bits.
+It should be easy to add this for any other resource where it makes
+sense. This will likely be a very powerful feature that we use a lot.
+
+* The file resource was changed slightly so that by default the "state"
+is undefined. As a result, if you want a file to be created and none is
+present, you need to specify the state. Otherwise specifying "content"
+will only edit a file if it already exists, and otherwise be an error.
+It turns out this is actually a better behaviour, even if it's not
+necessarily intuitive for puppet users. It turns out it simplifies the
+code drastically and makes the reversible file resource much more
+logical. It seems that Puppet and Ansible got this wrong, but Cfengine
+got it right. IIRC. Do you agree? (Look at the code!)
+
+* We now catch CR \r characters in code so that you don't wonder why
+the compiler is telling you about unexpected whitespace. This should
+make your life easier.
+
+* You can now read files from within the deploy. This can be used for
+templates or anything else. This was one of the last missing things
+that was blocking me from writing useful mcl modules.
+
+* Fixed a copy-pasta bug where the != operator (for strings only) was
+actually doing an ==. Woops! The good news is that we've been shaking
+out silly bugs because I've been using mgmt more and more. Hopefully
+there aren't any woops ones like this left!
+
+* A bunch of function, class, and include tests were added. We're
+getting really well tested!
+
+* We're looking for help writing Amazon, Google, DigitalOcean, etc
+resources if anyone is interested, reach out to us. Particularly if
+there is support from those organizations as well.
+
+* Many other bug fixes, changes, etc...
+
+* See the git log for more NEWS, and for anything notable I left out!
+
+BUGS
+
+* Function values as args don't work yet. This is blocking us from
+implementing functions like map/reduce/filter, because they'd want to
+receive a function as input. TBH, I'm a bit tunnel-visioned on this,
+because I'm not the compilers genius that you are. If you can help,
+please let me know. I'll be posting a bunch of test cases that show
+what's needed shortly.
+
+* Three patches have been submitted to mkosi to support the image
+building I've been doing. They're not merged yet, so you'll have to
+apply them yourself if you want to make your own distro images. This
+isn't a major requirement anyone should have, but if they're not
+merged, we'll store them here and apply them as needed.
+
+TALKS
+
+I'll be in Belgium for FOSDEM and CfgMgmtCamp in 2020. If I'm lucky
+I'll have at least one mgmt presentation. I might also consider going
+to DevConf.cz if I get a talk accepted. Feel to ping me if you'd like
+to hack, get consulting, training, etc while I'm in Europe!
+
+MISC
+
+We're still looking for new contributors, and there are easy, medium
+and hard issues available! You're also welcome to suggest your own!
+Please join us in #mgmtconfig on Freenode IRC, or ping this list if
+you'd like help getting started! For details please see:
+
+https://github.com/purpleidea/mgmt/blob/master/docs/faq.md#how-do-i-con
+tribute-to-the-project-if-i-dont-know-golang
+
+Many tagged #mgmtlove issues exist:
+https://github.com/purpleidea/mgmt/issues?q=is%3Aissue+is%3Aopen+label%
+3Amgmtlove
+
+MENTORING
+
+We offer mentoring for new golang/mgmt hackers who want to get
+involved. This is fun and friendly! You get to improve your skills,
+and we get some patches in return. Ping me off-list for details.
+
+THANKS
+
+Thanks (alphabetically) to everyone who contributed to the latest
+release:
+bjanssens, Donald Bakong, James Shubin
+We had 3 unique committers since 0.0.20, and have had 69 overall.
+run 'git log 0.0.20..0.0.21' to see what has changed since 0.0.20
+
+Happy hacking,
+James
+@purpleidea

docs/release-notes/0.0.22

@@ -0,0 +1,160 @@
+I've just released version 0.0.22 of mgmt!
+
+> 579 files changed, 17984 insertions(+), 3136 deletions(-)
+
+Compared to the last release, this is a monster. Previously:
+
+> 178 files changed, 4351 insertions(+), 829 deletions(-)
+
+I apologize for not doing a release earlier, but to be quite honest,
+I've been busy, the people who are playing with mgmt are doing their
+own builds anyways, and there are still some unimplemented, pre-
+production features missing.
+
+Also, I started a new job. News on my blog and how (if at all) it
+affects mgmt is there.
+
+This is still an incredibly important release, so let's cover some of
+the points!
+
+* This is the last release before we switch to go.mod
+* IRC channel moved to #mgmtconfig on libera.chat after Freenode died
+* New resources including tftp, dhcp, and http (all as servers, wow!)
+* New string interpolation implementation with many tests
+* Resource fields can accept complex structs and other types now
+* Improved type unification solver and new invariants like generator
+* A new polymorphic function API interface
+
+And much more...
+
+DOWNLOAD
+
+Prebuilt binaries are NOT available here for this release:
+https://github.com/purpleidea/mgmt/releases/tag/0.0.22
+
+NEWS
+
+* The file resource supports building files from other "fragments".
+This is magic and automatic and real-time. Docs and examples are in the
+repo.
+
+* The file resource (and others) have a new trait and queryable API to
+make decisions based on what other resources are in the graph.
+
+* The file resource has a "purge" option to remove unmanaged files from
+a managed directory.
+
+* A lot of built-in functions are auto-generated from the stdlib. Most
+things you would want are now present, particularly if they're pure
+functions.
+
+* There's a new consul KV resource.
+
+* File resources support symbolic modes now!
+
+* New tftp related resources are now present. They're great and I use
+them to provision things!
+
+* A docker image resource was added.
+
+* First-class constants now exist. So you can do
+$const.res.file.state.exists instead of typing "exists" which is prone
+to typos. This is more verbose, but it's safer if that's your priority.
+
+* We found a bug with fuzzing! Cool, thanks Patrick!
+
+* We have dhcp server related resources. This is pretty cool when
+combined with the tftp resource and you can provision a lot of stuff
+from your laptop and one binary now.
+
+* We also have http server resources. Combined with the tftp and dhcp
+resources mgmt starts to look like a powerful tool to greenfield a new
+datacentre and then take over and manage it continuous. All from a
+single, type-safe, code base. Of course you can do other things with
+this, and I'm looking forward to seeing the ideas that I haven't
+thought of yet! PS: An http:ui has been partially implemented too. Ping
+if you want to know more.
+
+* Resource fields couldn't previously accept anonymous structs as types
+because of how golang built its reflect library. Joe found an elegant
+workaround, thanks!
+
+* The type unification solver was improved to support some new
+invariants. This makes a lot of new things possible, and was done to
+support new complex functions including the eventual addition of map,
+reduce, and filter. One of the new invariants is a "generator"
+invariant, so that unification can take into account the entire
+relevant parts of the AST. It's not a textbook CS implementation, but
+it's based on sound theory I think, and it seems to work great. If you
+find an edge case, please let us know.
+
+* The polymorphic function interface was changed to use the new
+unification logic. This makes a lot more sense. All the functions have
+been ported to the new interface.
+
+* 42
+
+* We're looking for help writing Amazon, Google, DigitalOcean, etc
+resources if anyone is interested, reach out to us. Particularly if
+there is support from those organizations as well.
+
+* Many other bug fixes, changes, etc...
+
+* See the git log for more NEWS, and for anything notable I left out!
+
+BUGS
+
+* Function values as args don't work yet. This is blocking us from
+implementing functions like map/reduce/filter, because they'd want to
+receive a function as input. TBH, I'm a bit tunnel-visioned on this,
+because I'm not the compilers genius that you are. If you can help,
+please let me know. I'll be posting a bunch of test cases that show
+what's needed shortly.
+
+(This is the exact message I wrote last time. I've made a lot of
+progress since then, but motivation here has been low. Reach out if you
+can help.)
+
+TALKS
+
+Hopefully CfgMgmtCamp in 2022 will be back on. If we're lucky and safe,
+maybe I can travel there. TBD... Feel to ping me if you'd like to hack,
+talk, whatever if I'm in Europe.
+
+MISC
+
+We're still looking for new contributors, and there are easy, medium
+and hard issues available! You're also welcome to suggest your own!
+Please join us in #mgmtconfig on Libera IRC, or ping this list if
+you'd like help getting started! For details please see:
+
+https://github.com/purpleidea/mgmt/blob/master/docs/faq.md#how-do-i-con
+tribute-to-the-project-if-i-dont-know-golang
+
+Many tagged #mgmtlove issues exist:
+https://github.com/purpleidea/mgmt/issues?q=is%3Aissue+is%3Aopen+label%
+3Amgmtlove
+
+Although asking in IRC is the best way to find something to work on.
+
+MENTORING
+
+We offer mentoring for new golang/mgmt hackers who want to get
+involved. This is fun and friendly! You get to improve your skills,
+and we get some patches in return. Ping me off-list for details.
+
+THANKS
+
+Thanks (alphabetically) to everyone who contributed to the latest
+release:
+Adam Sigal, Ahmed Al-Hulaibi, David Randall, Derek Buckley, Donald
+Bakong, Felix Frank, Francois Rompre-Lanctot, Ivan Pejić, James Shubin,
+Jean-Philippe Evrard, Jimmy Tang, Joe Groocock, Jonathan Gold, Julien
+Pivotto, Kenneth Hoste, Matthew Lesko-Krleza, Patrick Meyer, viq, Yohan
+Belval
+We had 19 unique committers since 0.0.21, and have had 82 overall.
+run 'git log 0.0.21..0.0.22' to see what has changed since 0.0.21
+
+Happy hacking,
+James
+@purpleidea

docs/release-notes/0.0.23

@@ -0,0 +1,102 @@
+I've just released version 0.0.23 of mgmt!
+
+>  424 files changed, 7051 insertions(+), 2256 deletions(-)
+
+This is a fairly quiet release, and I'm mostly doing this to have a
+permanent tag before I start really breaking git master.
+
+I'd like to apologize for things being kind of quiet. I've had to focus
+on life and making money to pay my bills, and I've been struggling a
+bit to complete some of the tougher algorithmic parts that I think are
+necessary for a solid MVP. Hopefully I will succeed, but to do so, it's
+going to be easier if I break git master and then sort things out
+later.
+
+I'm feeling optimistic about the future, although help with some
+complex concurrent programming would certainly be appreciated.
+
+With that, here are a few highlights from the release:
+
+* We're using go.mod, but I'm not keeping it up-to-date regularly yet.
+
+* We have an os.system(`cmd`) function!
+
+* We replaced go-bindata with the embed package.
+
+* We added a hetzner vm resource.
+
+* We did a giant lang/ package refactor.
+
+* Printf supports %v now (any type).
+
+And much more...
+
+DOWNLOAD
+
+Prebuilt binaries are NOT available here for this release:
+https://github.com/purpleidea/mgmt/releases/tag/0.0.23
+
+NEWS
+
+* We have our Libera IRC channel bridged to Matrix, so you can join
+that way too.
+
+* We're looking for help writing Amazon, Google, DigitalOcean, Hetzner,
+etc, resources if anyone is interested, reach out to us. Particularly
+if there is support from those organizations as well.
+
+* Many other bug fixes, changes, etc...
+
+* See the git log for more NEWS, and for anything notable I left out!
+
+BUGS
+
+* Function values as args don't work yet. This is blocking us from
+implementing functions like map/reduce/filter, because they'd want to
+receive a function as input.
+
+(This is the exact message I wrote last time. I've made a lot of
+progress since then, but motivation here has been low. Reach out if you
+can help.)
+
+TALKS
+
+I don't have anything planned until CfgMgmtCamp 2024. If you'd like to
+book me for a private event, or sponsor my travel for your conference,
+please let me know.
+
+MISC
+
+We're still looking for new contributors, and there are easy, medium
+and hard issues available! You're also welcome to suggest your own!
+Please join us in #mgmtconfig on Libera IRC or Matrix, or ping this
+list if you'd like help getting started! For details please see:
+
+https://github.com/purpleidea/mgmt/blob/master/docs/faq.md#how-do-i-con
+tribute-to-the-project-if-i-dont-know-golang
+
+Many tagged #mgmtlove issues exist:
+https://github.com/purpleidea/mgmt/issues?q=is%3Aissue+is%3Aopen+label%
+3Amgmtlove
+
+Although asking in IRC is the best way to find something to work on.
+
+MENTORING
+
+We offer mentoring for new golang/mgmt hackers who want to get
+involved. This is fun and friendly! You get to improve your skills,
+and we get some patches in return. Ping me off-list for details.
+
+THANKS
+
+Thanks (alphabetically) to everyone who contributed to the latest
+release:
+dantefromhell, James Shubin, Jef Masereel, Joe Groocock, Samuel
+Gélineau
+We had 5 unique committers since 0.0.22, and have had 85 overall.
+run 'git log 0.0.22..0.0.23' to see what has changed since 0.0.22
+
+
+Happy hacking,
+James
+@purpleidea

docs/release-notes/0.0.24

@@ -0,0 +1,161 @@
+I've just released version 0.0.24 of mgmt!
+
+
+> 600 files changed, 13622 insertions(+), 6907 deletions(-)
+
+This is a huge and hugely important release! It has been a long time
+coming. We have lambdas! I could not have done this without the
+unrelentingly supportive and brilliant Samuel Gélineau.
+
+Getting past the blockers and tricky code changes was all thanks to
+him. Cleanups, polish and making it more golang idiomatic was my doing.
+(The easy stuff.) If Sam wanted to be a golang expert, he could have
+done it all, but it was more sensible than I do all the mundane and
+filler stuff.
+
+With that, here are a few highlights from the release:
+
+* We have working lambdas, including iter.map =D
+
+* We have a new function engine!
+
+* We have improved type unification!
+
+* We have an improved resource engine!
+
+* We have improved many of the internal API's.
+
+* We have so many tests.
+
+And much more...
+
+DOWNLOAD
+
+Prebuilt binaries are NOT available here for this release:
+https://github.com/purpleidea/mgmt/releases/tag/0.0.24
+
+NEWS
+
+* Our main tests are now in the excellent .txtar format. This makes
+things much easier to manage.
+
+* The iter.map function can be named as such due to parser tricks! No
+need to name it xmap anymore!
+
+* The unification solver has been greatly improved. We can infer a lot
+more about function types.
+
+* The resources API uses the context package for closing Watch and
+returning early from CheckApply. The next step would be to remove all
+the resource-specific timeout code and make that a metaparam.
+
+* A new sync primitive has been added in case you'd like to use it
+somewhere. We'd love help adding an even more complex one. Look in
+util/sync.go for more information.
+
+* Sam added some beautiful type inference debugging that makes things
+easier for those familiar with the standard literature.
+
+* An important bug in standalone etcd has been fixed. While embedded
+etcd and automatic clustering isn't "supported" (it's buggy) the
+status-quo of using your own etcd cluster is stable, and you can even
+use the embedded etcd server in standalone mode...
+
+* This means you can run `mgmt etcd` and get the standard etcd binary
+behviour that you'd get from running `etcd` normally. This makes it
+easy to use both together since you only need to transport one binary
+around. (And maybe mgmt will do that for you!)
+
+* I fixed and cleaned up some sketchy code in the resource engine. I
+had been unmotivated to fix this for a while because I really wanted
+lambdas first, but now that they are in, I took a good look at the
+code, made some fixes, and I'm really happy with it now.
+
+* Metaparams are appropriately stateful between graph switches now.
+Retry is the easy example. Limit/Burst need to be ported if you care
+about these fine details.
+
+* A RetryReset metaparam has been added. This is another good example
+of how powerful metaparams are and how much potential there is for
+building systems with these and future ones.
+
+* A bunch of internal API's have been updated. This makes it better for
+function and resource writers! Some GAPI changes also got pushed
+through that make things clearer for those reading code.
+
+* We have a listlookup function. It's still missing syntactic sugar
+though!
+
+* Our new function graph engine is called "dage". I think it's pretty
+clever. There's a chance there is still a bug inside, but it's unclear.
+Please report any issues. If you have some large machines I can test
+very large and fast graphs on, please let me know.
+
+* Lambdas really work now! The txn and ref/gc code is pretty fantastic.
+
+* Many bugs have been killed!
+
+* We're looking for help writing Amazon, Google, DigitalOcean, Hetzner,
+etc, resources if anyone is interested, reach out to us. Particularly
+if there is support from those organizations as well.
+
+* Many other bug fixes, changes, etc...
+
+* See the git log for more NEWS, and for anything notable I left out!
+
+BUGS/TODO
+
+* Function values getting _passed_ to resources doesn't work yet, but
+it's not a blocker, but it would definitely be useful. We're looking
+into it.
+
+* Function graphs are unnecessarily dynamic. We might make them more
+static so that we don't need as many transactions. This is really a
+compiler optimization and not a bug, but it's something important we'd
+like to have.
+
+* Running two Txn's during the same pause would be really helpful. I'm
+not sure how much of a performance improvement we'd get from this, but
+it would sure be interesting to build. If you want to build a fancy
+synchronization primitive, then let us know! Again this is not a bug.
+
+TALKS
+
+I don't have anything planned until CfgMgmtCamp 2024. If you'd like to
+book me for a private event, or sponsor my travel for your conference,
+please let me know.
+
+MISC
+
+We're still looking for new contributors, and there are easy, medium
+and hard issues available! You're also welcome to suggest your own!
+Please join us in #mgmtconfig on Libera IRC or Matrix, or ping this
+list if you'd like help getting started! For details please see:
+
+https://github.com/purpleidea/mgmt/blob/master/docs/faq.md#how-do-i-con
+tribute-to-the-project-if-i-dont-know-golang
+
+Many tagged #mgmtlove issues exist:
+https://github.com/purpleidea/mgmt/issues?q=is%3Aissue+is%3Aopen+label%
+3Amgmtlove
+
+Although asking in IRC/matrix is the best way to find something to work
+on.
+
+MENTORING
+
+We offer mentoring for new golang/mgmt hackers who want to get
+involved. This is fun and friendly! You get to improve your skills,
+and we get some patches in return. Ping me off-list for details.
+
+THANKS
+
+Thanks (alphabetically) to everyone who contributed to the latest
+release:
+James Shubin, Kaushal, Laurent Indermuehle, Ofek Atar, Samuel Gélineau
+We had 5 unique committers since 0.0.23, and have had 88 overall.
+run 'git log 0.0.23..0.0.24' to see what has changed since 0.0.23
+
+Happy hacking,
+James
+@purpleidea

docs/release-notes/0.0.25

@@ -0,0 +1,343 @@
+I've just released version 0.0.25 of mgmt!
+
+> 686 files changed, 28391 insertions(+), 6935 deletions(-)
+
+This is the first release that I consider to be generally useful at
+solving real-world problems, without needing to be an mgmt expert. It's
+also the first release that includes a very real `mcl` codebase. An
+accompanying blog post is also available:
+https://purpleidea.com/blog/2024/03/27/a-new-provisioning-tool/
+
+With that, here are a few highlights from the release:
+
+* We have a new mgmt partner program. Please sign-up for early access
+to these release notes, along with other special privileges. Details
+at: https://bit.ly/mgmt-partner-program
+
+* You can build self-contained mgmt binaries that contain a custom
+application. An initial "provisioning tool" has been built in this way.
+Please see the blog post for more details.
+
+* Default lookup functions exist in the language, with syntactic sugar
+(the || operator) , so you can get a default value if one doesn't
+exist, eg: $some_struct->some_struct_field || "some_default_value".
+
+* Resource fields can now accept interface{} (any) types.
+
+* A panic feature now exists in the language.
+
+* The exec resource has new `donecmd` and `creates` fields. Of note,
+`creates` supports watches too!
+
+* Send/recv now works for autogrouped resources!
+
+* Added `include as` (for classes) to the language. Nested
+(sugar/hierarchical) classes are now supported to make this more
+powerful!
+
+* Stats are printed if the function engine is waiting for too long.
+
+* There's a new http:flag resource, and also an http:proxy resource so
+that we can have caching http proxies!
+
+* Added a firewalld resource for opening ports!
+
+* Added a dhcp:range resource which is very powerful and has a fun API!
+
+* Added the "embedded" and "entry" packages, for building standalone
+tools. This goes perfectly with the new CLI library that we ported
+everything to.
+
+And much more...
+
+
+DOWNLOAD
+
+Prebuilt binaries are available here for this release:
+https://github.com/purpleidea/mgmt/releases/tag/0.0.25
+
+They can also be found on the Fedora mirror:
+https://dl.fedoraproject.org/pub/alt/purpleidea/mgmt/releases/0.0.25/
+
+
+NEWS
+
+* We changed the logical operators in mcl to use well-known English
+tokens: OR, AND, NOT. (but in lowercase of course)
+
+* The history function has been temporarily removed from the syntactic
+core. We'll add it back if we find it's useful to have sugar!
+
+* A bunch of lexer/parser cleanups and improvements were made.
+
+* Default lookup functions for lists, maps, and structs have been
+added. These come with syntactic sugar as mentioned above. (We plan to
+keep this syntax, but we're open to feedback and changes if they're
+good.)
+
+* Resources can accept the interface{} (any) type, although this should
+be used sparingly.
+
+* We added a new mcl test suite that checks resource output too!
+
+* Added a new `value` resource. This is a special resource kind that
+can be used for building some powerful state machines. Recommended for
+experienced users only.
+
+* Improved the golang function generation to allow functions that take
+[]str, so now we have a bunch more functions (like join) in our stdlib
+for free.
+
+* Add some mac address formatting functions. (core/net)
+
+* Added a panic resource and panic function into the core language.
+This is useful for safely shutting down a running mcl program to
+prevent something disastrous or unhandled.
+
+* Added a `donecmd` field to the exec resource. This runs a command
+after a successful CheckApply. This replaces the `&& echo foo > done`
+pattern that you'd see in some code.
+
+* Added a new internal `local` API that can be used for local machine
+operations. So far, read/writing/watching values that are stored
+locally.
+
+* Added `value` functions which bridge the `value` resource via the
+`local` API. To be used sparingly!
+
+* Bumped to golang 1.20, and we'll probably move again before the next
+release.
+
+* Allow send/recv with autogrouped resources. This adds many
+possibilities, in particular with the server style resources.
+
+* Added a bunch of tests for sneaky corner cases. Some of these were
+hard to write, but I think they're worth it.
+
+* ExprBind is now monomorphic! This was a design mistake that we
+introduced, but have since repaired. We now have far fewer copies
+running in the function graph, and things are much more efficient. This
+means lambdas can only have one type when used at two different call
+sites, which is much more logical, safer, faster and memory efficient.
+
+* Added an --only-unify option if you want to test your code but not
+run it.
+
+* Added a concat function for the common case of interpolation. This
+makes type unification significantly faster.
+
+* Eliminated some "benign" races. You might find this commit
+interesting to read: bc63b7608e84f60bf9d568188814d411a0688738
+
+* A pgraph bug was found and fixed. A test was added too! It's amazing
+this was here for so long, it just shows how subtle graph
+datastructures can be.
+
+* Added `include as` (for classes) to the language which lets our
+classes produce values which can then be used elsewhere. I decided this
+feature would be necessary after writing a bunch of mcl. It does have
+an extraneous scoping bug, but not anything that causes problems.
+
+* Nested classes are now supported. This lets you write the equivalent
+of nested classes, without actually having to nest them! This is not
+inheritance, but rather a way of handling scope and passing it
+downwards.
+
+* Improved the Ordering compiler step to catch a bunch of unhandled
+bugs. Sam is a genius and was able to figure out some of these using
+wizardry.
+
+* Added some convert functions to the mcl package.
+
+* Allow edges with colons...
+
+* ...Because we now support a new hierarchical autogrouping algorithm!
+This let's us have some very powerful resources.
+
+* ...Like http:*, dhcp:*, and so on, but we could even go deeper!
+
+* Fixed a super sneaky bug with resource swapping. Due to how we Cmp,
+this now preserves state more often, and in particular when we need it.
+I'm fairly certain that some code in a WIP branch of mine was actually
+blocked because of this issue. Pleased to run into it again, but now
+with a fix in place!
+
+* Added an http:flag resource. This let's a `wget` or similar call back
+to the http:server to kick off an action.
+
+* The http:flag resource supports directories now.
+
+* Stats are printed if the function engine is waiting for too long.
+This is mostly useful for developers who are building new functions and
+have a bug in their midst!
+
+* We added a --skip-unify option to prevent the double unification when
+running locally. When using `mgmt run` to test locally, we type check,
+and then deploy to ourselves, which then naturally type checks again.
+This skips the first one, which would be unsafe generally, but is
+perfectly safe when we're running a single instance.
+
+* Added a new http:proxy resource, and then tweaked it's API, and then
+added http streaming. This is an incredibly powerful resource that lets
+us build a caching http proxy with a single resource. I can't wait to
+see what else it gets used for. I'm using it for provisioning. It's not
+performance optimized at the moment as it uses a single mutex for
+everything, but this could be extended if we wanted to scale this out.
+
+* Added a ton of measuring/timing of common operations. This confirmed
+my belief that autoedges were slower than necessary. There are two ways
+to improve this. We might end up doing either one or both. Autogrouping
+is currently much faster than needed, so no improvements planned for
+now!
+
+* Started to clean up the internal FS API's. It would be really great
+if the core golang team would add something so we could get rid of the
+afero external interfaces.
+
+* Added an "embedded" package to offer API's related to embedded mcl
+programs! This lets us build standalone binaries which are powered by
+mcl.
+
+* Moved to a new CLI (go-arg) library. This has a few downsides, but
+they are fixable upstream, and this vastly improved our code quality
+and API's. This needed to happen, what with the mess that was
+urfave/cli. Look at our diff's, they're really elegant! This let us
+clean up our lib structs as well!
+
+* Added an "entry" package to kick-off the embedded API stuff. This
+uses the new CLI API's that we just built. The end-user now has a
+really easy time building new tools.
+
+* Added a bunch of util functions to aid in building certain standalone
+tools. I'm willing to accept more contributions in this space if
+they're sane, and related to our general mission. Please ask and then
+send patches if you're unsure.
+
+* Added a firewalld resource which makes opening up ports automatic
+when we need them. Perfect for the standalone laptop use-case.
+
+* Made type unification cancellable in case you get into a long-running
+scenario and want to end early.
+
+* Added a `creates` field to the exec resource. Very useful, and also
+supports watches! This is very useful for the common uses of exec.
+
+* Added a dhcp:range resource to offer any number of IP addresses to
+devices that we don't know the mac addresses of in advance. This makes
+building a provisioning tool even more ergonomic.
+
+* Optimized the name invariants since we can usually avoid an exclusive
+invariant in the common case. This roughly halved the type unification
+time. More improvements coming too!
+
+* Caught a sneaky list type that could get through type unification
+when it was interpolated alone. This now enforces the string invariant
+when we specify it, which is an important language design distinction.
+We added tests for this of course too!
+
+* The "log" package has been entirely refactored and is only visible in
+one place at the top of the program. Nice! I have a design for a
+"better logger / user interface" if we ever want to improve on this.
+
+* Added release targets for standalone binary builds. I also improved
+the Makefile release magic significantly.
+
+* Made a lot of small "polish" improvements to various resources.
+
+* Most interestingly, an embedded provisioner application has been
+built and made available in full. Please test and share with others.
+Hopefully this will encourage more interest in the project.
+
+* We're looking for help writing Amazon, Google, DigitalOcean, Hetzner,
+etc, resources if anyone is interested, reach out to us. Particularly
+if there is support from those organizations as well.
+
+* Many other bug fixes, changes, etc...
+
+* See the git log for more NEWS, and for anything notable I left out!
+
+
+BUGS/TODO
+
+* Function values getting _passed_ to resources doesn't work yet, but
+it's not a blocker, but it would definitely be useful. We're looking
+into it.
+
+* Function graphs are unnecessarily dynamic. We might make them more
+static so that we don't need as many transactions. This is really a
+compiler optimization and not a bug, but it's something important we'd
+like to have.
+
+* Running two Txn's during the same pause would be really helpful. I'm
+not sure how much of a performance improvement we'd get from this, but
+it would sure be interesting to build. If you want to build a fancy
+synchronization primitive, then let us know! Again this is not a bug.
+
+* Type unification performance can be improved drastically. I will have
+to implement the fast algorithm so that we can scale to very large mcl
+programs. Help is wanted if you are familiar with "unionfind" and/or
+type unification.
+
+
+TALKS
+
+I don't have anything planned until CfgMgmtCamp 2025. If you'd like to
+book me for a private event, or sponsor my travel for your conference,
+please let me know.
+
+I recently gave two talks: one at CfgMgmtCamp 2024, and one at FOSDEM
+in the golang room. Both are available online and demonstrated an
+earlier version of the provisioning tool which is fully available
+today. The talks can be found here: https://purpleidea.com/talks/
+
+
+PARTNER PROGRAM
+
+We have a new mgmt partner program which gets you early access to
+releases, bug fixes, support, and many other goodies. Please sign-up
+today: https://bit.ly/mgmt-partner-program
+
+
+MISC
+
+Our mailing list host (Red Hat) is no longer letting non-Red Hat
+employees use their infrastructure. We're looking for a new home. I've
+opened a ticket with Freedesktop. If you have any sway with them or
+other recommendations, please let me know:
+https://gitlab.freedesktop.org/freedesktop/freedesktop/-/issues/1082
+
+We're still looking for new contributors, and there are easy, medium
+and hard issues available! You're also welcome to suggest your own!
+Please join us in #mgmtconfig on Libera IRC or Matrix (preferred) and
+ping us if you'd like help getting started! For details please see:
+
+https://github.com/purpleidea/mgmt/blob/master/docs/faq.md#how-do-i-con
+tribute-to-the-project-if-i-dont-know-golang
+
+Many tagged #mgmtlove issues exist:
+https://github.com/purpleidea/mgmt/issues?q=is%3Aissue+is%3Aopen+label%
+3Amgmtlove
+
+Although asking in IRC/matrix is the best way to find something to work
+on.
+
+
+MENTORING
+
+We offer mentoring for new golang/mgmt hackers who want to get
+involved. This is fun and friendly! You get to improve your skills,
+and we get some patches in return. Ping me off-list for details.
+
+
+THANKS
+
+Thanks (alphabetically) to everyone who contributed to the latest
+release:
+Eng Zer Jun, James Shubin, Oliver Lowe, Samuel Gélineau
+We had 4 unique committers since 0.0.24, and have had 90 overall.
+run 'git log 0.0.24..0.0.25' to see what has changed since 0.0.24
+
+
+Happy hacking,
+James
+@purpleidea

docs/release-notes/0.0.26

@@ -0,0 +1,142 @@
+I've just released version 0.0.26 of mgmt!
+
+> 16 files changed, 869 insertions(+), 181 deletions(-)
+
+Hot off the heels of the recent large release (0.0.25) I've just
+released an incremental update...
+
+See more here:
+
+https://purpleidea.com/blog/2024/03/27/a-new-provisioning-tool/
+
+With that, here are a few highlights from the release:
+
+* We have a new mgmt partner program. Please sign-up for early access
+to these release notes, along with other special privileges. Details
+at: https://bit.ly/mgmt-partner-program
+
+* Type unification for the provisioning tool is about 40x faster.
+
+* We fix a small bug related to the upcoming fedora 40 release.
+
+And much more...
+
+
+DOWNLOAD
+
+Prebuilt binaries are available here for this release:
+https://github.com/purpleidea/mgmt/releases/tag/0.0.26
+
+They can also be found on the Fedora mirror:
+https://dl.fedoraproject.org/pub/alt/purpleidea/mgmt/releases/0.0.26/
+
+
+NEWS
+
+* Added old release notes into git
+
+* We now skip over unreleased Fedora versions (like "40 Beta") when
+trying to automatically determine the latest stable release.
+
+* Type unification was structurally refactored to make way for a bunch
+of future improvements and generally to modernize the code.
+
+* Added some unification optimizations and a unification flag
+optimizations system to allow solvers to support special flags. One of
+these new flags was used for the provisioner code with a substantial
+improvement in type unification time by about 40x.
+
+* New cli args are also available for using these flags.
+
+* We're looking for help writing Amazon, Google, DigitalOcean, Hetzner,
+etc, resources if anyone is interested, reach out to us. Particularly
+if there is support from those organizations as well.
+
+* Many other bug fixes, changes, etc...
+
+* See the git log for more NEWS, and for anything notable I left out!
+
+
+BUGS/TODO
+
+* Function values getting _passed_ to resources doesn't work yet, but
+it's not a blocker, but it would definitely be useful. We're looking
+into it.
+
+* Function graphs are unnecessarily dynamic. We might make them more
+static so that we don't need as many transactions. This is really a
+compiler optimization and not a bug, but it's something important we'd
+like to have.
+
+* Running two Txn's during the same pause would be really helpful. I'm
+not sure how much of a performance improvement we'd get from this, but
+it would sure be interesting to build. If you want to build a fancy
+synchronization primitive, then let us know! Again this is not a bug.
+
+* General type unification performance can be improved drastically. I
+will have to implement the fast algorithm so that we can scale to very
+large mcl programs. Help is wanted if you are familiar with "unionfind"
+and/or type unification.
+
+
+TALKS
+
+I don't have anything planned until CfgMgmtCamp 2025. If you'd like to
+book me for a private event, or sponsor my travel for your conference,
+please let me know.
+
+I recently gave two talks: one at CfgMgmtCamp 2024, and one at FOSDEM
+in the golang room. Both are available online and demonstrated an
+earlier version of the provisioning tool which is fully available
+today. The talks can be found here: https://purpleidea.com/talks/
+
+
+PARTNER PROGRAM
+
+We have a new mgmt partner program which gets you early access to
+releases, bug fixes, support, and many other goodies. Please sign-up
+today: https://bit.ly/mgmt-partner-program
+
+
+MISC
+
+Our mailing list host (Red Hat) is no longer letting non-Red Hat
+employees use their infrastructure. We're looking for a new home. I've
+opened a ticket with Freedesktop. If you have any sway with them or
+other recommendations, please let me know:
+https://gitlab.freedesktop.org/freedesktop/freedesktop/-/issues/1082
+
+We're still looking for new contributors, and there are easy, medium
+and hard issues available! You're also welcome to suggest your own!
+Please join us in #mgmtconfig on Libera IRC or Matrix (preferred) and
+ping us if you'd like help getting started! For details please see:
+
+https://github.com/purpleidea/mgmt/blob/master/docs/faq.md#how-do-i-con
+tribute-to-the-project-if-i-dont-know-golang
+
+Many tagged #mgmtlove issues exist:
+https://github.com/purpleidea/mgmt/issues?q=is%3Aissue+is%3Aopen+label%
+3Amgmtlove
+
+Although asking in IRC/matrix is the best way to find something to work
+on.
+
+
+MENTORING
+
+We offer mentoring for new golang/mgmt hackers who want to get
+involved. This is fun and friendly! You get to improve your skills,
+and we get some patches in return. Ping me off-list for details.
+
+
+THANKS
+
+Thanks (alphabetically) to everyone who contributed to the latest
+release:
+James Shubin
+We had 1 unique committers since 0.0.25, and have had 90 overall.
+
+
+Happy hacking,
+James
+@purpleidea

docs/release-notes/0.0.9

@@ -0,0 +1,71 @@
+I've just released 0.0.9!
+
+Release tags are now signed with my GPG key.
+
+From now on, I'm going to try and write a release email to the list
+with information about each release. Here we go!
+
+NEWS
+
+* There were far more commits that went into this release than I was
+expecting. I delayed a while because of some nasty races and deadlocks
+I encountered when cleaning some cruft out of the engine.
+
+134 files changed, 5394 insertions(+), 2168 deletions(-)
+
+* We grew initial integration with Prometheus thanks to new contributor
+`roidelapluie`.
+
+* The file resource can now chown/chmod files! Thanks to new
+contributor `mildred`.
+
+* The virt resource can now hotplug/hotunplug cpus. This made for some
+dope demos at CfgMgmtCamp. Special thanks to pkrempa for helping me
+with the libvirt API.
+
+* Native testing of all the golang code is now enabled again. I broke
+this when I split mgmt up into multiple packages. (Sorry!) Golang tests
+in foo_test.go should now run and be tested automatically. I'd
+especially like it if someone wrote some for pgraph/pgraph.go:
+GraphSync().
+
+* Documentation has been significantly improved. We have a resource
+guide (for writing new native resources) and much more. We also now
+build the docs as a RTD project:
+https://mgmt.readthedocs.io/en/master/
+Patches welcome to make it style as pretty as GitHub markdown does.
+
+* See the git log for more NEWS, and sorry for anything notable I left
+out!
+
+BUGS
+
+* I hope to resolve this soon, but it shouldn't block further
+development on git master for now. The issue: if you make extremely
+high speed graph changes to graphs involving the virt resource, it can
+eventually cause a panic. This is being tracked here:
+https://github.com/libvirt/libvirt-go/issues/7 (help welcome!)
+
+GLUSTERFS
+
+I've started working on some stuff for Glusterd so that it can embed
+mgmt as a lib, and use the resource model to simplify cluster
+management. This will involve writing Gluster resources in mgmt
+(volume, brick, etc...) and all sorts of other fun stuff. If you'd like
+to participate, please LMK!
+
+MISC
+
+We're still looking for new contributors, and there are easy, medium and hard issues available! You're also welcome to suggest your own! Please join us in #mgmtconfig on Freenode IRC, or ping this list if you'd like help getting started!
+
+THANKS
+
+We had 12 unique committers since 0.0.8, and have had 29 overall.
+Thanks (alphabetically) to everyone who contributed to the latest
+release: Daniele Sluijters, Daniel P. Berrange, Felix Frank, goberghen,
+James Shubin, Julien Pivotto, Kaushal M, Lars Kulseng, Mildred Ki'Lya,
+Sean Jones, Steve Milner, Tom Ritserveldt.
+
+Happy hacking,
+James
+@purpleidea

docs/resource-guide.md

@@ -61,8 +61,8 @@ struct name is ambiguous.
 
 If you'd like your resource to be accessible by the `YAML` graph API (GAPI),
 then you'll need to include the appropriate YAML fields as shown below. This is
-used by the `Puppet` compiler as well, so make sure you include these struct
-tags if you want existing `Puppet` code to be able to run using the `mgmt`
+used by the `puppet` compiler as well, so make sure you include these struct
+tags if you want existing `puppet` code to be able to run using the `mgmt`
 engine.
 
 #### Example
@@ -96,14 +96,16 @@ Default() engine.Res
 ```
 
 This returns a populated resource struct as a `Res`. It shouldn't populate any
-values which already have the correct default as the golang zero value. In
+values which already get a good default as the respective golang zero value. In
 general it is preferable if the zero values make for the correct defaults.
+(This is to say, resources are designed to behave safely and intuitively
+when parameters take a zero value, whenever this is possible.)
 
 #### Example
 
 ```golang
 // Default returns some sensible defaults for this resource.
-func (obj *FooRes) Default() Res {
+func (obj *FooRes) Default() engine.Res {
 	return &FooRes{
 		Answer: 42, // sometimes, defaults shouldn't be the zero value
 	}
@@ -175,10 +177,10 @@ this. In other words, you should expect `Validate` to have run first, but you
 shouldn't allow `Init` to dangerously `rm -rf /$the_world` if your code only
 checks `$the_world` in `Validate`. Remember to always program safely!
 
-### Close
+### Cleanup
 
 ```golang
-Close() error
+Cleanup() error
 ```
 
 This is called to cleanup after the resource. It is usually not necessary, but
@@ -190,8 +192,8 @@ loop.
 #### Example
 
 ```golang
-// Close runs some cleanup code for this resource.
-func (obj *FooRes) Close() error {
+// Cleanup is run by the engine to clean up after the resource is done.
+func (obj *FooRes) Cleanup() error {
 	err := obj.conn.Close() // close some internal connection
 	obj.someMap = nil       // free up some large data structure from memory
 	return err
@@ -204,7 +206,7 @@ on an error if something went wrong.
 ### CheckApply
 
 ```golang
-CheckApply(apply bool) (checkOK bool, err error)
+CheckApply(ctx context.Context, apply bool) (checkOK bool, err error)
 ```
 
 `CheckApply` is where the real _work_ is done. Under normal circumstances, this
@@ -213,7 +215,8 @@ should return: `(true, nil)`. If the `apply` variable is set to `true`, then
 this means that we should then proceed to run the changes required to bring the
 resource into the correct state. If the `apply` variable is set to `false`, then
 the resource is operating in _noop_ mode and _no operational changes_ should be
-made!
+made! The ctx should be monitored in case a shutdown has been requested. This
+may be used if a timeout occurred, or if the user shutdown the engine.
 
 After having executed the necessary operations to bring the resource back into
 the desired state, or after having detected that the state was incorrect, but
@@ -232,7 +235,7 @@ to `CheckApply`.
 
 ```golang
 // CheckApply does the idempotent work of checking and applying resource state.
-func (obj *FooRes) CheckApply(apply bool) (bool, error) {
+func (obj *FooRes) CheckApply(ctx context.Context, apply bool) (bool, error) {
 	// check the state
 	if state_is_okay { return true, nil } // done early! :)
 
@@ -275,7 +278,7 @@ will likely find the state to now be correct.
 ### Watch
 
 ```golang
-Watch() error
+Watch(ctx context.Context) error
 ```
 
 `Watch` is a main loop that runs and sends messages when it detects that the
@@ -302,26 +305,25 @@ If the resource is activated in `polling` mode, the `Watch` method will not get
 executed. As a result, the resource must still work even if the main loop is not
 running.
 
+You must make sure to cleanup any running code or goroutines before Watch exits.
+
 #### Select
 
 The lifetime of most resources `Watch` method should be spent in an infinite
 loop that is bounded by a `select` call. The `select` call is the point where
 our method hands back control to the engine (and the kernel) so that we can
-sleep until something of interest wakes us up. In this loop we must process
-events from the engine via the `<-obj.init.Events` channel, and receive events
-for our resource itself!
+sleep until something of interest wakes us up. In this loop we must wait until
+we get a shutdown event from the engine via the `<-ctx.Done()` channel, which
+closes when we'd like to shut everything down. At this point you should cleanup,
+and let `Watch` close.
 
 #### Events
 
-If we receive an internal event from the `<-obj.init.Events` channel, we should
-read it with the `obj.init.Read` helper function. This function tells us if we
-should shutdown our resource. It also handles pause functionality which blocks
-our resource temporarily in this method. If this channel shuts down, then we
-should treat that as an exit signal.
-
-When we want to send an event, we use the `Event` helper function. It is also
-important to mark the resource state as `dirty` if we believe it might have
-changed. We do this by calling the `obj.init.Dirty` function.
+If the `<-ctx.Done()` channel closes, we should shutdown our resource. When we
+want to send an event, we use the `Event` helper function. This automatically
+marks the resource state as `dirty`. If you're unsure, it's not harmful to send
+the event. This will ultimately cause `CheckApply` to run. This method can block
+if the resource is being paused.
 
 #### Startup
 
@@ -330,8 +332,7 @@ to generate one event to notify the `mgmt` engine that we're now listening
 successfully, so that it can run an initial `CheckApply` to ensure we're safely
 tracking a healthy state and that we didn't miss anything when `Watch` was down
 or from before `mgmt` was running. You must do this by calling the
-`obj.init.Running` method. If it returns an error, you must exit and return that
-error.
+`obj.init.Running` method.
 
 #### Converged
 
@@ -349,7 +350,7 @@ sending out erroneous `Event` messages to keep things alive until it finishes.
 
 ```golang
 // Watch is the listener and main loop for this resource.
-func (obj *FooRes) Watch() error {
+func (obj *FooRes) Watch(ctx context.Context) error {
 	// setup the Foo resource
 	var err error
 	if err, obj.foo = OpenFoo(); err != nil {
@@ -358,41 +359,29 @@ func (obj *FooRes) Watch() error {
 	defer obj.whatever.CloseFoo() // shutdown our Foo
 
 	// notify engine that we're running
-	if err := obj.init.Running(); err != nil {
-		return err // exit if requested
-	}
+	obj.init.Running() // when started, notify engine that we're running
 
 	var send = false // send event?
 	for {
 		select {
-		case event, ok := <-obj.init.Events:
-			if !ok {
-				// shutdown engine
-				// (it is okay if some `defer` code runs first)
-				return nil
-			}
-			if err := obj.init.Read(event); err != nil {
-				return err
-			}
-
 		// the actual events!
 		case event := <-obj.foo.Events:
 			if is_an_event {
 				send = true
-				obj.init.Dirty() // dirty
 			}
 
 		// event errors
 		case err := <-obj.foo.Errors:
 			return err // will cause a retry or permanent failure
+
+		case <-ctx.Done(): // signal for shutdown request
+			return nil
 		}
 
 		// do all our event sending all together to avoid duplicate msgs
 		if send {
 			send = false
-			if err := obj.init.Event(); err != nil {
-				return err // exit if requested
-			}
+			obj.init.Event()
 		}
 	}
 }
@@ -567,24 +556,6 @@ ready to detect changes.
 Event sends an event notifying the engine of a possible state change. It is
 only called from within `Watch`.
 
-### Events
-
-Events is a channel that we must watch for messages from the engine. When it
-closes, this is a signal to shutdown. It is
-only called from within `Watch`.
-
-### Read
-
-Read processes messages that come in from the `Events` channel. It is a helper
-method that knows how to handle the pause mechanism correctly. It is
-only called from within `Watch`.
-
-### Dirty
-
-Dirty marks the resource state as dirty. This signals to the engine that
-CheckApply will have some work to do in order to converge it. It is
-only called from within `Watch`.
-
 ### Refresh
 
 Refresh returns whether the resource received a notification. This flag can be
@@ -609,15 +580,15 @@ It is only called from within `CheckApply`.
 World provides a connection to the outside world. This is most often used for
 communicating with the distributed database. It can be used in `Init`,
 `CheckApply` and `Watch`. Use with discretion and understanding of the internals
-if needed in `Close`.
+if needed in `Cleanup`.
 
 ### VarDir
 
 VarDir is a facility for local storage. It is used to return a path to a
 directory which may be used for temporary storage. It should be cleaned up on
-resource `Close` if the resource would like to delete the contents. The resource
-should not assume that the initial directory is empty, and it should be cleaned
-on `Init` if that is a requirement.
+resource `Cleanup` if the resource would like to delete the contents. The
+resource should not assume that the initial directory is empty, and it should be
+cleaned on `Init` if that is a requirement.
 
 ### Debug
 
@@ -652,7 +623,7 @@ func init() { // special golang method that runs once
 
 To support YAML unmarshalling for your resource, you must implement an
 additional method. It is recommended if you want to use your resource with the
-`Puppet` compiler.
+`puppet` compiler.
 
 ```golang
 UnmarshalYAML(unmarshal func(interface{}) error) error // optional
@@ -669,8 +640,8 @@ The signature intentionally matches what is required to satisfy the `go-yaml`
 #### Example
 
 ```golang
-// UnmarshalYAML is the custom unmarshal handler for this struct.
-// It is primarily useful for setting the defaults.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
+// primarily useful for setting the defaults.
 func (obj *FooRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes FooRes // indirection to avoid infinite recursion
 
@@ -707,10 +678,10 @@ receiving one. This can _only_ be done inside of the `CheckApply` function!
 
 ```golang
 // inside CheckApply, probably near the top
-if val, exists := obj.init.Recv()["SomeKey"]; exists {
-	obj.init.Logf("the SomeKey param was sent to us from: %s.%s", val.Res, val.Key)
+if val, exists := obj.init.Recv()["some_key"]; exists {
+	obj.init.Logf("the some_key param was sent to us from: %s.%s", val.Res, val.Key)
 	if val.Changed {
-		obj.init.Logf("the SomeKey param was just updated!")
+		obj.init.Logf("the some_key param was just updated!")
 		// you may want to invalidate some local cache
 	}
 }
@@ -745,7 +716,7 @@ Higher level resource collections will be possible once the `mgmt` DSL is ready.
 ### Why does the resource API have `CheckApply` instead of two separate methods?
 
 In an early version we actually had both "parts" as separate methods, namely:
-`StateOK` (Check) and `Apply`, but the [decision](58f41eddd9c06b183f889f15d7c97af81b0331cc)
+`StateOK` (Check) and `Apply`, but the [decision](https://github.com/purpleidea/mgmt/commit/58f41eddd9c06b183f889f15d7c97af81b0331cc)
 was made to merge the two into a single method. There are two reasons for this:
 
 1. Many situations would involve the engine running both `Check` and `Apply`. If
@@ -754,16 +725,16 @@ two calls, this is much more difficult. A common example is that a resource
 might want to open a connection to `dbus` or `http` to do resource state testing
 and applying. If the methods are combined, there's no need to open and close
 them twice. A counter argument might be that you could open the connection in
-`Init`, and close it in `Close`, however you might not want that open for the
+`Init`, and close it in `Cleanup`, however you might not want that open for the
 full lifetime of the resource if you only change state occasionally.
 2. Suppose you came up with a really good reason why you wanted the two methods
 to be separate. It turns out that the current `CheckApply` can wrap this easily.
 It would look approximately like this:
 
 ```golang
-func (obj *FooRes) CheckApply(apply bool) (bool, error) {
+func (obj *FooRes) CheckApply(ctx context.Context, apply bool) (bool, error) {
 	// my private split implementation of check and apply
-	if c, err := obj.check(); err != nil {
+	if c, err := obj.check(ctx); err != nil {
 		return false, err // we errored
 	} else if c {
 		return true, nil // state was good!
@@ -773,7 +744,7 @@ func (obj *FooRes) CheckApply(apply bool) (bool, error) {
 		return false, nil // state needs fixing, but apply is false
 	}
 
-	err := obj.apply() // errors if failure or unable to apply
+	err := obj.apply(ctx) // errors if failure or unable to apply
 
 	return false, err // always return false, with an optional error
 }
@@ -783,6 +754,23 @@ Feel free to use this pattern if you're convinced it's necessary. Alternatively,
 if you think I got the `Res` API wrong and you have an improvement, please let
 us know!
 
+### Why do resources have both a `Cmp` method and an `IFF` (on the UID) method?
+
+The `Cmp()` methods are for determining if two resources are effectively the
+same, which is used to make graph change delta's efficient. This is when we want
+to change from the current running graph to a new graph, but preserve the common
+vertices. Since we want to make this process efficient, we only update the parts
+that are different, and leave everything else alone. This `Cmp()` method can
+tell us if two resources are the same. In case it is not obvious, `cmp` is an
+abbrev. for compare.
+
+The `IFF()` method is part of the whole UID system, which is for discerning if a
+resource meets the requirements another expects for an automatic edge. This is
+because the automatic edge system assumes a unified UID pattern to test for
+equality. In the future it might be helpful or sane to merge the two similar
+comparison functions although for now they are separate because they are
+actually answer different questions.
+
 ### What new resource primitives need writing?
 
 There are still many ideas for new resources that haven't been written yet. If

docs/resources.md

@@ -17,6 +17,7 @@ You might want to look at the [generated documentation](https://godoc.org/github
 for more up-to-date information about these resources.
 
 * [Augeas](#Augeas): Manipulate files using augeas.
+* [Consul:KV](#ConsulKV): Set keys in a Consul datastore.
 * [Docker](#Docker):[Container](#Container) Manage docker containers.
 * [Exec](#Exec): Execute shell commands on the system.
 * [File](#File): Manage files and directories.
@@ -32,6 +33,8 @@ for more up-to-date information about these resources.
 * [Print](#Print): Print messages to the console.
 * [Svc](#Svc): Manage system systemd services.
 * [Test](#Test): A mostly harmless resource that is used for internal testing.
+* [Tftp:File](#TftpFile): Add files to the small embedded embedded tftp server.
+* [Tftp:Server](#TftpServer): Run a small embedded tftp server.
 * [Timer](#Timer): Manage system systemd services.
 * [User](#User): Manage system users.
 * [Virt](#Virt): Manage virtual machines with libvirt.
@@ -69,9 +72,9 @@ identified by a trailing slash in their path name. File have no such slash.
 It has the following properties:
 
 * `path`: absolute file path (directories have a trailing slash here)
+* `state`: either `exists`, `absent`, or undefined
 * `content`: raw file content
-* `state`: either `exists` (the default value) or `absent`
-* `mode`: octal unix file permissions
+* `mode`: octal unix file permissions or symbolic string
 * `owner`: username or uid for the file owner
 * `group`: group name or gid for the file group
 
@@ -79,6 +82,16 @@ It has the following properties:
 
 The path property specifies the file or directory that we are managing.
 
+### State
+
+The state property describes the action we'd like to apply for the resource. The
+possible values are: `exists` and `absent`. If you do not specify either of
+these, it is undefined. Without specifying this value as `exists`, another param
+cannot cause a file to get implicitly created. When specifying this value as
+`absent`, you should not specify any other params that would normally change the
+file. For example, if you specify `content` and this param is `absent`, then you
+will get an engine validation error.
+
 ### Content
 
 The content property is a string that specifies the desired file contents.
@@ -88,10 +101,12 @@ The content property is a string that specifies the desired file contents.
 The source property points to a source file or directory path that we wish to
 copy over and use as the desired contents for our resource.
 
-### State
+### Fragments
 
-The state property describes the action we'd like to apply for the resource. The
-possible values are: `exists` and `absent`.
+The fragments property lets you specify a list of files to concatenate together
+to make up the contents of this file. They will be combined in the order that
+they are listed in. If one of the files specified is a directory, then the
+files in that top-level directory will be themselves combined together and used.
 
 ### Recurse
 
@@ -104,6 +119,12 @@ The force property is required if we want the file resource to be able to change
 a file into a directory or vice-versa. If such a change is needed, but the force
 property is not set to `true`, then this file resource will error.
 
+### Purge
+
+The purge property is used when this file represents a directory, and we'd like
+to remove any unmanaged files from within it. Please note that any unmanaged
+files in a directory with this flag set will be irreversibly deleted.
+
 ## Group
 
 The group resource manages the system groups from `/etc/group`.
@@ -206,6 +227,16 @@ The service resource is still very WIP. Please help us by improving it!
 
 The test resource is mostly harmless and is used for internal tests.
 
+## Tftp:File
+
+This adds files to the running tftp server. It's useful because it allows you to
+add individual files without needing to create them on disk.
+
+## Tftp:Server
+
+Run a small embedded tftp server. This doesn't apply any state, but instead runs
+a pure golang tftp server in the Watch loop.
+
 ## Timer
 
 This resource needs better documentation. Please help us by improving it!

docs/service-guide.md

@@ -0,0 +1,145 @@
+# Service API design guide
+
+This document is intended as a short instructional design guide in building a
+service management API. It is certainly intended for someone who wishes to use
+`mgmt` resources and functions to interact with their facilities, however it may
+be of more general use as well. Hopefully this will help you make smarter design
+considerations early on, and prevent some amount of unnecessary technical debt.
+
+## Main aspects
+
+What follows are some of the most common considerations which you may wish to
+take into account when building your service. This list is non-exhaustive. Of
+particular note, as of the writing of this document, many of these designs are
+not taken into account or not well-handled or implemented by the major API
+("cloud") providers.
+
+### Authentication
+
+#### The status-quo
+
+Many services naturally require you to authenticate yourself. Usually the
+initial user who sets up the account and provides credit card details will need
+to download secret credentials in order to access the service. The onus is on
+the user to keep those credentials private, and to prevent leaking them. It is
+convenient (and insecure) to store them in `git` repositories containing scripts
+and configuration management code. Since it's likely you will use multiple
+different services, it also means you will have a ton of different credentials
+to guard.
+
+#### An alternative
+
+Instead, build your service to accept a public key that you store in the users
+account. Only consumers that can correctly sign messages matching this public
+key should be authorized. This mechanism is well-understood by anyone who has
+ever uploaded their public SSH key to a server. You can use SSH keys, GPG keys,
+or even get into Kerberos if that's appropriate. Best of all, if you and other
+services use a standardized mechanism like GPG, a user might only need to keep
+track of their single key-pair, even when they're using multiple services!
+
+### Events
+
+#### The problem
+
+People have been building "[CRUD](https://en.wikipedia.org/wiki/Create,_read,_update_and_delete)"
+and "[REST](https://en.wikipedia.org/wiki/REST)"ful API's for years. The biggest
+missing part that most of them don't provide is events. If users want to know
+when a resource changes, they have to repeatedly poll the server, which is both
+network intensive, and introduces latency. When services were simpler, this
+wasn't as much of a consideration, but these days it matters. An embarrassingly
+small number of major software vendors implement these correctly, if at all.
+
+#### Why events?
+
+The `mgmt` tool is different from most other static tools in that it allows
+reading streams of incoming data, and stream of change events from resources we
+are managing. If an event API is not available, we can still poll, but this is
+not as desirable. An event-capable API doesn't prevent polling if that's
+preferred, you can always repeat a read request periodically.
+
+#### Variants
+
+The two common mechanisms for receiving events are "callbacks" and
+"long-polling". In the former, the service contacts the consumer when something
+happens. In the latter, the consumer opens a connection, and the service either
+closes the connection or sends the reply, when it's ready. Long-polling is often
+preferred since it doesn't require an open firewall on the consumers side.
+Callbacks are preferred because it's often cheaper for the service to implement
+that. It's also less reliable since it's hard to know if the callback message
+wasn't received because it was dropped, or if there just wasn't an event. And it
+requires static timeouts when retrying a callback message, and so on. It's best
+to implement long-polling or something equivalent at a minimum.
+
+#### "Since" requests
+
+When making an event request, some API's will let you tack on a "since" style
+parameter that tells the endpoint that we're interested in all of the events
+_since_ a particular timestamp, or _since_ a particular sequence ID. This can be
+very useful if missing an intermediate event is a concern. Implement this if you
+can, but it's better for all concerned if purely declarative facilities are all
+that is required. It also forces the endpoint to maintain some state, which may
+be undesirable for them.
+
+#### Out of band
+
+Some providers have the event system tacked on to a separate facility. If it's
+not part of the core API, then it's not useful. You shouldn't have to configure
+a separate system in order to start getting events.
+
+### Batching
+
+With so many resources, you might expect to have 1000's of long-polling
+connections all sitting open and idle. That can't be efficient! It's not, which
+is why good API's need a batching facility. This lets the consumer group
+together many watches (all waiting on a long-poll) inside of a single call. That
+way, a single connection might only be needed for a large amount of information.
+
+### Don't auto-generate junk
+
+Please build an elegant API. Many services auto-generate a "phone book" SDK of
+junk. It might seem inevitable, so if you absolutely need to do this, then put
+some extra effort into making it idiomatic. If I'm using an SDK generated for
+`golang` and I see an internal `foo.String` wrapper, then chances are you have
+designed your API and code to be easier to maintain for you, instead of
+prioritizing your customers. Surely the total volume of all customer code is
+more than your own, so why optimize for that instead of the putting the customer
+first?
+
+### Resources and functions
+
+`Mgmt` has a concept of "resources" and "functions". Resources are used in an
+idempotent model to express desired state and perform that work, and "functions"
+are used to receive and pull data into the system. That separation has shown to
+be an elegant one. Consider it when designing your API's. For example, if some
+vital information can only be obtained after performing a modifying operation,
+then it might signal that you're missing some sort of a lookup or event-log
+system. Design your API's to be idempotent, this solves many distributed-system
+problems involving receiving duplicate messages, and so on.
+
+## Using mgmt as a library
+
+Instead of building a new service from scratch, and re-inventing the typical
+management and CLI layer, consider using `mgmt` as a library, and directly
+benefiting from that work. This has not been done for a large production
+service, but the author believes it would be quite efficient, particularly if
+your application is written in golang. It's equivalently easy to do it for other
+languages as well, you just end up with two binaries instead of one. (Or you can
+embed the other binary into the new golang management tool.)
+
+## Cloud API considerations
+
+Many "cloud" companies have a lot of technical debt and a lot of customers. As a
+result, it might be very hard for them to improve their API's, particularly
+without breaking compatibility promises for their existing customers. As a
+result, they should either add a versioned API, which lets newer consumers get
+the benefit, or add new parallel services which offer the modern features. If
+they don't, the only solution is for new competitors to build-in these better
+efficiencies, eventually offering better value to cost ratios, which will then
+make legacy products less lucrative and therefore unmaintainable as compared to
+their competitors.
+
+## Suggestions
+
+If you have any ideas for suggestions or other improvements to this guide,
+please let us know! I hope this was helpful. Please reach out if you are
+building an API that you might like to have `mgmt` consume!

docs/style-guide.md

@@ -61,6 +61,43 @@ Occasionally inline, two line source code comments are used within a function.
 These should usually be balanced so that you don't have one line with 78
 characters and the second with only four. Split the comment between the two.
 
+### Default values
+
+Whenever a constant or function parameter is defined, try and have the safer or
+default value be the `zero` value. For example, instead of `const NoDanger`, use
+`const AllowDanger` so that the `false` value is the safe scenario.
+
+### Method receiver pointers
+
+You almost always want any method receivers to be declared on the pointer to the
+struct. There are only a few rare situations where this is not the case. This
+makes it easier to merge future changes that mutate the state without wondering
+why you now have two different copies of a struct. When you do need to copy a
+a struct, you can add a `Copy()` method to it. It's true that in many situations
+adding the pointer adds a small performance penalty, but we haven't found them
+to be significant in practice. If you do have a performance sensitive patch
+which benefits from skipping the pointer, please demonstrate this need with
+data first.
+
+#### Example
+
+```golang
+type Foo struct {
+	Whatever string
+	// ...
+}
+
+// Bar is implemented correctly as a pointer on Foo.
+func (obj *Foo) Bar(baz string) int {
+	// ...
+}
+
+// Bar is implemented *incorrectly* without a pointer to Foo.
+func (obj Foo) Bar(baz string) int {
+	// ...
+}
+```
+
 ### Method receiver naming
 
 [Contrary](https://github.com/golang/go/wiki/CodeReviewComments#receiver-names)
@@ -84,6 +121,75 @@ func (obj *Foo) Bar(baz string) int {
 }
 ```
 
+### Variable naming
+
+We prefer shorter, scoped variables rather than `unnecessarilyLongIdentifiers`.
+Remember the scoping rules and feel free to use new variables where appropriate.
+For example, in a short string snippet you can use `s` instead of `myString`, as
+well as other common choices. `i` is a common `int` counter, `f` for files, `fn`
+for functions, `x` for something else and so on.
+
+### Variable re-use
+
+Feel free to create and use new variables instead of attempting to re-use the
+same string. For example, if a function input arg is named `s`, you can use a
+new variable to receive the first computation result on `s` instead of storing
+it back into the original `s`. This avoids confusion if a different part of the
+code wants to read the original input, and it avoids any chance of edit by
+reference of the original callers copy of the variable.
+
+#### Example
+
+```golang
+MyNotIdealFunc(s string, b bool) string {
+	if !b {
+		return s + "hey"
+	}
+	s = strings.Replace(s, "blah", "", -1) // not ideal (re-use of `s` var)
+	return s
+}
+
+MyOkayFunc(s string, b bool) string {
+	if !b {
+		return s + "hey"
+	}
+	s2 := strings.Replace(s, "blah", "", -1) // doesn't re-use `s` variable
+	return s2
+}
+
+MyGreatFunc(s string, b bool) string {
+	if !b {
+		return s + "hey"
+	}
+	return strings.Replace(s, "blah", "", -1) // even cleaner
+}
+```
+
+### Constants in code
+
+If a function takes a specifier (often a bool) it's sometimes better to name
+that variable (often with a `const`) rather than leaving a naked `bool` in the
+code. For example, `x := MyFoo("blah", false)` is less clear than
+`const useMagic = false; x := MyFoo("blah", useMagic)`.
+
+### Empty slice declarations
+
+When declaring a new empty slice, there are three different mechanisms:
+
+1. `a := []string{}`
+
+2. `var a []string`
+
+3. `a := make([]string, 0)`
+
+In general, we prefer the first method because we find that it is succinct, and
+very readable. The third method is the least recommended because you're adding
+extra data that a smart compiler could probably figure out on its own. There are
+performance implications between these three methods, so unless your code is in
+a fast path or memory constrained environment where this matters (and that you
+ideally have proof of this) please use the methods as ordered as much as
+possible.
+
 ### Consistent ordering
 
 In general we try to preserve a logical ordering in source files which usually
@@ -96,6 +202,23 @@ declared in the interface.
 When implementing code for the various types in the language, please follow this
 order: `bool`, `str`, `int`, `float`, `list`, `map`, `struct`, `func`.
 
+For other aspects where you have a set of items, try to be internally consistent
+as well. For example, if you have two switch statements with `A`, `B`, and `C`,
+please use the same ordering for these elements elsewhere that they appear in
+the code and in the commentary if it is not illogical to do so.
+
+### Product identifiers
+
+Try to avoid references in the code to `mgmt` or a specific program name string
+if possible. This makes it easier to rename code if we ever pick a better name
+or support `libmgmt` better if we embed it. You can use the `Program` variable
+which is available in numerous places if you want a string to put in the logs.
+
+It is also recommended to avoid the `go` (programming language name) string if
+possible. Try to use `golang` if required, since the word `go` is already
+overloaded, and in particular it was even already used by the
+[`go!`](https://en.wikipedia.org/wiki/Go!_(programming_language)).
+
 ## Overview for mcl code
 
 The `mcl` language is quite new, so this guide will probably change over time as

docs/util/metadata.go

@@ -0,0 +1,103 @@
+// Mgmt
+// Copyright (C) James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+// Package util handles metadata for documentation generation.
+package util
+
+import (
+	"fmt"
+)
+
+var (
+	registeredResourceMetadata = make(map[string]*Metadata) // must initialize
+	registeredFunctionMetadata = make(map[string]*Metadata) // must initialize
+)
+
+// RegisterResource records the metadata for a resource of this kind.
+func RegisterResource(kind string, metadata *Metadata) error {
+	if _, exists := registeredResourceMetadata[kind]; exists {
+		return fmt.Errorf("metadata kind %s is already registered", kind)
+	}
+
+	registeredResourceMetadata[kind] = metadata
+
+	return nil
+}
+
+// LookupResource looks up the metadata for a resource of this kind.
+func LookupResource(kind string) (*Metadata, error) {
+	metadata, exists := registeredResourceMetadata[kind]
+	if !exists {
+		return nil, fmt.Errorf("not found")
+	}
+	return metadata, nil
+}
+
+// RegisterFunction records the metadata for a function of this name.
+func RegisterFunction(name string, metadata *Metadata) error {
+	if _, exists := registeredFunctionMetadata[name]; exists {
+		return fmt.Errorf("metadata named %s is already registered", name)
+	}
+
+	registeredFunctionMetadata[name] = metadata
+
+	return nil
+}
+
+// LookupFunction looks up the metadata for a function of this name.
+func LookupFunction(name string) (*Metadata, error) {
+	metadata, exists := registeredFunctionMetadata[name]
+	if !exists {
+		return nil, fmt.Errorf("not found")
+	}
+	return metadata, nil
+}
+
+// Metadata stores some additional information about the function or resource.
+// This is used to automatically generate documentation.
+type Metadata struct {
+	// Filename is the filename (without any base dir path) that this is in.
+	Filename string
+
+	// Typename is the string name of the main resource struct or function.
+	Typename string
+}
+
+// GetMetadata returns some metadata about the func. It can be called at any
+// time. This must not be named the same as the struct it's on or using it as an
+// anonymous embedded struct will stop us from being able to call this method.
+func (obj *Metadata) GetMetadata() *Metadata {
+	//if obj == nil { // TODO: Do I need this?
+	//	return nil
+	//}
+	return &Metadata{
+		Filename: obj.Filename,
+		Typename: obj.Typename,
+	}
+}

engine/autoedge.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -13,7 +13,19 @@
 // GNU General Public License for more details.
 //
 // You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
 
 package engine
 
@@ -68,7 +80,8 @@ type AutoEdge interface {
 	Test([]bool) bool // call until false
 }
 
-// ResUID is a unique identifier for a resource, namely it's name, and the kind ("type").
+// ResUID is a unique identifier for a resource, namely it's name, and the kind
+// ("type").
 type ResUID interface {
 	fmt.Stringer // String() string
 
@@ -104,9 +117,9 @@ func (obj *BaseUID) String() string {
 }
 
 // IFF looks at two UID's and if and only if they are equivalent, returns true.
-// If they are not equivalent, it returns false.
-// Most resources will want to override this method, since it does the important
-// work of actually discerning if two resources are identical in function.
+// If they are not equivalent, it returns false. Most resources will want to
+// override this method, since it does the important work of actually discerning
+// if two resources are identical in function.
 func (obj *BaseUID) IFF(uid ResUID) bool {
 	res, ok := uid.(*BaseUID)
 	if !ok {

engine/autoedge_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -13,9 +13,21 @@
 // GNU General Public License for more details.
 //
 // You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
 
-// +build !root
+//go:build !root
 
 package engine
 

engine/autogroup.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -13,7 +13,19 @@
 // GNU General Public License for more details.
 //
 // You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
 
 package engine
 
@@ -39,10 +51,11 @@ type GroupableRes interface {
 	SetAutoGroupMeta(*AutoGroupMeta)
 
 	// GroupCmp compares two resources and decides if they're suitable for
-	//grouping. This usually needs to be unique to your resource.
+	// grouping. This usually needs to be unique to your resource.
 	GroupCmp(res GroupableRes) error
 
-	// GroupRes groups resource argument (res) into self.
+	// GroupRes groups resource argument (res) into self. Callers of this
+	// method should probably also run SetParent.
 	GroupRes(res GroupableRes) error
 
 	// IsGrouped determines if we are grouped.
@@ -54,8 +67,15 @@ type GroupableRes interface {
 	// GetGroup returns everyone grouped inside me.
 	GetGroup() []GroupableRes // return everyone grouped inside me
 
-	// SetGroup sets the grouped resources into me.
+	// SetGroup sets the grouped resources into me. Callers of this method
+	// should probably also run SetParent.
 	SetGroup([]GroupableRes)
+
+	// Parent returns the parent groupable resource that I am inside of.
+	Parent() GroupableRes
+
+	// SetParent tells a particular grouped resource who their parent is.
+	SetParent(res GroupableRes)
 }
 
 // AutoGroupMeta provides some parameters specific to automatic grouping.

engine/cmp.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -13,7 +13,19 @@
 // GNU General Public License for more details.
 //
 // You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
 
 package engine
 
@@ -21,6 +33,7 @@ import (
 	"fmt"
 
 	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 // ResCmp compares two resources by checking multiple aspects. This is the main
@@ -106,6 +119,7 @@ func ResCmp(r1, r2 Res) error {
 			for k := range ix {
 				// compare sub resources
 				if err := ResCmp(ix[k], jx[k]); err != nil {
+					//fmt.Printf("bad Cmp: %+v <> %+v for: %+v <> %+v err: %+v\n", r1, r2, ix[k], jx[k], err)
 					return err
 				}
 			}
@@ -121,14 +135,29 @@ func ResCmp(r1, r2 Res) error {
 		v1 := r1r.Recv()
 		v2 := r2r.Recv()
 
+		// XXX: Our Send/Recv in the lib/main.go doesn't seem to be
+		// pulling this in, so this always compares differently. We can
+		// comment it out for now, since it's not too consequential.
+		// XXX: Find out what the issue is and fix it for here and send.
+		// XXX: The below errors are commented out until this is fixed.
 		if (v1 == nil) != (v2 == nil) { // xor
-			return fmt.Errorf("recv params differ")
+			//return fmt.Errorf("recv params differ")
 		}
 		if v1 != nil && v2 != nil {
-			// TODO: until we hit this code path, don't allow
-			// comparing anything that has this set to non-zero
-			if len(v1) != 0 || len(v2) != 0 {
-				return fmt.Errorf("recv params exist")
+			if len(v1) != len(v2) {
+				//return fmt.Errorf("recv param lengths differ")
+			}
+			for key, send1 := range v1 { // map[string]*engine.Send
+				send2, exists := v2[key]
+				if !exists {
+					//return fmt.Errorf("recv param key %s doesn't exist", key)
+				}
+				if (send1 == nil) != (send2 == nil) { // xor
+					//return fmt.Errorf("recv param key %s send differs", key)
+				}
+				if send1 != nil && send2 != nil && send1.Key != send2.Key {
+					//return fmt.Errorf("recv param key %s send key differs (%v != %v)", key, send1.Key, send2.Key)
+				}
 			}
 		}
 	}
@@ -142,13 +171,29 @@ func ResCmp(r1, r2 Res) error {
 		s1 := r1s.Sent()
 		s2 := r2s.Sent()
 
+		// XXX: Our Send/Recv in the lib/main.go doesn't seem to be
+		// pulling this in, so this always compares differently. We can
+		// comment it out for now, since it's not too consequential.
+		// XXX: Find out what the issue is and fix it for here and recv.
+		// XXX: The below errors are commented out until this is fixed.
 		if (s1 == nil) != (s2 == nil) { // xor
-			return fmt.Errorf("send params differ")
+			//return fmt.Errorf("send params differ")
 		}
 		if s1 != nil && s2 != nil {
-			// TODO: until we hit this code path, don't allow
-			// adapting anything that has this set to non-nil
-			return fmt.Errorf("send params exist")
+			// TODO: reflect.DeepEqual?
+			//return fmt.Errorf("send params exist")
+		}
+	}
+
+	// compare meta params for resources with reversible traits
+	r1v, ok1 := r1.(ReversibleRes)
+	r2v, ok2 := r2.(ReversibleRes)
+	if ok1 != ok2 {
+		return fmt.Errorf("reversible differs") // they must be different (optional)
+	}
+	if ok1 && ok2 {
+		if r1v.ReversibleMeta().Cmp(r2v.ReversibleMeta()) != nil {
+			return fmt.Errorf("reversible differs")
 		}
 	}
 
@@ -249,14 +294,29 @@ func AdaptCmp(r1, r2 CompatibleRes) error {
 		v1 := r1r.Recv()
 		v2 := r2r.Recv()
 
+		// XXX: Our Send/Recv in the lib/main.go doesn't seem to be
+		// pulling this in, so this always compares differently. We can
+		// comment it out for now, since it's not too consequential.
+		// XXX: Find out what the issue is and fix it for here and send.
+		// XXX: The below errors are commented out until this is fixed.
 		if (v1 == nil) != (v2 == nil) { // xor
-			return fmt.Errorf("recv params differ")
+			//return fmt.Errorf("recv params differ")
 		}
 		if v1 != nil && v2 != nil {
-			// TODO: until we hit this code path, don't allow
-			// adapting anything that has this set to non-zero
-			if len(v1) != 0 || len(v2) != 0 {
-				return fmt.Errorf("recv params exist")
+			if len(v1) != len(v2) {
+				//return fmt.Errorf("recv param lengths differ")
+			}
+			for key, send1 := range v1 { // map[string]*engine.Send
+				send2, exists := v2[key]
+				if !exists {
+					//return fmt.Errorf("recv param key %s doesn't exist", key)
+				}
+				if (send1 == nil) != (send2 == nil) { // xor
+					//return fmt.Errorf("recv param key %s send differs", key)
+				}
+				if send1 != nil && send2 != nil && send1.Key != send2.Key {
+					//return fmt.Errorf("recv param key %s send key differs (%v != %v)", key, send1.Key, send2.Key)
+				}
 			}
 		}
 	}
@@ -270,13 +330,29 @@ func AdaptCmp(r1, r2 CompatibleRes) error {
 		s1 := r1s.Sent()
 		s2 := r2s.Sent()
 
+		// XXX: Our Send/Recv in the lib/main.go doesn't seem to be
+		// pulling this in, so this always compares differently. We can
+		// comment it out for now, since it's not too consequential.
+		// XXX: Find out what the issue is and fix it for here and recv.
+		// XXX: The below errors are commented out until this is fixed.
 		if (s1 == nil) != (s2 == nil) { // xor
-			return fmt.Errorf("send params differ")
+			//return fmt.Errorf("send params differ")
 		}
 		if s1 != nil && s2 != nil {
-			// TODO: until we hit this code path, don't allow
-			// adapting anything that has this set to non-nil
-			return fmt.Errorf("send params exist")
+			// TODO: reflect.DeepEqual?
+			//return fmt.Errorf("send params exist")
+		}
+	}
+
+	// compare meta params for resources with reversible traits
+	r1v, ok1 := r1.(ReversibleRes)
+	r2v, ok2 := r2.(ReversibleRes)
+	if ok1 != ok2 {
+		return fmt.Errorf("reversible differs") // they must be different (optional)
+	}
+	if ok1 && ok2 {
+		if r1v.ReversibleMeta().Cmp(r2v.ReversibleMeta()) != nil {
+			return fmt.Errorf("reversible differs")
 		}
 	}
 
@@ -296,9 +372,11 @@ func VertexCmpFn(v1, v2 pgraph.Vertex) (bool, error) {
 		return false, fmt.Errorf("v2 is not a Res")
 	}
 
-	if ResCmp(r1, r2) != nil {
+	if err := ResCmp(r1, r2); err != nil {
+		//fmt.Printf("bad Cmp: %p %+v <> %p %+v err: %+v\n", r1, r1, r2, r2, err)
 		return false, nil
 	}
+	//fmt.Printf("ok Cmp: %p %+v <> %p %+v\n", r1, r1, r2, r2)
 
 	return true, nil
 }
@@ -317,3 +395,55 @@ func EdgeCmpFn(e1, e2 pgraph.Edge) (bool, error) {
 	}
 	return edge1.Cmp(edge2) == nil, nil
 }
+
+// ResGraphMapper compares two graphs, and gives us a mapping from new to old
+// based on the resource kind and name only. This allows us to know which
+// previous resource might have data to pass on to the new version in the next
+// generation.
+// FIXME: Optimize this for performance since it runs a lot...
+func ResGraphMapper(oldGraph, newGraph *pgraph.Graph) (map[RecvableRes]RecvableRes, error) {
+	mapper := make(map[RecvableRes]RecvableRes) // new -> old based on name and kind only?
+	cmp := func(r1, r2 Res) error {
+		if r1.Kind() != r2.Kind() {
+			return fmt.Errorf("kind differs")
+		}
+		if r1.Name() != r2.Name() {
+			return fmt.Errorf("name differs")
+		}
+		return nil
+	}
+
+	// XXX: run this as a topological sort or reverse topological sort?
+	for v := range newGraph.Adjacency() { // loop through the vertices (resources)
+		r, ok := v.(RecvableRes)
+		if !ok {
+			continue // skip
+		}
+		fn := func(vv pgraph.Vertex) (bool, error) {
+			rr, ok := vv.(Res)
+			if !ok {
+				return false, fmt.Errorf("not a Res")
+			}
+
+			if err := cmp(rr, r); err != nil {
+				return false, nil
+			}
+			return true, nil
+		}
+		vertex, err := oldGraph.VertexMatchFn(fn)
+		if err != nil {
+			return nil, errwrap.Wrapf(err, "VertexMatchFn failed")
+		}
+		if vertex == nil {
+			continue // skip (error?)
+		}
+		res, ok := vertex.(RecvableRes)
+		if !ok {
+			continue // skip (error?)
+		}
+
+		mapper[r] = res
+	}
+
+	return mapper, nil
+}

engine/copy.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -13,14 +13,26 @@
 // GNU General Public License for more details.
 //
 // You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
 
 package engine
 
 import (
 	"fmt"
 
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 // ResCopy copies a resource. This is the main entry point for copying a
@@ -81,6 +93,7 @@ func ResCopy(r CopyableRes) (CopyableRes, error) {
 			if !ok {
 				return nil, fmt.Errorf("resource wasn't groupable")
 			}
+			g2.SetParent(dst) // store who my parent is
 			grouped = append(grouped, g2)
 		}
 		dst.SetGroup(grouped)
@@ -106,6 +119,16 @@ func ResCopy(r CopyableRes) (CopyableRes, error) {
 		}
 	}
 
+	// copy meta params for resources with reversible traits
+	if x, ok := r.(ReversibleRes); ok {
+		dst, ok := res.(ReversibleRes)
+		if !ok {
+			// programming error
+			panic("reversible interfaces are illogical")
+		}
+		dst.SetReversibleMeta(x.ReversibleMeta()) // no need to copy atm
+	}
+
 	return res, nil
 }
 

engine/doc.go

@@ -0,0 +1,33 @@
+// Mgmt
+// Copyright (C) James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+// Package engine represents the implementation of the resource engine that runs
+// the graph of resources in real-time. This package has the common imports that
+// most consumers use directly.
+package engine

engine/edge.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -13,7 +13,19 @@
 // GNU General Public License for more details.
 //
 // You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
 
 package engine
 

engine/error.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -13,7 +13,19 @@
 // GNU General Public License for more details.
 //
 // You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
 
 package engine
 
@@ -24,9 +36,9 @@ type Error string
 func (e Error) Error() string { return string(e) }
 
 const (
-	// ErrWatchExit represents an exit from the Watch loop via chan closure.
-	ErrWatchExit = Error("watch exit")
+	// ErrClosed means we couldn't complete a task because we had closed.
+	ErrClosed = Error("closed")
 
-	// ErrSignalExit represents an exit from the Watch loop via exit signal.
-	ErrSignalExit = Error("signal exit")
+	// ErrBackPoke means we're postponing due to a needed backpoke.
+	ErrBackPoke = Error("backpoke")
 )

engine/event/event.go

@@ -1,83 +0,0 @@
-// Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
-// Written by James Shubin <james@shubin.ca> and the project contributors
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-// Package event provides some primitives that are used for message passing.
-package event
-
-//go:generate stringer -type=Kind -output=kind_stringer.go
-
-// Kind represents the type of event being passed.
-type Kind int
-
-// The different event kinds are used in different contexts.
-const (
-	KindNil Kind = iota
-	KindStart
-	KindPause
-	KindPoke
-	KindExit
-)
-
-// Pre-built messages so they can be used directly without having to use NewMsg.
-// These are useful when we don't want a response via ACK().
-var (
-	Start = &Msg{Kind: KindStart}
-	Pause = &Msg{Kind: KindPause} // probably unused b/c we want a resp
-	Poke  = &Msg{Kind: KindPoke}
-	Exit  = &Msg{Kind: KindExit}
-)
-
-// Msg is an event primitive that represents a kind of event, and optionally a
-// request for an ACK.
-type Msg struct {
-	Kind Kind
-
-	resp chan struct{}
-}
-
-// NewMsg builds a new message struct. It will want an ACK. If you don't want an
-// ACK then use the pre-built messages in the package variable globals.
-func NewMsg(kind Kind) *Msg {
-	return &Msg{
-		Kind: kind,
-		resp: make(chan struct{}),
-	}
-}
-
-// CanACK determines if an ACK is possible for this message. It does not say
-// whether one has already been sent or not.
-func (obj *Msg) CanACK() bool {
-	return obj.resp != nil
-}
-
-// ACK acknowledges the event. It must not be called more than once for the same
-// event. It unblocks the past and future calls of Wait for this event.
-func (obj *Msg) ACK() {
-	close(obj.resp)
-}
-
-// Wait on ACK for this event. It doesn't matter if this runs before or after
-// the ACK. It will unblock either way.
-// TODO: consider adding a context if it's ever useful.
-func (obj *Msg) Wait() error {
-	select {
-	//case <-ctx.Done():
-	//	return ctx.Err()
-	case <-obj.resp:
-		return nil
-	}
-}

engine/fs.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -13,7 +13,19 @@
 // GNU General Public License for more details.
 //
 // You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
 
 package engine
 
@@ -23,39 +35,64 @@ import (
 	"github.com/spf13/afero"
 )
 
-// from the ioutil package:
-// NopCloser(r io.Reader) io.ReadCloser // not implemented here
-// ReadAll(r io.Reader) ([]byte, error)
-// ReadDir(dirname string) ([]os.FileInfo, error)
-// ReadFile(filename string) ([]byte, error)
-// TempDir(dir, prefix string) (name string, err error)
-// TempFile(dir, prefix string) (f *os.File, err error) // slightly different here
-// WriteFile(filename string, data []byte, perm os.FileMode) error
-
-// Fs is an interface that represents this file system API that we support.
-// TODO: this should be in the gapi package or elsewhere.
+// Fs is an interface that represents the file system API that we support.
+// TODO: rename this to FS for consistency with the io/fs.FS naming scheme
 type Fs interface {
 	//fmt.Stringer // TODO: add this method?
-	afero.Fs     // TODO: why doesn't this interface exist in the os pkg?
+
+	// URI returns a unique string handle to access this filesystem.
 	URI() string // returns the URI for this file system
 
-	//DirExists(path string) (bool, error)
-	//Exists(path string) (bool, error)
-	//FileContainsAnyBytes(filename string, subslices [][]byte) (bool, error)
-	//FileContainsBytes(filename string, subslice []byte) (bool, error)
-	//FullBaseFsPath(basePathFs *BasePathFs, relativePath string) string
-	//GetTempDir(subPath string) string
-	//IsDir(path string) (bool, error)
-	//IsEmpty(path string) (bool, error)
-	//NeuterAccents(s string) string
-	//ReadAll(r io.Reader) ([]byte, error) // not needed, same as ioutil
+	afero.Fs // TODO: why doesn't this interface exist in the os pkg?
+
+	// FS is the read-only filesystem interface from the io/fs.FS package.
+	//fs.FS // io/fs.FS
+
+	// ReadDir reads the named directory and returns a list of directory
+	// entries sorted by filename.
+	//
+	// This mimics the signature from io/fs.ReadDirFS and has the same docs.
+	//
+	// XXX: Not currently implemented because of legacy Afero.Fs above
+	//ReadDir(name string) ([]fs.DirEntry, error) // io/fs.ReadDirFS
+
+	// ReadFile reads the named file and returns its contents. A successful
+	// call returns a nil error, not io.EOF. (Because ReadFile reads the
+	// whole file, the expected EOF from the final Read is not treated as an
+	// error to be reported.)
+	//
+	// The caller is permitted to modify the returned byte slice. This
+	// method should return a copy of the underlying data.
+	//
+	// This mimics the signature from io/fs.ReadFileFS and has the same
+	// docs.
+	ReadFile(name string) ([]byte, error) // io/fs.ReadFileFS
+
+	// Stat returns a FileInfo describing the file. If there is an error, it
+	// should be of type *fs.PathError.
+	//
+	// This mimics the signature from io/fs.StatFS and has the same docs.
+	//
+	// XXX: Not currently implemented because of legacy Afero.Fs above
+	//Stat(name string) (FileInfo, error) // io/fs.StatFS
+
+	// afero.Fs versions:
+
 	ReadDir(dirname string) ([]os.FileInfo, error)
-	ReadFile(filename string) ([]byte, error)
-	//SafeWriteReader(path string, r io.Reader) (err error)
-	TempDir(dir, prefix string) (name string, err error)
-	TempFile(dir, prefix string) (f afero.File, err error) // slightly different from upstream
-	//UnicodeSanitize(s string) string
-	//Walk(root string, walkFn filepath.WalkFunc) error
-	WriteFile(filename string, data []byte, perm os.FileMode) error
-	//WriteReader(path string, r io.Reader) (err error)
+}
+
+// WriteableFS is our internal filesystem interface for filesystems we write to.
+// It can wrap whatever implementations we want.
+type WriteableFS interface {
+	Fs
+
+	// WriteFile writes data to the named file, creating it if necessary. If
+	// the file does not exist, WriteFile creates it with permissions perm
+	// (before umask); otherwise WriteFile truncates it before writing,
+	// without changing permissions. Since Writefile requires multiple
+	// system calls to complete, a failure mid-operation can leave the file
+	// in a partially written state.
+	//
+	// This mimics the internal os.WriteFile function and has the same docs.
+	WriteFile(name string, data []byte, perm os.FileMode) error
 }

engine/graph/actions.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -13,22 +13,34 @@
 // GNU General Public License for more details.
 //
 // You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
 
 package graph
 
 import (
+	"context"
 	"fmt"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/purpleidea/mgmt/engine"
-	"github.com/purpleidea/mgmt/engine/event"
+	engineUtil "github.com/purpleidea/mgmt/engine/util"
 	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/util/errwrap"
 
-	//multierr "github.com/hashicorp/go-multierror"
-	errwrap "github.com/pkg/errors"
 	"golang.org/x/time/rate"
 )
 
@@ -41,14 +53,18 @@ func (obj *Engine) OKTimestamp(vertex pgraph.Vertex) bool {
 // be bad.
 func (obj *Engine) BadTimestamps(vertex pgraph.Vertex) []pgraph.Vertex {
 	vs := []pgraph.Vertex{}
-	ts := obj.state[vertex].timestamp
+	obj.state[vertex].mutex.RLock()   // concurrent read start
+	ts := obj.state[vertex].timestamp // race
+	obj.state[vertex].mutex.RUnlock() // concurrent read end
 	// these are all the vertices pointing TO vertex, eg: ??? -> vertex
 	for _, v := range obj.graph.IncomingGraphVertices(vertex) {
 		// If the vertex has a greater timestamp than any prerequisite,
 		// then we can't run right now. If they're equal (eg: initially
 		// with a value of 0) then we also can't run because we should
 		// let our pre-requisites go first.
-		t := obj.state[v].timestamp
+		obj.state[v].mutex.RLock()   // concurrent read start
+		t := obj.state[v].timestamp  // race
+		obj.state[v].mutex.RUnlock() // concurrent read end
 		if obj.Debug {
 			obj.Logf("OKTimestamp: %d >= %d (%s): !%t", ts, t, v.String(), ts >= t)
 		}
@@ -61,33 +77,33 @@ func (obj *Engine) BadTimestamps(vertex pgraph.Vertex) []pgraph.Vertex {
 }
 
 // Process is the primary function to execute a particular vertex in the graph.
-func (obj *Engine) Process(vertex pgraph.Vertex) error {
+func (obj *Engine) Process(ctx context.Context, vertex pgraph.Vertex) error {
 	res, isRes := vertex.(engine.Res)
 	if !isRes {
 		return fmt.Errorf("vertex is not a Res")
 	}
 
-	// Engine Guarantee: Do not allow CheckApply to run while we are paused.
-	// This makes the resource able to know that synchronous channel sending
-	// to the main loop select in Watch from within CheckApply, will succeed
-	// without blocking because the resource went into a paused state. If we
-	// are using the Poll metaparam, then Watch will (of course) not be run.
-	// FIXME: should this lock be here, or wrapped right around CheckApply ?
-	obj.state[vertex].eventsLock.Lock() // this lock is taken within Event()
-	defer obj.state[vertex].eventsLock.Unlock()
-
 	// backpoke! (can be async)
 	if vs := obj.BadTimestamps(vertex); len(vs) > 0 {
 		// back poke in parallel (sync b/c of waitgroup)
+		wg := &sync.WaitGroup{}
 		for _, v := range obj.graph.IncomingGraphVertices(vertex) {
 			if !pgraph.VertexContains(v, vs) { // only poke what's needed
 				continue
 			}
 
-			go obj.state[v].Poke() // async
+			// doesn't really need to be in parallel, but we can...
+			wg.Add(1)
+			go func(vv pgraph.Vertex) {
+				defer wg.Done()
+				obj.state[vv].Poke() // async
+			}(v)
 
 		}
-		return nil // can't continue until timestamp is in sequence
+		wg.Wait()
+
+		// can't continue until timestamp is in sequence, defer for now
+		return engine.ErrBackPoke
 	}
 
 	// semaphores!
@@ -112,21 +128,41 @@ func (obj *Engine) Process(vertex pgraph.Vertex) error {
 
 	// sendrecv!
 	// connect any senders to receivers and detect if values changed
+	// this actually checks and sends into resource trees recursively...
 	if res, ok := vertex.(engine.RecvableRes); ok {
-		if updated, err := obj.SendRecv(res); err != nil {
+		if obj.Debug {
+			obj.Logf("SendRecv: %s", res) // receiving here
+		}
+		if updated, err := SendRecv(res, nil); err != nil {
 			return errwrap.Wrapf(err, "could not SendRecv")
 		} else if len(updated) > 0 {
-			for _, changed := range updated {
-				if changed { // at least one was updated
-					// invalidate cache, mark as dirty
-					obj.state[vertex].tuid.StopTimer()
-					obj.state[vertex].isStateOK = false
-					break
+			//for _, s := range graph.UpdatedStrings(updated) {
+			//	obj.Logf("SendRecv: %s", s)
+			//}
+			for r, m := range updated { // map[engine.RecvableRes]map[string]*engine.Send
+				v, ok := r.(pgraph.Vertex)
+				if !ok {
+					continue
+				}
+				_, stateExists := obj.state[v] // autogrouped children probably don't have a state
+				if !stateExists {
+					continue
+				}
+				for s, send := range m {
+					if !send.Changed {
+						continue
+					}
+					obj.Logf("Send/Recv: %v.%s -> %v.%s", send.Res, send.Key, r, s)
+					// if send.Changed == true, at least one was updated
+					// invalidate cache, mark as dirty
+					obj.state[v].setDirty()
+					//break // we might have more vertices now
+				}
+
+				// re-validate after we change any values
+				if err := engine.Validate(r); err != nil {
+					return errwrap.Wrapf(err, "failed Validate after SendRecv")
 				}
-			}
-			// re-validate after we change any values
-			if err := engine.Validate(res); err != nil {
-				return errwrap.Wrapf(err, "failed Validate after SendRecv")
 			}
 		}
 	}
@@ -148,18 +184,22 @@ func (obj *Engine) Process(vertex pgraph.Vertex) error {
 	// Check cached state, to skip CheckApply, but can't skip if refreshing!
 	// If the resource doesn't implement refresh, skip the refresh test.
 	// FIXME: if desired, check that we pass through refresh notifications!
-	if (!refresh || !isRefreshableRes) && obj.state[vertex].isStateOK {
+	if (!refresh || !isRefreshableRes) && obj.state[vertex].isStateOK.Load() { // mutex RLock/RUnlock
 		checkOK, err = true, nil
 
 	} else if noop && (refresh && isRefreshableRes) { // had a refresh to do w/ noop!
 		checkOK, err = false, nil // therefore the state is wrong
 
-		// run the CheckApply!
 	} else {
-		obj.Logf("%s: CheckApply(%t)", res, !noop)
+		// run the CheckApply!
+		if obj.Debug {
+			obj.Logf("%s: CheckApply(%t)", res, !noop)
+		}
 		// if this fails, don't UpdateTimestamp()
-		checkOK, err = res.CheckApply(!noop)
-		obj.Logf("%s: CheckApply(%t): Return(%t, %+v)", res, !noop, checkOK, err)
+		checkOK, err = res.CheckApply(ctx, !noop)
+		if !checkOK && obj.Debug { // don't log on (checkOK == true)
+			obj.Logf("%s: CheckApply(%t): Return(%t, %s)", res, !noop, checkOK, engineUtil.CleanError(err))
+		}
 	}
 
 	if checkOK && err != nil { // should never return this way
@@ -176,7 +216,9 @@ func (obj *Engine) Process(vertex pgraph.Vertex) error {
 	// if CheckApply ran without noop and without error, state should be good
 	if !noop && err == nil { // aka !noop || checkOK
 		obj.state[vertex].tuid.StartTimer()
-		obj.state[vertex].isStateOK = true // reset
+		//obj.state[vertex].mutex.Lock()
+		obj.state[vertex].isStateOK.Store(true) // reset
+		//obj.state[vertex].mutex.Unlock()
 		if refresh {
 			obj.SetUpstreamRefresh(vertex, false) // refresh happened, clear the request
 			if isRefreshableRes {
@@ -213,7 +255,9 @@ func (obj *Engine) Process(vertex pgraph.Vertex) error {
 		wg := &sync.WaitGroup{}
 		// update this timestamp *before* we poke or the poked
 		// nodes might fail due to having a too old timestamp!
-		obj.state[vertex].timestamp = time.Now().UnixNano() // update timestamp
+		obj.state[vertex].mutex.Lock()                      // concurrent write start
+		obj.state[vertex].timestamp = time.Now().UnixNano() // update timestamp (race)
+		obj.state[vertex].mutex.Unlock()                    // concurrent write end
 		for _, v := range obj.graph.OutgoingGraphVertices(vertex) {
 			if !obj.OKTimestamp(v) {
 				// there is at least another one that will poke this...
@@ -244,14 +288,32 @@ func (obj *Engine) Process(vertex pgraph.Vertex) error {
 
 // Worker is the common run frontend of the vertex. It handles all of the retry
 // and retry delay common code, and ultimately returns the final status of this
-// vertex execution.
+// vertex execution. This function cannot be "re-run" for the same vertex. The
+// retry mechanism stuff happens inside of this. To actually "re-run" you need
+// to remove the vertex and build a new one. The engine guarantees that we do
+// not allow CheckApply to run while we are paused. That is enforced here.
 func (obj *Engine) Worker(vertex pgraph.Vertex) error {
 	res, isRes := vertex.(engine.Res)
 	if !isRes {
 		return fmt.Errorf("vertex is not a resource")
 	}
 
-	defer close(obj.state[vertex].stopped) // done signal
+	// bonus safety check
+	if res.MetaParams().Burst == 0 && !(res.MetaParams().Limit == rate.Inf) { // blocked
+		return fmt.Errorf("permanently limited (rate != Inf, burst = 0)")
+	}
+
+	// initialize or reinitialize the meta state for this resource uid
+	obj.mlock.Lock()
+	if _, exists := obj.metas[engine.PtrUID(res)]; !exists || res.MetaParams().Reset {
+		obj.metas[engine.PtrUID(res)] = &engine.MetaState{
+			CheckApplyRetry: res.MetaParams().Retry, // lookup the retry value
+		}
+	}
+	metas := obj.metas[engine.PtrUID(res)] // handle
+	obj.mlock.Unlock()
+
+	//defer close(obj.state[vertex].stopped) // done signal
 
 	obj.state[vertex].cuid = obj.Converger.Register()
 	obj.state[vertex].tuid = obj.Converger.Register()
@@ -265,7 +327,29 @@ func (obj *Engine) Worker(vertex pgraph.Vertex) error {
 	obj.state[vertex].wg.Add(1)
 	go func() {
 		defer obj.state[vertex].wg.Done()
-		defer close(obj.state[vertex].outputChan) // we close this on behalf of res
+		defer close(obj.state[vertex].eventsChan) // we close this on behalf of res
+
+		// This is a close reverse-multiplexer. If any of the channels
+		// close, then it will cause the doneCtx to cancel. That way,
+		// multiple different folks can send a close signal, without
+		// every worrying about duplicate channel close panics.
+		obj.state[vertex].wg.Add(1)
+		go func() {
+			defer obj.state[vertex].wg.Done()
+
+			// reverse-multiplexer: any close, causes *the* close!
+			select {
+			case <-obj.state[vertex].processDone:
+			case <-obj.state[vertex].watchDone:
+			case <-obj.state[vertex].limitDone:
+			case <-obj.state[vertex].retryDone:
+			case <-obj.state[vertex].removeDone:
+			case <-obj.state[vertex].eventsDone:
+			}
+
+			// the main "done" signal gets activated here!
+			obj.state[vertex].doneCtxCancel() // cancels doneCtx
+		}()
 
 		var err error
 		var retry = res.MetaParams().Retry // lookup the retry value
@@ -283,13 +367,8 @@ func (obj *Engine) Worker(vertex pgraph.Vertex) error {
 						case <-timer.C: // the wait is over
 							return errDelayExpired // special
 
-						case event, ok := <-obj.state[vertex].init.Events:
-							if !ok {
-								return nil
-							}
-							if err := obj.state[vertex].init.Read(event); err != nil {
-								return err
-							}
+						case <-obj.state[vertex].doneCtx.Done():
+							return nil
 						}
 					}
 				}()
@@ -299,180 +378,312 @@ func (obj *Engine) Worker(vertex pgraph.Vertex) error {
 				}
 			} else if interval := res.MetaParams().Poll; interval > 0 { // poll instead of watching :(
 				obj.state[vertex].cuid.StartTimer()
-				err = obj.state[vertex].poll(interval)
+				err = obj.state[vertex].poll(obj.state[vertex].doneCtx, interval)
 				obj.state[vertex].cuid.StopTimer() // clean up nicely
 			} else {
 				obj.state[vertex].cuid.StartTimer()
-				obj.Logf("Watch(%s)", vertex)
-				err = res.Watch() // run the watch normally
-				obj.Logf("Watch(%s): Exited(%+v)", vertex, err)
+				if obj.Debug {
+					obj.Logf("%s: Watch...", vertex)
+				}
+				err = res.Watch(obj.state[vertex].doneCtx) // run the watch normally
+				if obj.Debug {
+					if s := engineUtil.CleanError(err); err != nil {
+						obj.Logf("%s: Watch Error: %s", vertex, s)
+					} else {
+						obj.Logf("%s: Watch Exited...", vertex)
+					}
+				}
 				obj.state[vertex].cuid.StopTimer() // clean up nicely
 			}
-			if err == nil || err == engine.ErrWatchExit || err == engine.ErrSignalExit {
+			if err == nil { // || err == engine.ErrClosed
 				return // exited cleanly, we're done
 			}
 			// we've got an error...
 			delay = res.MetaParams().Delay
 
 			if retry < 0 { // infinite retries
-				obj.state[vertex].reset()
 				continue
 			}
 			if retry > 0 { // don't decrement past 0
 				retry--
 				obj.state[vertex].init.Logf("retrying Watch after %.4f seconds (%d left)", float64(delay)/1000, retry)
-				obj.state[vertex].reset()
 				continue
 			}
 			//if retry == 0 { // optional
 			//	err = errwrap.Wrapf(err, "permanent watch error")
 			//}
 			break // break out of this and send the error
-		}
+		} // for retry loop
+
 		// this section sends an error...
 		// If the CheckApply loop exits and THEN the Watch fails with an
 		// error, then we'd be stuck here if exit signal didn't unblock!
 		select {
-		case obj.state[vertex].outputChan <- errwrap.Wrapf(err, "watch failed"):
+		case obj.state[vertex].eventsChan <- errwrap.Wrapf(err, "watch failed"):
 			// send
-		case <-obj.state[vertex].exit.Signal():
-			// pass
 		}
 	}()
 
-	// bonus safety check
-	if res.MetaParams().Burst == 0 && !(res.MetaParams().Limit == rate.Inf) { // blocked
-		return fmt.Errorf("permanently limited (rate != Inf, burst = 0)")
-	}
-	var limiter = rate.NewLimiter(res.MetaParams().Limit, res.MetaParams().Burst)
-	// It is important that we shutdown the Watch loop if this exits.
-	// Example, if Process errors permanently, we should ask Watch to exit.
-	defer obj.state[vertex].Event(event.Exit) // signal an exit
-	for {
+	// If this exits cleanly, we must unblock the reverse-multiplexer.
+	// I think this additional close is unnecessary, but it's not harmful.
+	defer close(obj.state[vertex].eventsDone) // causes doneCtx to cancel
+	limiter := rate.NewLimiter(res.MetaParams().Limit, res.MetaParams().Burst)
+	var reserv *rate.Reservation
+	var reterr error
+	var failed bool // has Process permanently failed?
+	var closed bool // has the resumeSignal channel closed?
+Loop:
+	for { // process loop
+		// This is the main select where things happen and where we exit
+		// from. It's similar to the two "satellite" select's which we
+		// might spend some time in if we're retrying or rate limiting.
+		// This select is also the main event receiver and is also the
+		// only place where we read from the poke channel.
 		select {
-		case err, ok := <-obj.state[vertex].outputChan: // read from watch channel
+		case err, ok := <-obj.state[vertex].eventsChan: // read from watch channel
 			if !ok {
-				return nil
+				return reterr // we only return when chan closes
 			}
+			// If the Watch method exits with an error, then this
+			// channel will get that error propagated to it, which
+			// we then save so we can return it to the caller of us.
 			if err != nil {
-				return err // permanent failure
+				failed = true
+				close(obj.state[vertex].watchDone)   // causes doneCtx to cancel
+				reterr = errwrap.Append(reterr, err) // permanent failure
+				continue
 			}
+			if obj.Debug {
+				obj.Logf("event received")
+			}
+			reserv = limiter.ReserveN(time.Now(), 1) // one event
+			// reserv.OK() seems to always be true here!
 
-			// safe to go run the process...
-		case <-obj.state[vertex].exit.Signal(): // TODO: is this needed?
-			return nil
+		case _, ok := <-obj.state[vertex].pokeChan: // read from buffered poke channel
+			if !ok { // we never close it
+				panic("unexpected close of poke channel")
+			}
+			if obj.Debug {
+				obj.Logf("poke received")
+			}
+			reserv = nil // we didn't receive a real event here...
+
+		case _, ok := <-obj.state[vertex].pauseSignal: // one message
+			if !ok {
+				obj.state[vertex].pauseSignal = nil
+				continue // this is not a new pause message
+			}
+			// NOTE: If we allowed a doneCtx below to let us out
+			// of the resumeSignal wait, then we could loop around
+			// and run this again, causing a panic. Instead of this
+			// being made safe with a sync.Once, we instead run a
+			// close() call inside of the vertexRemoveFn function,
+			// which should unblock resumeSignal so we can shutdown.
+
+			// we are paused now, and waiting for resume or exit...
+			select {
+			case _, ok := <-obj.state[vertex].resumeSignal: // channel closes
+				if !ok {
+					closed = true
+				}
+				// resumed!
+				// pass through to allow a Process to try to run
+				// TODO: consider adding this fast pause here...
+				//if obj.fastPause {
+				//	obj.Logf("fast pausing on resume")
+				//	continue
+				//}
+			}
 		}
 
-		now := time.Now()
-		r := limiter.ReserveN(now, 1) // one event
-		// r.OK() seems to always be true here!
-		d := r.DelayFrom(now)
-		if d > 0 { // delay
-			obj.state[vertex].init.Logf("limited (rate: %v/sec, burst: %d, next: %v)", res.MetaParams().Limit, res.MetaParams().Burst, d)
-			var count int
+		// drop redundant pokes
+		for len(obj.state[vertex].pokeChan) > 0 {
+			select {
+			case <-obj.state[vertex].pokeChan:
+			default:
+				// race, someone else read one!
+			}
+		}
+
+		// don't Process anymore if we've already failed or shutdown...
+		if failed || closed {
+			continue Loop
+		}
+
+		// limit delay
+		d := time.Duration(0)
+		if reserv != nil {
+			d = reserv.DelayFrom(time.Now())
+		}
+		if reserv != nil && d > 0 { // delay
+			obj.state[vertex].init.Logf("limited (rate: %v/sec, burst: %d, next: %dms)", res.MetaParams().Limit, res.MetaParams().Burst, d/time.Millisecond)
 			timer := time.NewTimer(time.Duration(d) * time.Millisecond)
 		LimitWait:
 			for {
+				// This "satellite" select doesn't need a poke
+				// channel because we're already in "event
+				// received" mode, and poke doesn't block.
 				select {
 				case <-timer.C: // the wait is over
 					break LimitWait
 
 				// consume other events while we're waiting...
-				case e, ok := <-obj.state[vertex].outputChan: // read from watch channel
+				case e, ok := <-obj.state[vertex].eventsChan: // read from watch channel
 					if !ok {
-						// FIXME: is this logic correct?
-						if count == 0 {
-							return nil
-						}
-						// loop, because we have
-						// the previous event to
-						// run process on first!
-						continue
+						return reterr // we only return when chan closes
 					}
 					if e != nil {
-						return e // permanent failure
+						failed = true
+						close(obj.state[vertex].limitDone) // causes doneCtx to cancel
+						reterr = errwrap.Append(reterr, e) // permanent failure
+						break LimitWait
 					}
-					count++                         // count the events...
+					if obj.Debug {
+						obj.Logf("event received in limit")
+					}
+					// TODO: does this get added in properly?
 					limiter.ReserveN(time.Now(), 1) // one event
+
+				// this pause/resume block is the same as the upper main one
+				case _, ok := <-obj.state[vertex].pauseSignal:
+					if !ok {
+						obj.state[vertex].pauseSignal = nil
+						break LimitWait
+					}
+					select {
+					case _, ok := <-obj.state[vertex].resumeSignal: // channel closes
+						if !ok {
+							closed = true
+						}
+						// resumed!
+					}
 				}
 			}
 			timer.Stop() // it's nice to cleanup
 			obj.state[vertex].init.Logf("rate limiting expired!")
 		}
+		// don't Process anymore if we've already failed or shutdown...
+		if failed || closed {
+			continue Loop
+		}
+		// end of limit delay
 
+		// retry...
 		var err error
-		var retry = res.MetaParams().Retry // lookup the retry value
+		//var retry = res.MetaParams().Retry // lookup the retry value
 		var delay uint64
-	Loop:
+	RetryLoop:
 		for { // retry loop
 			if delay > 0 {
-				var count int
 				timer := time.NewTimer(time.Duration(delay) * time.Millisecond)
 			RetryWait:
 				for {
+					// This "satellite" select doesn't need
+					// a poke channel because we're already
+					// in "event received" mode, and poke
+					// doesn't block.
 					select {
 					case <-timer.C: // the wait is over
 						break RetryWait
 
 					// consume other events while we're waiting...
-					case e, ok := <-obj.state[vertex].outputChan: // read from watch channel
+					case e, ok := <-obj.state[vertex].eventsChan: // read from watch channel
 						if !ok {
-							// FIXME: is this logic correct?
-							if count == 0 {
-								// last process error
-								return err
-							}
-							// loop, because we have
-							// the previous event to
-							// run process on first!
-							continue
+							return reterr // we only return when chan closes
 						}
 						if e != nil {
-							return e // permanent failure
+							failed = true
+							close(obj.state[vertex].retryDone) // causes doneCtx to cancel
+							reterr = errwrap.Append(reterr, e) // permanent failure
+							break RetryWait
 						}
-						count++                         // count the events...
+						if obj.Debug {
+							obj.Logf("event received in retry")
+						}
+						// TODO: does this get added in properly?
 						limiter.ReserveN(time.Now(), 1) // one event
+
+					// this pause/resume block is the same as the upper main one
+					case _, ok := <-obj.state[vertex].pauseSignal:
+						if !ok {
+							obj.state[vertex].pauseSignal = nil
+							break RetryWait
+						}
+						select {
+						case _, ok := <-obj.state[vertex].resumeSignal: // channel closes
+							if !ok {
+								closed = true
+							}
+							// resumed!
+						}
 					}
 				}
 				timer.Stop() // it's nice to cleanup
 				delay = 0    // reset
 				obj.state[vertex].init.Logf("the CheckApply delay expired!")
 			}
+			// don't Process anymore if we've already failed or shutdown...
+			if failed || closed {
+				continue Loop
+			}
 
 			if obj.Debug {
 				obj.Logf("Process(%s)", vertex)
 			}
-			err = obj.Process(vertex)
-			if obj.Debug {
-				obj.Logf("Process(%s): Return(%+v)", vertex, err)
+			backPoke := false
+			err = obj.Process(obj.state[vertex].doneCtx, vertex)
+			if err == engine.ErrBackPoke {
+				backPoke = true
+				err = nil // for future code safety
 			}
-			if err == nil {
-				break Loop
+			if obj.Debug && backPoke {
+				obj.Logf("Process(%s): BackPoke!", vertex)
+			}
+			if obj.Debug && !backPoke {
+				obj.Logf("Process(%s): Return(%s)", vertex, engineUtil.CleanError(err))
+			}
+			if err == nil && !backPoke && res.MetaParams().RetryReset { // reset it on success!
+				metas.CheckApplyRetry = res.MetaParams().Retry // lookup the retry value
+			}
+			if err == nil || backPoke {
+				break RetryLoop
 			}
 			// we've got an error...
 			delay = res.MetaParams().Delay
 
-			if retry < 0 { // infinite retries
+			if metas.CheckApplyRetry < 0 { // infinite retries
 				continue
 			}
-			if retry > 0 { // don't decrement past 0
-				retry--
-				obj.state[vertex].init.Logf("retrying CheckApply after %.4f seconds (%d left)", float64(delay)/1000, retry)
+			if metas.CheckApplyRetry > 0 { // don't decrement past 0
+				metas.CheckApplyRetry--
+				obj.state[vertex].init.Logf(
+					"retrying CheckApply after %.4f seconds (%d left)",
+					float64(delay)/1000,
+					metas.CheckApplyRetry,
+				)
 				continue
 			}
-			//if retry == 0 { // optional
+			//if metas.CheckApplyRetry == 0 { // optional
 			//	err = errwrap.Wrapf(err, "permanent process error")
 			//}
 
-			// If this exits, defer calls: obj.Event(event.Exit),
-			// which will cause the Watch loop to shutdown. Also,
-			// if the Watch loop shuts down, that will cause this
-			// Process loop to shut down. Also the graph sync can
-			// run an: obj.Event(event.Exit) which causes this to
-			// shutdown as well. Lastly, it is possible that more
-			// that one of these scenarios happens simultaneously.
-			return err
-		}
-	}
+			// It is important that we shutdown the Watch loop if
+			// this dies. If Process fails permanently, we ask it
+			// to exit right here... (It happens when we loop...)
+			failed = true
+			close(obj.state[vertex].processDone) // causes doneCtx to cancel
+			reterr = errwrap.Append(reterr, err) // permanent failure
+			continue
+
+		} // retry loop
+
+		// When this Process loop exits, it's because something has
+		// caused Watch() to shutdown (even if it's our permanent
+		// failure from Process), which caused this channel to close.
+		// On or more exit signals are possible, and more than one can
+		// happen simultaneously.
+
+	} // process loop
+
 	//return nil // unreachable
 }

engine/graph/autoedge.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -13,7 +13,19 @@
 // GNU General Public License for more details.
 //
 // You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
 
 package graph
 

engine/graph/autoedge/autoedge.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -13,7 +13,19 @@
 // GNU General Public License for more details.
 //
 // You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
 
 package autoedge
 
@@ -22,14 +34,12 @@ import (
 
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/pgraph"
-
-	multierr "github.com/hashicorp/go-multierror"
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 // AutoEdge adds the automatic edges to the graph.
 func AutoEdge(graph *pgraph.Graph, debug bool, logf func(format string, v ...interface{})) error {
-	logf("adding autoedges...")
+	logf("building...")
 
 	// initially get all of the autoedges to seek out all possible errors
 	var err error
@@ -49,11 +59,13 @@ func AutoEdge(graph *pgraph.Graph, debug bool, logf func(format string, v ...int
 	for _, res := range sorted { // for each vertexes autoedges
 		autoEdgeObj, e := res.AutoEdges()
 		if e != nil {
-			err = multierr.Append(err, e) // collect all errors
+			err = errwrap.Append(err, e) // collect all errors
 			continue
 		}
 		if autoEdgeObj == nil {
-			logf("no auto edges were found for: %s", res)
+			if debug {
+				logf("no auto edges were found for: %s", res)
+			}
 			continue // next vertex
 		}
 		autoEdgeObjMap[res] = autoEdgeObj // save for next loop
@@ -76,9 +88,9 @@ func AutoEdge(graph *pgraph.Graph, debug bool, logf func(format string, v ...int
 				break // inner loop
 			}
 			if debug {
-				logf("autoedge: UIDS:")
+				logf("UIDS:")
 				for i, u := range uids {
-					logf("autoedge: UID%d: %v", i, u)
+					logf("UID%d: %v", i, u)
 				}
 			}
 
@@ -91,6 +103,9 @@ func AutoEdge(graph *pgraph.Graph, debug bool, logf func(format string, v ...int
 			}
 		}
 	}
+
+	// It would be great to ensure we didn't add any graph cycles here, but
+	// instead of checking now, we'll move the check into the main loop.
 	return nil
 }
 
@@ -116,7 +131,7 @@ func addEdgesByMatchingUIDS(res engine.EdgeableRes, uids []engine.ResUID, graph
 				continue
 			}
 			if debug {
-				logf("autoedge: Match: %s with UID: %s", r, uid)
+				logf("match: %s with UID: %s", r, uid)
 			}
 			// we must match to an effective UID for the resource,
 			// that is to say, the name value of a res is a helpful
@@ -125,13 +140,13 @@ func addEdgesByMatchingUIDS(res engine.EdgeableRes, uids []engine.ResUID, graph
 			if UIDExistsInUIDs(uid, r.UIDs()) {
 				// add edge from: r -> res
 				if uid.IsReversed() {
-					txt := fmt.Sprintf("%s -> %s (autoedge)", r, res)
-					logf("autoedge: adding: %s", txt)
+					txt := fmt.Sprintf("%s -> %s", r, res)
+					logf("adding: %s", txt)
 					edge := &engine.Edge{Name: txt}
 					graph.AddEdge(r, res, edge)
 				} else { // edges go the "normal" way, eg: pkg resource
-					txt := fmt.Sprintf("%s -> %s (autoedge)", res, r)
-					logf("autoedge: adding: %s", txt)
+					txt := fmt.Sprintf("%s -> %s", res, r)
+					logf("adding: %s", txt)
 					edge := &engine.Edge{Name: txt}
 					graph.AddEdge(res, r, edge)
 				}

engine/graph/autogroup.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -13,7 +13,19 @@
 // GNU General Public License for more details.
 //
 // You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
 
 package graph
 
@@ -24,8 +36,7 @@ import (
 	"github.com/purpleidea/mgmt/engine/graph/autogroup"
 	"github.com/purpleidea/mgmt/pgraph"
 	"github.com/purpleidea/mgmt/util"
-
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 // AutoGroup runs the auto grouping on the loaded graph.
@@ -75,10 +86,10 @@ func (obj *wrappedGrouper) VertexCmp(v1, v2 pgraph.Vertex) error {
 		return fmt.Errorf("v2 is not a GroupableRes")
 	}
 
-	if r1.Kind() != r2.Kind() { // we must group similar kinds
-		// TODO: maybe future resources won't need this limitation?
-		return fmt.Errorf("the two resources aren't the same kind")
-	}
+	// Some resources of different kinds can now group together!
+	//if r1.Kind() != r2.Kind() { // we must group similar kinds
+	//	return fmt.Errorf("the two resources aren't the same kind")
+	//}
 	// someone doesn't want to group!
 	if r1.AutoGroupMeta().Disabled || r2.AutoGroupMeta().Disabled {
 		return fmt.Errorf("one of the autogroup flags is false")
@@ -110,6 +121,7 @@ func (obj *wrappedGrouper) VertexMerge(v1, v2 pgraph.Vertex) (v pgraph.Vertex, e
 	if err = r1.GroupRes(r2); err != nil { // GroupRes skips stupid groupings
 		return // return early on error
 	}
+	r2.SetParent(r1) // store who my parent is
 
 	// merging two resources into one should yield the sum of their semas
 	if semas := r2.MetaParams().Sema; len(semas) > 0 {

engine/graph/autogroup/autogroup.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -13,7 +13,19 @@
 // GNU General Public License for more details.
 //
 // You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
 
 package autogroup
 
@@ -22,8 +34,7 @@ import (
 
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/pgraph"
-
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 // AutoGroup is the mechanical auto group "runner" that runs the interface spec.
@@ -55,7 +66,7 @@ func AutoGroup(ag engine.AutoGrouper, g *pgraph.Graph, debug bool, logf func(for
 			logf("!VertexMerge for: %s into: %s", wStr, vStr)
 
 		} else { // success!
-			logf("success for: %s into: %s", wStr, vStr)
+			logf("%s into %s", wStr, vStr)
 			merged = true // woo
 		}
 
@@ -67,5 +78,8 @@ func AutoGroup(ag engine.AutoGrouper, g *pgraph.Graph, debug bool, logf func(for
 		}
 	}
 
+	// It would be great to ensure we didn't add any graph cycles here, but
+	// instead of checking now, we'll move the check into the main loop.
+
 	return nil
 }

engine/graph/autogroup/autogroup_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -13,13 +13,26 @@
 // GNU General Public License for more details.
 //
 // You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
 
-// +build !root
+//go:build !root
 
 package autogroup
 
 import (
+	"context"
 	"fmt"
 	"reflect"
 	"sort"
@@ -31,8 +44,7 @@ import (
 	"github.com/purpleidea/mgmt/engine/traits"
 	"github.com/purpleidea/mgmt/pgraph"
 	"github.com/purpleidea/mgmt/util"
-
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 func init() {
@@ -63,15 +75,15 @@ func (obj *NoopResTest) Init(init *engine.Init) error {
 	return nil
 }
 
-func (obj *NoopResTest) Close() error {
+func (obj *NoopResTest) Cleanup() error {
 	return nil
 }
 
-func (obj *NoopResTest) Watch() error {
+func (obj *NoopResTest) Watch(context.Context) error {
 	return nil // not needed
 }
 
-func (obj *NoopResTest) CheckApply(apply bool) (checkOK bool, err error) {
+func (obj *NoopResTest) CheckApply(ctx context.Context, apply bool) (checkOK bool, err error) {
 	return true, nil // state is always okay
 }
 
@@ -190,6 +202,8 @@ func (obj *testGrouper) VertexMerge(v1, v2 pgraph.Vertex) (v pgraph.Vertex, err
 	if err := r1.GroupRes(r2); err != nil { // group them first
 		return nil, err
 	}
+	r2.SetParent(r1) // store who my parent is
+
 	// HACK: update the name so it matches full list of self+grouped
 	res := v1.(engine.GroupableRes)
 	names := strings.Split(res.Name(), ",") // load in stored names
@@ -596,10 +610,12 @@ func TestPgraphGrouping11(t *testing.T) {
 	runGraphCmp(t, g1, g2)
 }
 
+/*
 // simple merge 1
 // a1   a2         a1,a2
 //   \ /     >>>     |     (arrows point downwards)
 //    b              b
+*/
 func TestPgraphGrouping12(t *testing.T) {
 	g1, _ := pgraph.NewGraph("g1") // original graph
 	{
@@ -621,10 +637,12 @@ func TestPgraphGrouping12(t *testing.T) {
 	runGraphCmp(t, g1, g2)
 }
 
+/*
 // simple merge 2
 //    b              b
 //   / \     >>>     |     (arrows point downwards)
 // a1   a2         a1,a2
+*/
 func TestPgraphGrouping13(t *testing.T) {
 	g1, _ := pgraph.NewGraph("g1") // original graph
 	{
@@ -646,10 +664,12 @@ func TestPgraphGrouping13(t *testing.T) {
 	runGraphCmp(t, g1, g2)
 }
 
+/*
 // triple merge
 // a1 a2  a3         a1,a2,a3
 //   \ | /     >>>       |      (arrows point downwards)
 //     b                 b
+*/
 func TestPgraphGrouping14(t *testing.T) {
 	g1, _ := pgraph.NewGraph("g1") // original graph
 	{
@@ -674,12 +694,14 @@ func TestPgraphGrouping14(t *testing.T) {
 	runGraphCmp(t, g1, g2)
 }
 
+/*
 // chain merge
 //    a1             a1
 //   /  \             |
 // b1    b2   >>>   b1,b2   (arrows point downwards)
 //   \  /             |
 //    c1             c1
+*/
 func TestPgraphGrouping15(t *testing.T) {
 	g1, _ := pgraph.NewGraph("g1") // original graph
 	{
@@ -709,6 +731,7 @@ func TestPgraphGrouping15(t *testing.T) {
 	runGraphCmp(t, g1, g2)
 }
 
+/*
 // re-attach 1 (outer)
 // technically the second possibility is valid too, depending on which order we
 // merge edges in, and if we don't filter out any unnecessary edges afterwards!
@@ -717,6 +740,7 @@ func TestPgraphGrouping15(t *testing.T) {
 // b1  /      >>>    b1     OR    b1  /   (arrows point downwards)
 //  | /               |            | /
 // c1                c1           c1
+*/
 func TestPgraphGrouping16(t *testing.T) {
 	g1, _ := pgraph.NewGraph("g1") // original graph
 	{
@@ -731,25 +755,40 @@ func TestPgraphGrouping16(t *testing.T) {
 		g1.AddEdge(b1, c1, e2)
 		g1.AddEdge(a2, c1, e3)
 	}
-	g2, _ := pgraph.NewGraph("g2") // expected result
+	//g2, _ := pgraph.NewGraph("g2") // expected result
+	//{
+	//	a := NewNoopResTest("a1,a2")
+	//	b1 := NewNoopResTest("b1")
+	//	c1 := NewNoopResTest("c1")
+	//	e1 := NE("e1,e3")
+	//	e2 := NE("e2,e3") // e3 gets "merged through" to BOTH edges!
+	//	g2.AddEdge(a, b1, e1)
+	//	g2.AddEdge(b1, c1, e2)
+	//}
+	//runGraphCmp(t, g1, g2)
+	g3, _ := pgraph.NewGraph("g3") // alternative expected result
 	{
 		a := NewNoopResTest("a1,a2")
 		b1 := NewNoopResTest("b1")
 		c1 := NewNoopResTest("c1")
-		e1 := NE("e1,e3")
-		e2 := NE("e2,e3") // e3 gets "merged through" to BOTH edges!
-		g2.AddEdge(a, b1, e1)
-		g2.AddEdge(b1, c1, e2)
+		e1 := NE("e1")
+		e2 := NE("e2")
+		e3 := NE("e3")
+		g3.AddEdge(a, b1, e1)
+		g3.AddEdge(b1, c1, e2)
+		g3.AddEdge(a, c1, e3)
 	}
-	runGraphCmp(t, g1, g2)
+	runGraphCmp(t, g1, g3)
 }
 
+/*
 // re-attach 2 (inner)
 // a1    b2          a1
 //  |   /             |
 // b1  /      >>>   b1,b2   (arrows point downwards)
 //  | /               |
 // c1                c1
+*/
 func TestPgraphGrouping17(t *testing.T) {
 	g1, _ := pgraph.NewGraph("g1") // original graph
 	{
@@ -777,13 +816,16 @@ func TestPgraphGrouping17(t *testing.T) {
 	runGraphCmp(t, g1, g2)
 }
 
+/*
 // re-attach 3 (double)
 // similar to "re-attach 1", technically there is a second possibility for this
-// a2   a1    b2         a1,a2
-//   \   |   /             |
-//    \ b1  /      >>>   b1,b2   (arrows point downwards)
-//     \ | /               |
-//      c1                c1
+// TODO: verify this second possibility manually
+// a2   a1    b2         a1,a2        a1,a2
+//   \   |   /             |            |   \
+//    \ b1  /      >>>   b1,b2   OR   b1,b2 /   (arrows point downwards)
+//     \ | /               |            |  /
+//      c1                c1           c1
+*/
 func TestPgraphGrouping18(t *testing.T) {
 	g1, _ := pgraph.NewGraph("g1") // original graph
 	{
@@ -801,23 +843,38 @@ func TestPgraphGrouping18(t *testing.T) {
 		g1.AddEdge(a2, c1, e3)
 		g1.AddEdge(b2, c1, e4)
 	}
-	g2, _ := pgraph.NewGraph("g2") // expected result
+	//g2, _ := pgraph.NewGraph("g2") // expected result
+	//{
+	//	a := NewNoopResTest("a1,a2")
+	//	b := NewNoopResTest("b1,b2")
+	//	c1 := NewNoopResTest("c1")
+	//	e1 := NE("e1,e3")
+	//	e2 := NE("e2,e3,e4") // e3 gets "merged through" to BOTH edges!
+	//	g2.AddEdge(a, b, e1)
+	//	g2.AddEdge(b, c1, e2)
+	//}
+	//runGraphCmp(t, g1, g2)
+	g3, _ := pgraph.NewGraph("g3") // alternative expected result
 	{
 		a := NewNoopResTest("a1,a2")
 		b := NewNoopResTest("b1,b2")
 		c1 := NewNoopResTest("c1")
-		e1 := NE("e1,e3")
-		e2 := NE("e2,e3,e4") // e3 gets "merged through" to BOTH edges!
-		g2.AddEdge(a, b, e1)
-		g2.AddEdge(b, c1, e2)
+		e1 := NE("e1")
+		e2 := NE("e2,e4")
+		e3 := NE("e3")
+		g3.AddEdge(a, b, e1)
+		g3.AddEdge(b, c1, e2)
+		g3.AddEdge(a, c1, e3)
 	}
-	runGraphCmp(t, g1, g2)
+	runGraphCmp(t, g1, g3)
 }
 
+/*
 // connected merge 0, (no change!)
 // a1            a1
 //   \     >>>     \     (arrows point downwards)
 //    a2            a2
+*/
 func TestPgraphGroupingConnected0(t *testing.T) {
 	g1, _ := pgraph.NewGraph("g1") // original graph
 	{
@@ -836,12 +893,14 @@ func TestPgraphGroupingConnected0(t *testing.T) {
 	runGraphCmp(t, g1, g2)
 }
 
+/*
 // connected merge 1, (no change!)
 // a1              a1
 //   \               \
 //    b      >>>      b      (arrows point downwards)
 //     \               \
 //      a2              a2
+*/
 func TestPgraphGroupingConnected1(t *testing.T) {
 	g1, _ := pgraph.NewGraph("g1") // original graph
 	{

engine/graph/autogroup/base.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -13,7 +13,19 @@
 // GNU General Public License for more details.
 //
 // You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
 
 package autogroup
 
@@ -43,8 +55,24 @@ func (ag *baseGrouper) Init(g *pgraph.Graph) error {
 	if ag.graph != nil {
 		return fmt.Errorf("the init method has already been called")
 	}
-	ag.graph = g                            // pointer
-	ag.vertices = ag.graph.VerticesSorted() // cache in deterministic order!
+	ag.graph = g // pointer
+
+	// We sort deterministically, first by kind, and then by name. In
+	// particular, longer kind chunks sort first. So http:ui:text should
+	// appear before http:server and http:ui. This is a hack so that if we
+	// are doing hierarchical automatic grouping, it gives the http:ui:text
+	// a chance to get grouped into http:ui, before http:ui gets grouped
+	// into http:server, because once that happens, http:ui:text will never
+	// get grouped, and this won't work properly. This works, because when
+	// we start comparing iteratively the list of resources, it does this
+	// with a O(n^2) loop that compares the X and Y zero indexes first, and
+	// and then continues along. If the "longer" resources appear first,
+	// then they'll group together first. We should probably put this into
+	// a new Grouper struct, but for now we might as well leave it here.
+	//vertices := ag.graph.VerticesSorted() // formerly
+	vertices := RHVSort(ag.graph.Vertices())
+
+	ag.vertices = vertices // cache in deterministic order!
 	ag.i = 0
 	ag.j = 0
 	if len(ag.vertices) == 0 { // empty graph
@@ -89,6 +117,19 @@ func (ag *baseGrouper) VertexNext() (v1, v2 pgraph.Vertex, err error) {
 			ag.done = true
 		}
 	}
+	// TODO: is this index swap better or even valid?
+	//if ag.i < l {
+	//	ag.i++
+	//}
+	//if ag.i == l {
+	//	ag.i = 0
+	//	if ag.j < l {
+	//		ag.j++
+	//	}
+	//	if ag.j == l {
+	//		ag.done = true
+	//	}
+	//}
 
 	return
 }
@@ -110,7 +151,7 @@ func (ag *baseGrouper) VertexMerge(v1, v2 pgraph.Vertex) (v pgraph.Vertex, err e
 	return nil, fmt.Errorf("vertexMerge needs to be overridden")
 }
 
-// EdgeMerge can be overridden, since it just simple returns the first edge.
+// EdgeMerge can be overridden, since it just simply returns the first edge.
 func (ag *baseGrouper) EdgeMerge(e1, e2 pgraph.Edge) pgraph.Edge {
 	return e1 // noop
 }

engine/graph/autogroup/nonreachability.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -13,14 +13,25 @@
 // GNU General Public License for more details.
 //
 // You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
 
 package autogroup
 
 import (
 	"github.com/purpleidea/mgmt/pgraph"
-
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 // NonReachabilityGrouper is the most straight-forward algorithm for grouping.

engine/graph/autogroup/util.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -13,19 +13,35 @@
 // GNU General Public License for more details.
 //
 // You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
 
 package autogroup
 
 import (
-	"github.com/purpleidea/mgmt/pgraph"
+	"fmt"
+	"sort"
+	"strings"
 
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/engine"
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
-// VertexMerge merges v2 into v1 by reattaching the edges where appropriate,
-// and then by deleting v2 from the graph. Since more than one edge between two
-// vertices is not allowed, duplicate edges are merged as well. an edge merge
+// VertexMerge merges v2 into v1 by reattaching the edges where appropriate, and
+// then by deleting v2 from the graph. Since more than one edge between two
+// vertices is not allowed, duplicate edges are merged as well. An edge merge
 // function can be provided if you'd like to control how you merge the edges!
 func VertexMerge(g *pgraph.Graph, v1, v2 pgraph.Vertex, vertexMergeFn func(pgraph.Vertex, pgraph.Vertex) (pgraph.Vertex, error), edgeMergeFn func(pgraph.Edge, pgraph.Edge) pgraph.Edge) error {
 	// methodology
@@ -113,8 +129,17 @@ func VertexMerge(g *pgraph.Graph, v1, v2 pgraph.Vertex, vertexMergeFn func(pgrap
 			// note: This branch isn't used if the vertexMergeFn
 			// decides to just merge logically on its own instead
 			// of actually returning something that we then merge.
-			v1 = v // TODO: ineffassign?
+			v1 = v // XXX: ineffassign?
 			//*v1 = *v
+
+			// Ensure that everything still validates. (For safety!)
+			r, ok := v1.(engine.Res) // TODO: v ?
+			if !ok {
+				return fmt.Errorf("not a Res")
+			}
+			if err := engine.Validate(r); err != nil {
+				return errwrap.Wrapf(err, "the Res did not Validate")
+			}
 		}
 	}
 	g.DeleteVertex(v2) // remove grouped vertex
@@ -125,3 +150,67 @@ func VertexMerge(g *pgraph.Graph, v1, v2 pgraph.Vertex, vertexMergeFn func(pgrap
 	}
 	return nil // success
 }
+
+// RHVSlice is a linear list of vertices. It can be sorted by the Kind, taking
+// into account the hierarchy of names separated by colons. Afterwards, it uses
+// String() to avoid the non-determinism in the map type. RHV stands for Reverse
+// Hierarchical Vertex, meaning the hierarchical topology of the vertex
+// (resource) names are used.
+type RHVSlice []pgraph.Vertex
+
+// Len returns the length of the slice of vertices.
+func (obj RHVSlice) Len() int { return len(obj) }
+
+// Swap swaps two elements in the slice.
+func (obj RHVSlice) Swap(i, j int) { obj[i], obj[j] = obj[j], obj[i] }
+
+// Less returns the smaller element in the sort order according to the
+// aforementioned rules.
+// XXX: Add some tests to make sure I didn't get any "reverse" part backwards.
+func (obj RHVSlice) Less(i, j int) bool {
+	resi, oki := obj[i].(engine.Res)
+	resj, okj := obj[j].(engine.Res)
+	if !oki || !okj || resi.Kind() == "" || resj.Kind() == "" {
+		// One of these isn't a normal Res, so just compare normally.
+		return obj[i].String() > obj[j].String() // reverse
+	}
+
+	si := strings.Split(resi.Kind(), ":")
+	sj := strings.Split(resj.Kind(), ":")
+	// both lengths should each be at least one now
+	li := len(si)
+	lj := len(sj)
+
+	if li != lj { // eg: http:ui vs. http:ui:text
+		return li > lj // reverse
+	}
+
+	// same number of chunks
+	for k := 0; k < li; k++ {
+		if si[k] != sj[k] { // lhs chunk differs
+			return si[k] > sj[k] // reverse
+		}
+
+		// if the chunks are the same, we continue...
+	}
+
+	// They must all have the same chunks, so finally we compare the names.
+	return resi.Name() > resj.Name() // reverse
+}
+
+// Sort is a convenience method.
+func (obj RHVSlice) Sort() { sort.Sort(obj) }
+
+// RHVSort returns a deterministically sorted slice of all vertices in the list.
+// The order is sorted by the Kind, taking into account the hierarchy of names
+// separated by colons. Afterwards, it uses String() to avoid the
+// non-determinism in the map type. RHV stands for Reverse Hierarchical Vertex,
+// meaning the hierarchical topology of the vertex (resource) names are used.
+func RHVSort(vertices []pgraph.Vertex) []pgraph.Vertex {
+	var vs []pgraph.Vertex
+	for _, v := range vertices { // copy first
+		vs = append(vs, v)
+	}
+	sort.Sort(RHVSlice(vs)) // add determinism
+	return vs
+}

engine/graph/engine.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -13,8 +13,23 @@
 // GNU General Public License for more details.
 //
 // You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
 
+// Package graph contains the actual implementation of the resource graph engine
+// that runs the graph of resources in real-time. This package has the algorithm
+// that runs all the graph transitions.
 package graph
 
 import (
@@ -25,38 +40,50 @@ import (
 
 	"github.com/purpleidea/mgmt/converger"
 	"github.com/purpleidea/mgmt/engine"
-	"github.com/purpleidea/mgmt/engine/event"
+	"github.com/purpleidea/mgmt/engine/local"
+	engineUtil "github.com/purpleidea/mgmt/engine/util"
 	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/util/errwrap"
 	"github.com/purpleidea/mgmt/util/semaphore"
+)
 
-	multierr "github.com/hashicorp/go-multierror"
-	errwrap "github.com/pkg/errors"
+const (
+	// StateDir is the name of the sub directory where all the local
+	// resource state is stored.
+	StateDir = "state"
 )
 
 // Engine encapsulates a generic graph and manages its operations.
 type Engine struct {
 	Program  string
+	Version  string
 	Hostname string
-	World    engine.World
+
+	Converger *converger.Coordinator
+	Local     *local.API
+	World     engine.World
 
 	// Prefix is a unique directory prefix which can be used. It should be
 	// created if needed.
-	Prefix    string
-	Converger converger.Converger
-
-	Debug bool
-	Logf  func(format string, v ...interface{})
+	Prefix string
+	Debug  bool
+	Logf   func(format string, v ...interface{})
 
 	graph     *pgraph.Graph
 	nextGraph *pgraph.Graph
 	state     map[pgraph.Vertex]*State
-	waits     map[pgraph.Vertex]*sync.WaitGroup
+	waits     map[pgraph.Vertex]*sync.WaitGroup // wg for the Worker func
+	wlock     *sync.Mutex                       // lock around waits map
+
+	mlock *sync.Mutex                            // metas lock
+	metas map[engine.ResPtrUID]*engine.MetaState // meta state
 
 	slock *sync.Mutex // semaphore lock
 	semas map[string]*semaphore.Semaphore
 
-	wg *sync.WaitGroup
+	wg *sync.WaitGroup // wg for the whole engine (only used for close)
 
+	paused    bool // are we paused?
 	fastPause bool
 }
 
@@ -64,6 +91,13 @@ type Engine struct {
 // If the struct does not validate, or it cannot initialize, then this errors.
 // Initially it will contain an empty graph.
 func (obj *Engine) Init() error {
+	if obj.Program == "" {
+		return fmt.Errorf("the Program is empty")
+	}
+	if obj.Hostname == "" {
+		return fmt.Errorf("the Hostname is empty")
+	}
+
 	var err error
 	if obj.graph, err = pgraph.NewGraph("graph"); err != nil {
 		return err
@@ -72,18 +106,25 @@ func (obj *Engine) Init() error {
 	if obj.Prefix == "" || obj.Prefix == "/" {
 		return fmt.Errorf("the prefix of `%s` is invalid", obj.Prefix)
 	}
-	if err := os.MkdirAll(obj.Prefix, 0770); err != nil {
+	// 0775 since we want children to be able to read this!
+	if err := os.MkdirAll(obj.Prefix, 0775); err != nil {
 		return errwrap.Wrapf(err, "can't create prefix")
 	}
 
 	obj.state = make(map[pgraph.Vertex]*State)
 	obj.waits = make(map[pgraph.Vertex]*sync.WaitGroup)
+	obj.wlock = &sync.Mutex{}
+
+	obj.mlock = &sync.Mutex{}
+	obj.metas = make(map[engine.ResPtrUID]*engine.MetaState)
 
 	obj.slock = &sync.Mutex{}
 	obj.semas = make(map[string]*semaphore.Semaphore)
 
 	obj.wg = &sync.WaitGroup{}
 
+	obj.paused = true // start off true, so we can Resume after first Commit
+
 	return nil
 }
 
@@ -118,7 +159,7 @@ func (obj *Engine) Validate() error {
 		}
 
 		if err := engine.Validate(res); err != nil {
-			return errwrap.Wrapf(err, "the Res did not Validate")
+			return fmt.Errorf("%s did not Validate: %v", res, err)
 		}
 	}
 	return nil
@@ -135,8 +176,22 @@ func (obj *Engine) Apply(fn func(*pgraph.Graph) error) error {
 // it errors, then the running graph wasn't changed. It is recommended that you
 // pause the engine before running this, and resume it after you're done.
 func (obj *Engine) Commit() error {
+	// It would be safer to lock this, but it would be slower and mask bugs.
+	//obj.mutex.Lock()
+	//defer obj.mutex.Unlock()
+
 	// TODO: Does this hurt performance or graph changes ?
 
+	activeMetas := make(map[engine.ResPtrUID]struct{})
+	for vertex := range obj.state {
+		res, ok := vertex.(engine.Res)
+		if !ok { // should not happen, previously validated
+			return fmt.Errorf("not a Res")
+		}
+		activeMetas[engine.PtrUID(res)] = struct{}{} // add
+	}
+
+	start := []func() error{} // functions to run after graphsync to start...
 	vertexAddFn := func(vertex pgraph.Vertex) error {
 		// some of these validation steps happen before this Commit step
 		// in Validate() to avoid erroring here. These are redundant.
@@ -153,36 +208,40 @@ func (obj *Engine) Commit() error {
 			return fmt.Errorf("the Res state already exists")
 		}
 
+		activeMetas[engine.PtrUID(res)] = struct{}{} // add
+
 		if obj.Debug {
 			obj.Logf("Validate(%s)", res)
 		}
 		err := engine.Validate(res)
 		if obj.Debug {
-			obj.Logf("Validate(%s): Return(%+v)", res, err)
+			obj.Logf("Validate(%s): Return(%s)", res, engineUtil.CleanError(err))
 		}
 		if err != nil {
 			return errwrap.Wrapf(err, "the Res did not Validate")
 		}
 
-		// FIXME: is res.Name() sufficiently unique to use as a UID here?
-		pathUID := fmt.Sprintf("%s-%s", res.Kind(), res.Name())
-		statePrefix := fmt.Sprintf("%s/", path.Join(obj.Prefix, "state", pathUID))
+		pathUID := engineUtil.ResPathUID(res)
+		statePrefix := fmt.Sprintf("%s/", path.Join(obj.statePrefix(), pathUID))
+
 		// don't create this unless it *will* be used
-		//if err := os.MkdirAll(statePrefix, 0770); err != nil {
+		//if err := os.MkdirAll(statePrefix, 0775); err != nil {
 		//	return errwrap.Wrapf(err, "can't create state prefix")
 		//}
 
 		obj.waits[vertex] = &sync.WaitGroup{}
 		obj.state[vertex] = &State{
-			//Graph: obj.graph, // TODO: what happens if we swap the graph?
+			Graph:  obj.graph, // Update if we swap the graph!
 			Vertex: vertex,
 
 			Program:  obj.Program,
+			Version:  obj.Version,
 			Hostname: obj.Hostname,
 
+			//Converger: obj.Converger,
+			Local:  obj.Local,
 			World:  obj.World,
 			Prefix: statePrefix,
-			//Converger: obj.Converger,
 
 			Debug: obj.Debug,
 			Logf: func(format string, v ...interface{}) {
@@ -192,18 +251,65 @@ func (obj *Engine) Commit() error {
 		if err := obj.state[vertex].Init(); err != nil {
 			return errwrap.Wrapf(err, "the Res did not Init")
 		}
+
+		fn := func() error {
+			// start the Worker
+			obj.wg.Add(1)
+			obj.wlock.Lock()
+			obj.waits[vertex].Add(1)
+			obj.wlock.Unlock()
+			go func(v pgraph.Vertex) {
+				defer obj.wg.Done()
+				defer func() {
+					// we need this lock, because this go
+					// routine could run when the next fn
+					// function above here is running...
+					obj.wlock.Lock()
+					obj.waits[v].Done()
+					obj.wlock.Unlock()
+				}()
+
+				if obj.Debug {
+					obj.Logf("%s: Working...", v)
+				}
+				// contains the Watch and CheckApply loops
+				err := obj.Worker(v)
+				if obj.Debug || true {
+					if s := engineUtil.CleanError(err); err != nil {
+						obj.Logf("%s: Error: %s", v, s)
+					} else {
+						obj.Logf("%s: Exited...", v)
+					}
+				}
+				obj.state[v].workerErr = err // store the error
+				// If the Rewatch metaparam is true, then this will get
+				// restarted if we do a graph cmp swap. This is why the
+				// graph cmp function runs the removes before the adds.
+				// XXX: This should feed into an $error var in the lang.
+			}(vertex)
+			return nil
+		}
+		start = append(start, fn) // do this at the end, if it's needed
 		return nil
 	}
+
 	free := []func() error{} // functions to run after graphsync to reset...
 	vertexRemoveFn := func(vertex pgraph.Vertex) error {
+		res, ok := vertex.(engine.Res)
+		if !ok { // should not happen, previously validated
+			return fmt.Errorf("not a Res")
+		}
+		delete(activeMetas, engine.PtrUID(res))
+
 		// wait for exit before starting new graph!
-		obj.state[vertex].Event(event.Exit) // signal an exit
-		obj.waits[vertex].Wait()            // sync
+		close(obj.state[vertex].removeDone)   // causes doneCtx to cancel
+		close(obj.state[vertex].resumeSignal) // unblock (it only closes here)
+		obj.waits[vertex].Wait()              // sync
 
 		// close the state and resource
 		// FIXME: will this mess up the sync and block the engine?
-		if err := obj.state[vertex].Close(); err != nil {
-			return errwrap.Wrapf(err, "the Res did not Close")
+		if err := obj.state[vertex].Cleanup(); err != nil {
+			return errwrap.Wrapf(err, "the Res did not Cleanup")
 		}
 
 		// delete to free up memory from old graphs
@@ -216,15 +322,75 @@ func (obj *Engine) Commit() error {
 		return nil
 	}
 
+	// add the Worker swap (reload) on error decision into this vertexCmpFn
+	vertexCmpFn := func(v1, v2 pgraph.Vertex) (bool, error) {
+		r1, ok1 := v1.(engine.Res)
+		r2, ok2 := v2.(engine.Res)
+		if !ok1 || !ok2 { // should not happen, previously validated
+			return false, fmt.Errorf("not a Res")
+		}
+		m1 := r1.MetaParams()
+		m2 := r2.MetaParams()
+		swap1, swap2 := true, true // assume default of true
+		if m1 != nil {
+			swap1 = m1.Rewatch
+		}
+		if m2 != nil {
+			swap2 = m2.Rewatch
+		}
+
+		s1, ok1 := obj.state[v1]
+		s2, ok2 := obj.state[v2]
+		x1, x2 := false, false
+		if ok1 {
+			x1 = s1.workerErr != nil && swap1
+		}
+		if ok2 {
+			x2 = s2.workerErr != nil && swap2
+		}
+
+		if x1 || x2 {
+			// We swap, even if they're the same, so that we reload!
+			// This causes an add and remove of the "same" vertex...
+			return false, nil
+		}
+
+		return engine.VertexCmpFn(v1, v2) // do the normal cmp otherwise
+	}
+
 	// If GraphSync succeeds, it updates the receiver graph accordingly...
 	// Running the shutdown in vertexRemoveFn does not need to happen in a
 	// topologically sorted order because it already paused in that order.
 	obj.Logf("graph sync...")
-	if err := obj.graph.GraphSync(obj.nextGraph, engine.VertexCmpFn, vertexAddFn, vertexRemoveFn, engine.EdgeCmpFn); err != nil {
+	if err := obj.graph.GraphSync(obj.nextGraph, vertexCmpFn, vertexAddFn, vertexRemoveFn, engine.EdgeCmpFn); err != nil {
 		return errwrap.Wrapf(err, "error running graph sync")
 	}
-	// we run these afterwards, so that the state structs (that might get
-	// referenced) aren't destroyed while someone might poke or use one.
+
+	// This happens after GraphSync when vertexRemoveFn and vertexAddFn are
+	// done running. Those two modified the activeMetas map. It's important
+	// that vertexRemoveFn runs before vertexAddFn, but GraphSync guarantees
+	// that, and it would be kind of illogical to not run things that way.
+	metaGC := make(map[engine.ResPtrUID]struct{}) // which metas should we garbage collect?
+	obj.mlock.Lock()
+	for ptrUID := range obj.metas {
+		if _, exists := activeMetas[ptrUID]; !exists {
+			metaGC[ptrUID] = struct{}{}
+		}
+	}
+	for ptrUID := range metaGC {
+		delete(obj.metas, ptrUID) // otherwise, this could grow forever
+	}
+	obj.mlock.Unlock()
+
+	// We run these afterwards, so that we don't unnecessarily start anyone
+	// if GraphSync failed in some way. Otherwise we'd have to do clean up!
+	for _, fn := range start {
+		if err := fn(); err != nil {
+			return errwrap.Wrapf(err, "error running start fn")
+		}
+	}
+	// We run these afterwards, so that the state structs (that might get
+	// referenced) are not destroyed while someone might poke or use one.
 	for _, fn := range free {
 		if err := fn(); err != nil {
 			return errwrap.Wrapf(err, "error running free fn")
@@ -236,62 +402,59 @@ func (obj *Engine) Commit() error {
 	// the changes that we'd made to the previously primary graph. This is
 	// because this function is meant to atomically swap the graphs safely.
 
-	// TODO: update all the `State` structs with the new Graph pointer
-	//for _, vertex := range obj.graph.Vertices() {
-	//	state, exists := obj.state[vertex]
-	//	if !exists {
-	//		continue
-	//	}
-	//	state.Graph = obj.graph // update pointer to graph
-	//}
+	// Update all the `State` structs with the new Graph pointer.
+	for _, vertex := range obj.graph.Vertices() {
+		state, exists := obj.state[vertex]
+		if !exists {
+			continue
+		}
+		state.Graph = obj.graph // update pointer to graph
+	}
 
 	return nil
 }
 
-// Start runs the currently active graph. It also un-pauses the graph if it was
-// paused.
-func (obj *Engine) Start() error {
+// Resume runs the currently active graph. It also un-pauses the graph if it was
+// paused. Very little that is interesting should happen here. It all happens in
+// the Commit method. After Commit, new things are already started, but we still
+// need to Resume any pre-existing resources. Do not call this concurrently with
+// the Pause method.
+func (obj *Engine) Resume() error {
+	// It would be safer to lock this, but it would be slower and mask bugs.
+	//obj.mutex.Lock()
+	//defer obj.mutex.Unlock()
+
+	if !obj.paused {
+		return fmt.Errorf("already resumed")
+	}
+
 	topoSort, err := obj.graph.TopologicalSort()
 	if err != nil {
 		return err
 	}
-	indegree := obj.graph.InDegree() // compute all of the indegree's
+	//indegree := obj.graph.InDegree() // compute all of the indegree's
 	reversed := pgraph.Reverse(topoSort)
 
 	for _, vertex := range reversed {
-		state := obj.state[vertex]
-		state.starter = (indegree[vertex] == 0)
-		var unpause = true // assume true
+		// The very first resume is skipped as those resources are
+		// already running! We could do that by checking here, but it is
+		// more convenient to just have a state struct field (paused) to
+		// track things for this instead. As a bonus, it helps us know
+		// if a resource is paused or not if we print for debugging.
+		//if !obj.state[vertex].initialStartupDone {
+		//	obj.state[vertex].initialStartupDone = true
+		//	continue
+		//}
 
-		if !state.working { // if not running...
-			state.working = true
-			unpause = false // doesn't need unpausing if starting
-			obj.wg.Add(1)
-			obj.waits[vertex].Add(1)
-			go func(v pgraph.Vertex) {
-				defer obj.wg.Done()
-				defer obj.waits[vertex].Done()
-				defer func() {
-					obj.state[v].working = false
-				}()
-
-				obj.Logf("Worker(%s)", v)
-				// contains the Watch and CheckApply loops
-				err := obj.Worker(v)
-				obj.Logf("Worker(%s): Exited(%+v)", v, err)
-			}(vertex)
-		}
-
-		select {
-		case <-state.started:
-		case <-state.stopped: // we failed on Watch start
-		}
-
-		if unpause { // unpause (if needed)
-			obj.state[vertex].Event(event.Start)
-		}
+		//obj.state[vertex].starter = (indegree[vertex] == 0)
+		obj.state[vertex].Resume() // doesn't error
+		// This always works because if a resource errored while it was
+		// paused, then we're in the paused state and we can still exit
+		// from there. If a resource errors when we're trying to Pause
+		// then it will only succeed without error if the resource ACKs.
 	}
 	// we wait for everyone to start before exiting!
+	obj.paused = false
 	return nil
 }
 
@@ -302,40 +465,52 @@ func (obj *Engine) Start() error {
 // This is because once you've started a fast pause, some dependencies might
 // have been skipped when fast pausing, and future resources might have missed a
 // poke. In general this is only called when you're trying to hurry up the exit.
+// XXX: Not implemented
 func (obj *Engine) SetFastPause() {
 	obj.fastPause = true
 }
 
-// Pause the active, running graph. At the moment this cannot error.
-func (obj *Engine) Pause(fastPause bool) {
+// Pause the active, running graph.
+func (obj *Engine) Pause(fastPause bool) error {
+	// It would be safer to lock this, but it would be slower and mask bugs.
+	//obj.mutex.Lock()
+	//defer obj.mutex.Unlock()
+
+	if obj.paused {
+		return fmt.Errorf("already paused")
+	}
+
 	obj.fastPause = fastPause
 	topoSort, _ := obj.graph.TopologicalSort()
 	for _, vertex := range topoSort { // squeeze out the events...
 		// The Event is sent to an unbuffered channel, so this event is
 		// synchronous, and as a result it blocks until it is received.
-		obj.state[vertex].Event(event.Pause)
+		if err := obj.state[vertex].Pause(); err != nil && err != engine.ErrClosed {
+			return err
+		}
 	}
 
+	obj.paused = true
+
 	// we are now completely paused...
 	obj.fastPause = false // reset
+	return nil
 }
 
-// Close triggers a shutdown. Engine must be already paused before this is run.
-func (obj *Engine) Close() error {
-	var reterr error
-
-	emptyGraph, err := pgraph.NewGraph("empty")
-	if err != nil {
-		reterr = multierr.Append(reterr, err) // list of errors
-	}
+// Shutdown the engine. Engine must be already paused before this is run. It is
+// actually just a Load of an empty graph and a Commit. It waits for all the
+// resources to exit before returning.
+func (obj *Engine) Shutdown() error {
+	emptyGraph, reterr := pgraph.NewGraph("empty")
 
 	// this is a graph switch (graph sync) that switches to an empty graph!
 	if err := obj.Load(emptyGraph); err != nil { // copy in empty graph
-		reterr = multierr.Append(reterr, err)
+		reterr = errwrap.Append(reterr, err)
 	}
+	// FIXME: Do we want to run commit if Load failed? Does this even work?
 	// the commit will cause the graph sync to shut things down cleverly...
 	if err := obj.Commit(); err != nil {
-		reterr = multierr.Append(reterr, err)
+		reterr = errwrap.Append(reterr, err)
 	}
 
 	obj.wg.Wait() // for now, this doesn't need to be a separate Wait() method
@@ -346,3 +521,8 @@ func (obj *Engine) Close() error {
 func (obj *Engine) Graph() *pgraph.Graph {
 	return obj.graph
 }
+
+// statePrefix returns the dir where all the resource state is stored locally.
+func (obj *Engine) statePrefix() string {
+	return fmt.Sprintf("%s/", path.Join(obj.Prefix, StateDir))
+}

engine/graph/graph_test.go

@@ -0,0 +1,49 @@
+// Mgmt
+// Copyright (C) James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+//go:build !root
+
+package graph
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/purpleidea/mgmt/util/errwrap"
+)
+
+func TestMultiErr(t *testing.T) {
+	var err error
+	e := fmt.Errorf("some error")
+	err = errwrap.Append(err, e) // build an error from a nil base
+	// ensure that this lib allows us to append to a nil
+	if err == nil {
+		t.Errorf("missing error")
+	}
+}

engine/graph/refresh.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -13,7 +13,19 @@
 // GNU General Public License for more details.
 //
 // You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
 
 package graph
 

engine/graph/reverse.go

@@ -0,0 +1,311 @@
+// Mgmt
+// Copyright (C) James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+package graph
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"sort"
+
+	"github.com/purpleidea/mgmt/engine"
+	engineUtil "github.com/purpleidea/mgmt/engine/util"
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/util/errwrap"
+)
+
+const (
+	// ReverseFile is the file name in the resource state dir where any
+	// reversal information is stored.
+	ReverseFile = "reverse"
+
+	// ReversePerm is the permissions mode used to create the ReverseFile.
+	ReversePerm = 0600
+)
+
+// Reversals adds the reversals onto the loaded graph. This should happen last,
+// and before Commit.
+func (obj *Engine) Reversals() error {
+	if obj.nextGraph == nil {
+		return fmt.Errorf("there is no active graph to add reversals to")
+	}
+
+	// Initially get all of the reversals to seek out all possible errors.
+	// XXX: The engine needs to know where data might have been stored if we
+	// XXX: want to potentially allow alternate read/write paths, like etcd.
+	// XXX: In this scenario, we'd have to store a token somewhere to let us
+	// XXX: know to look elsewhere for the special ReversalList read method.
+	data, err := obj.ReversalList() // (map[string]string, error)
+	if err != nil {
+		return errwrap.Wrapf(err, "the reversals had errors")
+	}
+
+	if len(data) == 0 {
+		return nil // end early
+	}
+
+	resMatch := func(r1, r2 engine.Res) bool { // simple match on UID only!
+		if r1.Kind() != r2.Kind() {
+			return false
+		}
+		if r1.Name() != r2.Name() {
+			return false
+		}
+		return true
+	}
+	resInList := func(needle engine.Res, haystack []engine.Res) bool {
+		for _, res := range haystack {
+			if resMatch(needle, res) {
+				return true
+			}
+		}
+		return false
+	}
+
+	if obj.Debug {
+		obj.Logf("decoding %d reversals...", len(data))
+	}
+	resources := []engine.Res{}
+
+	// do this in a sorted order so that it errors deterministically
+	sorted := []string{}
+	for key := range data {
+		sorted = append(sorted, key)
+	}
+	sort.Strings(sorted)
+	for _, key := range sorted {
+		val := data[key]
+		// XXX: replace this ResToB64 method with one that stores it in
+		// a human readable format, in case someone wants to hack and
+		// edit it manually.
+		// XXX: we probably want this to be YAML, it works with the diff
+		// too...
+		r, err := engineUtil.B64ToRes(val)
+		if err != nil {
+			return errwrap.Wrapf(err, "error decoding res with UID: `%s`", key)
+		}
+
+		res, ok := r.(engine.ReversibleRes)
+		if !ok {
+			// this requirement is here to keep things simpler...
+			return errwrap.Wrapf(err, "decoded res with UID: `%s` was not reversible", key)
+		}
+
+		matchFn := func(vertex pgraph.Vertex) (bool, error) {
+			r, ok := vertex.(engine.Res)
+			if !ok {
+				return false, fmt.Errorf("not a Res")
+			}
+			if !resMatch(r, res) {
+				return false, nil
+			}
+			return true, nil
+		}
+
+		// FIXME: not efficient, we could build a cache-map first
+		vertex, err := obj.nextGraph.VertexMatchFn(matchFn) // (Vertex, error)
+		if err != nil {
+			return errwrap.Wrapf(err, "error searching graph for match")
+		}
+		if vertex != nil { // found one!
+			continue // it doesn't need reversing yet
+		}
+
+		// TODO: check for (incompatible?) duplicates instead
+		if resInList(res, resources) { // we've already got this one...
+			continue
+		}
+
+		// We set this in two different places to be safe. It ensures
+		// that we erase the reversal state file after we've used it.
+		res.ReversibleMeta().Reversal = true // set this for later...
+
+		resources = append(resources, res)
+	}
+
+	if len(resources) == 0 {
+		return nil // end early
+	}
+
+	// Now that we've passed the chance of any errors, we modify the graph.
+	obj.Logf("adding %d reversals...", len(resources))
+	for _, res := range resources {
+		obj.nextGraph.AddVertex(res)
+	}
+	// TODO: Do we want a way for stored reversals to add edges too?
+
+	// It would be great to ensure we didn't add any graph cycles here, but
+	// instead of checking now, we'll move the check into the main loop.
+	return nil
+}
+
+// ReversalList returns all the available pending reversal data on this host. It
+// can then be decoded by whatever method is appropriate for.
+func (obj *Engine) ReversalList() (map[string]string, error) {
+	result := make(map[string]string) // some key to contents
+
+	dir := obj.statePrefix() // loop through this dir...
+	files, err := os.ReadDir(dir)
+	if err != nil && !os.IsNotExist(err) {
+		return nil, errwrap.Wrapf(err, "error reading list of state dirs")
+	} else if err != nil {
+		return result, nil // nothing found, no state dir exists yet
+	}
+
+	for _, x := range files {
+		key := x.Name() // some uid for the resource
+		file := path.Join(dir, key, ReverseFile)
+		content, err := os.ReadFile(file)
+		if err != nil && !os.IsNotExist(err) {
+			return nil, errwrap.Wrapf(err, "could not read reverse file: %s", file)
+		} else if err != nil {
+			continue // file does not exist, skip
+		}
+
+		// file exists!
+		str := string(content)
+		result[key] = str // save
+	}
+
+	return result, nil
+}
+
+// ReversalInit performs the reversal initialization steps if necessary for this
+// resource.
+func (obj *State) ReversalInit() error {
+	res, ok := obj.Vertex.(engine.ReversibleRes)
+	if !ok {
+		return nil // nothing to do
+	}
+
+	if res.ReversibleMeta().Disabled {
+		return nil // nothing to do, reversal isn't enabled
+	}
+
+	// If the reversal is enabled, but we are the result of a previous
+	// reversal, then this will overwrite that older reversal request, and
+	// our resource should be designed to deal with that. This happens if we
+	// return a reversible resource as the reverse of a resource that was
+	// reversed. It's probably fairly rare.
+	if res.ReversibleMeta().Reversal {
+		obj.Logf("triangle reversal") // warn!
+	}
+
+	r, err := res.Reversed()
+	if err != nil {
+		return errwrap.Wrapf(err, "could not reverse: %s", res.String())
+	}
+	if r == nil {
+		return nil // this can't be reversed, or isn't implemented here
+	}
+
+	// We set this in two different places to be safe. It ensures that we
+	// erase the reversal state file after we've used it.
+	r.ReversibleMeta().Reversal = true // set this for later...
+
+	// XXX: replace this ResToB64 method with one that stores it in a human
+	// readable format, in case someone wants to hack and edit it manually.
+	// XXX: we probably want this to be YAML, it works with the diff too...
+	str, err := engineUtil.ResToB64(r)
+	if err != nil {
+		return errwrap.Wrapf(err, "could not encode: %s", res.String())
+	}
+
+	// TODO: put this method on traits.Reversible as part of the interface?
+	return obj.ReversalWrite(str, res.ReversibleMeta().Overwrite) // Store!
+}
+
+// ReversalCleanup performs the reversal shutdown steps if necessary for this
+// resource.
+func (obj *State) ReversalCleanup() error {
+	res, ok := obj.Vertex.(engine.ReversibleRes)
+	if !ok {
+		return nil // nothing to do
+	}
+
+	// Don't check res.ReversibleMeta().Disabled because we're removing the
+	// previous one. That value only applies if we're doing a new reversal.
+
+	if !res.ReversibleMeta().Reversal {
+		return nil // nothing to erase, we're not a reversal resource
+	}
+
+	if !obj.isStateOK.Load() { // did we successfully reverse? (mutex RLock/RUnlock)
+		obj.Logf("did not complete reversal") // warn
+		return nil
+	}
+
+	// TODO: put this method on traits.Reversible as part of the interface?
+	return obj.ReversalDelete() // Erase our reversal instructions.
+}
+
+// ReversalWrite stores the reversal state information for this resource.
+func (obj *State) ReversalWrite(str string, overwrite bool) error {
+	dir, err := obj.varDir("") // private version
+	if err != nil {
+		return errwrap.Wrapf(err, "could not get VarDir for reverse")
+	}
+	file := path.Join(dir, ReverseFile) // return a unique file
+
+	content, err := os.ReadFile(file)
+	if err != nil && !os.IsNotExist(err) {
+		return errwrap.Wrapf(err, "could not read reverse file: %s", file)
+	}
+
+	// file exists and we shouldn't overwrite if different
+	if err == nil && !overwrite {
+		// compare to existing file
+		oldStr := string(content)
+		if str != oldStr {
+			obj.Logf("existing, pending, reversible resource exists")
+			//obj.Logf("diff:")
+			//obj.Logf("") // TODO: print the diff w/o secret values
+			return fmt.Errorf("existing, pending, reversible resource exists")
+		}
+	}
+
+	return os.WriteFile(file, []byte(str), ReversePerm)
+}
+
+// ReversalDelete removes the reversal state information for this resource.
+func (obj *State) ReversalDelete() error {
+	dir, err := obj.varDir("") // private version
+	if err != nil {
+		return errwrap.Wrapf(err, "could not get VarDir for reverse")
+	}
+	file := path.Join(dir, ReverseFile) // return a unique file
+
+	// FIXME: why do we see these removals when there isn't a state file?
+	if err = os.Remove(file); os.IsNotExist(err) {
+		return nil // ignore missing files
+	}
+
+	return errwrap.Wrapf(err, "could not remove reverse state file")
+}

engine/graph/semaphore.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -13,7 +13,19 @@
 // GNU General Public License for more details.
 //
 // You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
 
 package graph
 
@@ -23,9 +35,8 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/purpleidea/mgmt/util/errwrap"
 	"github.com/purpleidea/mgmt/util/semaphore"
-
-	multierr "github.com/hashicorp/go-multierror"
 )
 
 // SemaSep is the trailing separator to split the semaphore id from the size.
@@ -46,9 +57,8 @@ func (obj *Engine) semaLock(semas []string) error {
 		}
 		obj.slock.Unlock()
 
-		if err := sema.P(1); err != nil { // lock!
-			reterr = multierr.Append(reterr, err) // list of errors
-		}
+		err := sema.P(1)                     // lock!
+		reterr = errwrap.Append(reterr, err) // list of errors
 	}
 	return reterr
 }
@@ -65,9 +75,8 @@ func (obj *Engine) semaUnlock(semas []string) error {
 			panic(fmt.Sprintf("graph: sema: %s does not exist", id))
 		}
 
-		if err := sema.V(1); err != nil { // unlock!
-			reterr = multierr.Append(reterr, err) // list of errors
-		}
+		err := sema.V(1)                     // unlock!
+		reterr = errwrap.Append(reterr, err) // list of errors
 	}
 	return reterr
 }

engine/graph/semaphore_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -13,9 +13,21 @@
 // GNU General Public License for more details.
 //
 // You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
 
-// +build !root
+//go:build !root
 
 package graph
 

engine/graph/sendrecv.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -13,94 +13,267 @@
 // GNU General Public License for more details.
 //
 // You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
 
 package graph
 
 import (
 	"fmt"
 	"reflect"
+	"sort"
 
 	"github.com/purpleidea/mgmt/engine"
-
-	multierr "github.com/hashicorp/go-multierror"
-	errwrap "github.com/pkg/errors"
+	engineUtil "github.com/purpleidea/mgmt/engine/util"
+	"github.com/purpleidea/mgmt/lang/types"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
+// RecvFn represents a custom Recv function which can be used in place of the
+// stock, built-in one. This is needed if we want to receive from a different
+// resource data source than our own. (Only for special occasions of course!)
+type RecvFn func(engine.RecvableRes) (map[string]*engine.Send, error)
+
 // SendRecv pulls in the sent values into the receive slots. It is called by the
 // receiver and must be given as input the full resource struct to receive on.
-// It applies the loaded values to the resource.
-func (obj *Engine) SendRecv(res engine.RecvableRes) (map[string]bool, error) {
-	recv := res.Recv()
-	if obj.Debug {
-		// NOTE: this could expose private resource data like passwords
-		obj.Logf("%s: SendRecv: %+v", res, recv)
+// It applies the loaded values to the resource. It is called recursively, as it
+// recurses into any grouped resources found within the first receiver. It
+// returns a map of resource pointer, to resource field key, to changed boolean.
+func SendRecv(res engine.RecvableRes, fn RecvFn) (map[engine.RecvableRes]map[string]*engine.Send, error) {
+	updated := make(map[engine.RecvableRes]map[string]*engine.Send) // list of updated keys
+	if groupableRes, ok := res.(engine.GroupableRes); ok {
+		for _, x := range groupableRes.GetGroup() { // grouped elements
+			recvableRes, ok := x.(engine.RecvableRes)
+			if !ok {
+				continue
+			}
+			//if obj.Debug {
+			//	obj.Logf("SendRecv: %s: grouped: %s", res, x) // receiving here
+			//}
+			// We need to recurse here so that autogrouped resources
+			// inside autogrouped resources would work... In case we
+			// work correctly. We just need to make sure that things
+			// are grouped in the correct order, but that is not our
+			// problem! Recurse and merge in the changed results...
+			innerUpdated, err := SendRecv(recvableRes, fn)
+			if err != nil {
+				return nil, errwrap.Wrapf(err, "recursive SendRecv error")
+			}
+			for r, m := range innerUpdated { // res ptr, map
+				if _, exists := updated[r]; !exists {
+					updated[r] = make(map[string]*engine.Send)
+				}
+				for s, send := range m { // map[string]*engine.Send
+					b := send.Changed
+					// don't overwrite in case one exists...
+					if old, exists := updated[r][s]; exists {
+						b = b || old.Changed // unlikely i think
+					}
+					if _, exists := updated[r][s]; !exists {
+						newSend := &engine.Send{
+							Res:     send.Res,
+							Key:     send.Key,
+							Changed: b,
+						}
+						updated[r][s] = newSend
+					}
+					updated[r][s].Changed = b
+				}
+			}
+		}
 	}
-	var updated = make(map[string]bool) // list of updated keys
+
 	var err error
-	for k, v := range recv {
-		updated[k] = false // default
-		v.Changed = false  // reset to the default
+	recv := res.Recv()
+	if fn != nil {
+		recv, err = fn(res) // use a custom Recv function
+		if err != nil {
+			return nil, err
+		}
+	}
+	keys := []string{}
+	for k := range recv { // map[string]*Send
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	//if obj.Debug && len(keys) > 0 {
+	//	// NOTE: this could expose private resource data like passwords
+	//	obj.Logf("SendRecv: %s recv: %+v", res, strings.Join(keys, ", "))
+	//}
+	for k, v := range recv { // map[string]*Send
+		// v.Res // SendableRes // a handle to the resource which is sending a value
+		// v.Key // string      // the key in the resource that we're sending
+		if _, exists := updated[res]; !exists {
+			updated[res] = make(map[string]*engine.Send)
+		}
+
+		//updated[res][k] = false // default
+		v.Changed = false   // reset to the default
+		updated[res][k] = v // default
 
 		var st interface{} = v.Res // old style direct send/recv
 		if true {                  // new style send/recv API
 			st = v.Res.Sent()
 		}
 
+		if st == nil {
+			e := fmt.Errorf("received nil value from: %s", v.Res)
+			err = errwrap.Append(err, e) // list of errors
+			continue
+		}
+
+		if e := engineUtil.StructFieldCompat(st, v.Key, res, k); e != nil {
+			err = errwrap.Append(err, e) // list of errors
+			continue
+		}
+
 		// send
+		m1, e := engineUtil.StructTagToFieldName(st)
+		if e != nil {
+			err = errwrap.Append(err, e) // list of errors
+			continue
+		}
+		key1, exists := m1[v.Key]
+		if !exists {
+			e := fmt.Errorf("requested key of `%s` not found in send struct", v.Key)
+			err = errwrap.Append(err, e) // list of errors
+			continue
+		}
+
 		obj1 := reflect.Indirect(reflect.ValueOf(st))
-		type1 := obj1.Type()
-		value1 := obj1.FieldByName(v.Key)
+		//type1 := obj1.Type()
+		value1 := obj1.FieldByName(key1)
 		kind1 := value1.Kind()
 
 		// recv
+		m2, e := engineUtil.StructTagToFieldName(res)
+		if e != nil {
+			err = errwrap.Append(err, e) // list of errors
+			continue
+		}
+		key2, exists := m2[k]
+		if !exists {
+			e := fmt.Errorf("requested key of `%s` not found in recv struct", k)
+			err = errwrap.Append(err, e) // list of errors
+			continue
+		}
+
 		obj2 := reflect.Indirect(reflect.ValueOf(res)) // pass in full struct
-		type2 := obj2.Type()
-		value2 := obj2.FieldByName(k)
+		//type2 := obj2.Type()
+		value2 := obj2.FieldByName(key2)
 		kind2 := value2.Kind()
 
-		if obj.Debug {
-			obj.Logf("Send(%s) has %v: %v", type1, kind1, value1)
-			obj.Logf("Recv(%s) has %v: %v", type2, kind2, value2)
+		//orig := value1
+		dest := value2 // save the o.g. because we need the real dest!
+
+		// NOTE: Reminder: obj1 comes from st and it is the *<Res>Sends
+		// struct which contains whichever fields that resource sends.
+		// For example, this might be *TestSends for the Test resource.
+		// The receiver is obj2 and that is actually the resource struct
+		// which is a *<Res> and which gets it's fields directly set on.
+		// For example, this might be *TestRes for the Test resource.
+		//fmt.Printf("obj1(%T): %+v\n", obj1, obj1)
+		//fmt.Printf("obj2(%T): %+v\n", obj2, obj2)
+		// Lastly, remember that many of the type incompatibilities are
+		// caught during type unification, and so we might have overly
+		// relaxed the checks here and something could slip by. If we
+		// find something, this code will need new checks added back.
+
+		// Here we unpack one-level, and then leave the complex stuff
+		// for the Into() method below.
+		// for kind1 == reflect.Interface || kind1 == reflect.Ptr // wrong
+		// if kind1 == reflect.Interface || kind1 == reflect.Ptr  // wrong
+		// for kind1 == reflect.Interface // wrong
+		if kind1 == reflect.Interface {
+			value1 = value1.Elem() // un-nest one interface
+			kind1 = value1.Kind()
 		}
 
-		// i think we probably want the same kind, at least for now...
-		if kind1 != kind2 {
-			e := fmt.Errorf("kind mismatch between %s: %s and %s: %s", v.Res, kind1, res, kind2)
-			err = multierr.Append(err, e) // list of errors
-			continue
+		// This second block is identical, but it's just accidentally
+		// symmetrical. The types of input structs are different shapes.
+		// for kind2 == reflect.Interface || kind2 == reflect.Ptr // wrong
+		// if kind2 == reflect.Interface || kind2 == reflect.Ptr  // wrong
+		// for kind2 == reflect.Interface // wrong
+		if kind2 == reflect.Interface {
+			value2 = value2.Elem() // un-nest one interface
+			kind2 = value2.Kind()
 		}
 
-		// if the types don't match, we can't use send->recv
-		// FIXME: do we want to relax this for string -> *string ?
-		if e := TypeCmp(value1, value2); e != nil {
-			e := errwrap.Wrapf(e, "type mismatch between %s and %s", v.Res, res)
-			err = multierr.Append(err, e) // list of errors
-			continue
-		}
+		//if obj.Debug {
+		//	obj.Logf("Send(%s) has %v: %v", type1, kind1, value1)
+		//	obj.Logf("Recv(%s) has %v: %v", type2, kind2, value2)
+		//}
+
+		// Skip this check in favour of the more complex Into() below...
+		//if kind1 != kind2 {
+		//	e := fmt.Errorf("send/recv kind mismatch between %s: %s and %s: %s", v.Res, kind1, res, kind2)
+		//	err = errwrap.Append(err, e) // list of errors
+		//	continue
+		//}
+
+		// Skip this check in favour of the more complex Into() below...
+		//if e := TypeCmp(value1, value2); e != nil {
+		//	e := errwrap.Wrapf(e, "type mismatch between %s and %s", v.Res, res)
+		//	err = errwrap.Append(err, e) // list of errors
+		//	continue
+		//}
 
 		// if we can't set, then well this is pointless!
-		if !value2.CanSet() {
+		if !dest.CanSet() {
 			e := fmt.Errorf("can't set %s.%s", res, k)
-			err = multierr.Append(err, e) // list of errors
+			err = errwrap.Append(err, e) // list of errors
 			continue
 		}
 
 		// if we can't interface, we can't compare...
-		if !value1.CanInterface() || !value2.CanInterface() {
+		if !value1.CanInterface() {
+			e := fmt.Errorf("can't interface %s.%s", v.Res, v.Key)
+			err = errwrap.Append(err, e) // list of errors
+			continue
+		}
+		if !value2.CanInterface() {
 			e := fmt.Errorf("can't interface %s.%s", res, k)
-			err = multierr.Append(err, e) // list of errors
+			err = errwrap.Append(err, e) // list of errors
 			continue
 		}
 
 		// if the values aren't equal, we're changing the receiver
-		if !reflect.DeepEqual(value1.Interface(), value2.Interface()) {
-			// TODO: can we catch the panics here in case they happen?
-			value2.Set(value1) // do it for all types that match
-			updated[k] = true  // we updated this key!
-			v.Changed = true   // tag this key as updated!
-			obj.Logf("SendRecv: %s.%s -> %s.%s", v.Res, v.Key, res, k)
+		if reflect.DeepEqual(value1.Interface(), value2.Interface()) {
+			continue // skip as they're the same, no error needed
 		}
+
+		// TODO: can we catch the panics here in case they happen?
+
+		fv, e := types.ValueOf(value1)
+		if e != nil {
+			e := errwrap.Wrapf(e, "bad value %s.%s", v.Res, v.Key)
+			err = errwrap.Append(err, e) // list of errors
+			continue
+		}
+
+		// mutate the struct field dest with the mcl data in fv
+		if e := types.Into(fv, dest); e != nil {
+			// runtime error, probably from using value res
+			e := errwrap.Wrapf(e, "mismatch: %s.%s (%s) -> %s.%s (%s)", v.Res, v.Key, kind1, res, k, kind2)
+			err = errwrap.Append(err, e) // list of errors
+			continue
+		}
+		//dest.Set(orig)  // do it for all types that match
+		//updated[res][k] = true // we updated this key!
+		v.Changed = true    // tag this key as updated!
+		updated[res][k] = v // we updated this key!
+		//obj.Logf("SendRecv: %s.%s -> %s.%s (%+v)", v.Res, v.Key, res, k, fv) // fv may be private data
 	}
 	return updated, err
 }
@@ -116,3 +289,19 @@ func TypeCmp(a, b reflect.Value) error {
 
 	return nil // identical Type()'s
 }
+
+// UpdatedStrings returns a list of strings showing what was updated after a
+// Send/Recv run returned the updated datastructure. This is useful for logs.
+func UpdatedStrings(updated map[engine.RecvableRes]map[string]*engine.Send) []string {
+	out := []string{}
+	for r, m := range updated { // map[engine.RecvableRes]map[string]*engine.Send
+		for s, send := range m {
+			if !send.Changed {
+				continue
+			}
+			x := fmt.Sprintf("%v.%s -> %v.%s", send.Res, send.Key, r, s)
+			out = append(out, x)
+		}
+	}
+	return out
+}

engine/graph/state.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -13,30 +13,41 @@
 // GNU General Public License for more details.
 //
 // You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
 
 package graph
 
 import (
+	"context"
 	"fmt"
-	"os"
-	"path"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/purpleidea/mgmt/converger"
 	"github.com/purpleidea/mgmt/engine"
-	"github.com/purpleidea/mgmt/engine/event"
+	"github.com/purpleidea/mgmt/engine/local"
+	engineUtil "github.com/purpleidea/mgmt/engine/util"
 	"github.com/purpleidea/mgmt/pgraph"
-	"github.com/purpleidea/mgmt/util"
-
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 // State stores some state about the resource it is mapped to.
 type State struct {
 	// Graph is a pointer to the graph that this vertex is part of.
-	//Graph pgraph.Graph
+	Graph *pgraph.Graph
 
 	// Vertex is the pointer in the graph that this state corresponds to. It
 	// can be converted to a `Res` if necessary.
@@ -44,66 +55,92 @@ type State struct {
 	Vertex pgraph.Vertex
 
 	Program  string
+	Version  string
 	Hostname string
-	World    engine.World
+
+	//Converger *converger.Coordinator
+
+	Local *local.API
+	World engine.World
 
 	// Prefix is a unique directory prefix which can be used. It should be
 	// created if needed.
 	Prefix string
 
-	//Converger converger.Converger
-
 	// Debug turns on additional output and behaviours.
 	Debug bool
 
 	// Logf is the logging function that should be used to display messages.
 	Logf func(format string, v ...interface{})
 
-	timestamp int64 // last updated timestamp
-	isStateOK bool  // is state OK or do we need to run CheckApply ?
+	timestamp int64        // last updated timestamp
+	isStateOK *atomic.Bool // is state OK or do we need to run CheckApply ?
+	workerErr error        // did the Worker error?
 
-	// events is a channel of incoming events which is read by the Watch
-	// loop for that resource. It receives events like pause, start, and
-	// poke. The channel shuts down to signal for Watch to exit.
-	eventsChan chan *event.Msg // incoming to resource
-	eventsLock *sync.Mutex     // lock around sending and closing of events channel
-	eventsDone bool            // is channel closed?
+	mutex *sync.RWMutex // used for editing state properties
 
-	// outputChan is the channel that the engine listens on for events from
+	// doneCtx is cancelled when Watch should shut down. When any of the
+	// following channels close, it causes this to close.
+	doneCtx context.Context
+
+	// doneCtxCancel is the cancel function for doneCtx.
+	doneCtxCancel func()
+
+	// processDone is closed when the Process/CheckApply function fails
+	// permanently, and wants to cause Watch to exit.
+	processDone chan struct{}
+	// watchDone is closed when the Watch function fails permanently, and we
+	// close this to signal we should definitely exit. (Often redundant.)
+	watchDone chan struct{} // could be shared with limitDone or retryDone
+	// limitDone is closed when the Watch function fails permanently, and we
+	// close this to signal we should definitely exit. This happens inside
+	// of the limit loop of the Process section of Worker.
+	limitDone chan struct{} // could be shared with watchDone or retryDone
+	// retryDone is closed when the Watch function fails permanently, and we
+	// close this to signal we should definitely exit. This happens inside
+	// of the retry loop of the Process section of Worker.
+	retryDone chan struct{} // could be shared with watchDone or limitDone
+	// removeDone is closed when the vertexRemoveFn method asks for an exit.
+	// This happens when we're switching graphs. The switch to an "empty" is
+	// the equivalent of asking for a final shutdown.
+	removeDone chan struct{}
+	// eventsDone is closed when we shutdown the Process loop because we
+	// closed without error. In theory this shouldn't happen, but it could
+	// if Watch returns without error for some reason.
+	eventsDone chan struct{}
+
+	// eventsChan is the channel that the engine listens on for events from
 	// the Watch loop for that resource. The event is nil normally, except
 	// when events are sent on this channel from the engine. This only
 	// happens as a signaling mechanism when Watch has shutdown and we want
 	// to notify the Process loop which reads from this.
-	outputChan chan error // outgoing from resource
+	eventsChan chan error // outgoing from resource
 
-	wg   *sync.WaitGroup
-	exit *util.EasyExit
+	// pokeChan is a separate channel that the Process loop listens on to
+	// know when we might need to run Process. It never closes, and is safe
+	// to send on since it is buffered.
+	pokeChan chan struct{} // outgoing from resource
 
-	started chan struct{} // closes when it's started
-	stopped chan struct{} // closes when it's stopped
+	// paused represents if this particular res is paused or not. This is
+	// primarily used to avoid running an unnecessary Resume on the first
+	// run of this resource.
+	paused bool
+	// pauseSignal receives a message to request a pause of this resource.
+	pauseSignal chan struct{}
+	// resumeSignal receives a message to resume this resource. The channel
+	// closes when the resource is removed from the graph.
+	resumeSignal chan struct{}
 
-	starter bool // do we have an indegree of 0 ?
-	working bool // is the Main() loop running ?
+	wg *sync.WaitGroup // used for all vertex specific processes
 
-	cuid converger.UID // primary converger
-	tuid converger.UID // secondary converger
+	cuid *converger.UID // primary converger
+	tuid *converger.UID // secondary converger
 
 	init *engine.Init // a copy of the init struct passed to res Init
 }
 
 // Init initializes structures like channels.
 func (obj *State) Init() error {
-	obj.eventsChan = make(chan *event.Msg)
-	obj.eventsLock = &sync.Mutex{}
-
-	obj.outputChan = make(chan error)
-
-	obj.wg = &sync.WaitGroup{}
-	obj.exit = util.NewEasyExit()
-
-	obj.started = make(chan struct{})
-	obj.stopped = make(chan struct{})
-
 	res, isRes := obj.Vertex.(engine.Res)
 	if !isRes {
 		return fmt.Errorf("vertex is not a Res")
@@ -121,32 +158,39 @@ func (obj *State) Init() error {
 		return fmt.Errorf("the Logf function is missing")
 	}
 
+	obj.isStateOK = &atomic.Bool{}
+
+	obj.mutex = &sync.RWMutex{}
+	obj.doneCtx, obj.doneCtxCancel = context.WithCancel(context.Background())
+
+	obj.processDone = make(chan struct{})
+	obj.watchDone = make(chan struct{})
+	obj.limitDone = make(chan struct{})
+	obj.retryDone = make(chan struct{})
+	obj.removeDone = make(chan struct{})
+	obj.eventsDone = make(chan struct{})
+
+	obj.eventsChan = make(chan error)
+
+	obj.pokeChan = make(chan struct{}, 1) // must be buffered
+
+	//obj.paused = false // starts off as started
+	obj.pauseSignal = make(chan struct{})
+	obj.resumeSignal = make(chan struct{})
+
+	obj.wg = &sync.WaitGroup{}
+
 	//obj.cuid = obj.Converger.Register() // gets registered in Worker()
 	//obj.tuid = obj.Converger.Register() // gets registered in Worker()
 
 	obj.init = &engine.Init{
 		Program:  obj.Program,
+		Version:  obj.Version,
 		Hostname: obj.Hostname,
 
 		// Watch:
-		Running: func() error {
-			obj.tuid.StopTimer()
-			close(obj.started)    // this is reset in the reset func
-			obj.isStateOK = false // assume we're initially dirty
-			// optimization: skip the initial send if not a starter
-			// because we'll get poked from a starter soon anyways!
-			if !obj.starter {
-				return nil
-			}
-			return obj.event()
-		},
-		Event:  obj.event,
-		Events: obj.eventsChan,
-		Read:   obj.read,
-		Dirty: func() { // TODO: should we rename this SetDirty?
-			obj.tuid.StopTimer()
-			obj.isStateOK = false
-		},
+		Running: obj.event,
+		Event:   obj.event,
 
 		// CheckApply:
 		Refresh: func() bool {
@@ -156,33 +200,72 @@ func (obj *State) Init() error {
 			}
 			return res.Refresh()
 		},
-		Send: func(st interface{}) error {
-			res, ok := obj.Vertex.(engine.SendableRes)
-			if !ok {
-				panic("res does not support the Sendable trait")
-			}
-			// XXX: type check this
-			//expected := res.Sends()
-			//if err := XXX_TYPE_CHECK(expected, st); err != nil {
-			//	return err
-			//}
 
-			return res.Send(st) // send the struct
-		},
-		Recv: func() map[string]*engine.Send { // TODO: change this API?
-			res, ok := obj.Vertex.(engine.RecvableRes)
-			if !ok {
-				panic("res does not support the Recvable trait")
+		Send: engine.GenerateSendFunc(res),
+		Recv: engine.GenerateRecvFunc(res),
+
+		// FIXME: pass in a safe, limited query func instead?
+		// TODO: not implemented, use FilteredGraph
+		//Graph: func() *pgraph.Graph {
+		//	_, ok := obj.Vertex.(engine.CanGraphQueryRes)
+		//	if !ok {
+		//		panic("res does not support the GraphQuery trait")
+		//	}
+		//	return obj.Graph // we return in a func so it's fresh!
+		//},
+
+		FilteredGraph: func() (*pgraph.Graph, error) {
+			graph, err := pgraph.NewGraph("filtered")
+			if err != nil {
+				return nil, err
 			}
-			return res.Recv()
+
+			// filter graph and build a new one...
+			adjacency := obj.Graph.Adjacency()
+			for v1 := range adjacency {
+				// check we're allowed
+				r1, ok := v1.(engine.GraphQueryableRes)
+				if !ok {
+					continue
+				}
+				// pass in information on requestor...
+				if err := r1.GraphQueryAllowed(
+					engine.GraphQueryableOptionKind(res.Kind()),
+					engine.GraphQueryableOptionName(res.Name()),
+					// TODO: add more information...
+				); err != nil {
+					continue
+				}
+				graph.AddVertex(v1)
+
+				for v2, edge := range adjacency[v1] {
+					r2, ok := v2.(engine.GraphQueryableRes)
+					if !ok {
+						continue
+					}
+					// pass in information on requestor...
+					if err := r2.GraphQueryAllowed(
+						engine.GraphQueryableOptionKind(res.Kind()),
+						engine.GraphQueryableOptionName(res.Name()),
+						// TODO: add more information...
+					); err != nil {
+						continue
+					}
+					//graph.AddVertex(v2) // redundant
+					graph.AddEdge(v1, v2, edge)
+				}
+			}
+
+			return graph, nil // we return in a func so it's fresh!
 		},
 
+		Local:  obj.Local,
 		World:  obj.World,
 		VarDir: obj.varDir,
 
 		Debug: obj.Debug,
 		Logf: func(format string, v ...interface{}) {
-			obj.Logf("resource: "+format, v...)
+			obj.Logf(format, v...)
 		},
 	}
 
@@ -190,9 +273,15 @@ func (obj *State) Init() error {
 	if obj.Debug {
 		obj.Logf("Init(%s)", res)
 	}
+
+	// write the reverse request to the disk...
+	if err := obj.ReversalInit(); err != nil {
+		return err // TODO: test this code path...
+	}
+
 	err := res.Init(obj.init)
 	if obj.Debug {
-		obj.Logf("Init(%s): Return(%+v)", res, err)
+		obj.Logf("Init(%s): Return(%s)", res, engineUtil.CleanError(err))
 	}
 	if err != nil {
 		return errwrap.Wrapf(err, "could not Init() resource")
@@ -201,9 +290,9 @@ func (obj *State) Init() error {
 	return nil
 }
 
-// Close shuts down and performs any cleanup. This is most akin to a "post" or
+// Cleanup shuts down and performs any cleanup. This is most akin to a "post" or
 // cleanup command as the initiator for closing a vertex happens in graph sync.
-func (obj *State) Close() error {
+func (obj *State) Cleanup() error {
 	res, isRes := obj.Vertex.(engine.Res)
 	if !isRes {
 		return fmt.Errorf("vertex is not a Res")
@@ -223,231 +312,121 @@ func (obj *State) Close() error {
 	if obj.Debug {
 		obj.Logf("Close(%s)", res)
 	}
-	err := res.Close()
+
+	var reverr error
+	// clear the reverse request from the disk...
+	if err := obj.ReversalCleanup(); err != nil {
+		// TODO: test this code path...
+		// TODO: should this be an error or a warning?
+		reverr = err
+	}
+
+	reterr := res.Cleanup()
 	if obj.Debug {
-		obj.Logf("Close(%s): Return(%+v)", res, err)
+		obj.Logf("Close(%s): Return(%s)", res, engineUtil.CleanError(reterr))
 	}
 
-	return err
+	reterr = errwrap.Append(reterr, reverr)
+
+	return reterr
 }
 
-// reset is run to reset the state so that Watch can run a second time. Thus is
-// needed for the Watch retry in particular.
-func (obj *State) reset() {
-	obj.started = make(chan struct{})
-	obj.stopped = make(chan struct{})
-}
-
-// Poke sends a nil message on the outputChan. This channel is used by the
-// resource to signal a possible change. This will cause the Process loop to
-// run if it can.
+// Poke sends a notification on the poke channel. This channel is used to notify
+// the Worker to run the Process/CheckApply when it can. This is used when there
+// is a need to schedule or reschedule some work which got postponed or dropped.
+// This doesn't contain any internal synchronization primitives or wait groups,
+// callers are expected to make sure that they don't leave any of these running
+// by the time the Worker() shuts down.
 func (obj *State) Poke() {
-	// add a wait group on the vertex we're poking!
-	obj.wg.Add(1)
-	defer obj.wg.Done()
-
-	// now that we've added to the wait group, obj.outputChan won't close...
-	// so see if there's an exit signal before we release the wait group!
-	// XXX: i don't think this is necessarily happening, but maybe it is?
-	// XXX: re-write some of the engine to ensure that: "the sender closes"!
 	select {
-	case <-obj.exit.Signal():
-		return // skip sending the poke b/c we're closing
-	default:
-	}
-
-	select {
-	case obj.outputChan <- nil:
-
-	case <-obj.exit.Signal():
+	case obj.pokeChan <- struct{}{}:
+	default: // if chan is now full because more than one poke happened...
 	}
 }
 
-// Event sends a Pause or Start event to the resource. It can also be used to
-// send Poke events, but it's much more efficient to send them directly instead
-// of passing them through the resource.
-func (obj *State) Event(msg *event.Msg) {
-	// TODO: should these happen after the lock?
-	obj.wg.Add(1)
-	defer obj.wg.Done()
-
-	obj.eventsLock.Lock()
-	defer obj.eventsLock.Unlock()
-
-	if obj.eventsDone { // closing, skip events...
-		return
+// Pause pauses this resource. It must not be called on any already paused
+// resource. It will block until the resource pauses with an acknowledgment, or
+// until an exit for that resource is seen. If the latter happens it will error.
+// It must not be called concurrently with either the Resume() method or itself,
+// so only call these one at a time and alternate between the two.
+func (obj *State) Pause() error {
+	if obj.paused {
+		panic("already paused")
 	}
 
-	if msg.Kind == event.KindExit { // set this so future events don't deadlock
-		obj.Logf("exit event...")
-		obj.eventsDone = true
-		close(obj.eventsChan) // causes resource Watch loop to close
-		obj.exit.Done(nil)    // trigger exit signal to unblock some cases
+	// wait for ack (or exit signal)
+	select {
+	case obj.pauseSignal <- struct{}{}:
+		// we're paused
+
+	case <-obj.doneCtx.Done():
+		return engine.ErrClosed
+	}
+	obj.paused = true
+
+	return nil
+}
+
+// Resume unpauses this resource. It can be safely called once on a brand-new
+// resource that has just started running, without incident. It must not be
+// called concurrently with either the Pause() method or itself, so only call
+// these one at a time and alternate between the two.
+func (obj *State) Resume() {
+	// This paused check prevents unnecessary "resume" calls to the resource
+	// on its first run, since resources start in the running state!
+	if !obj.paused { // no need to unpause brand-new resources
 		return
 	}
 
 	select {
-	case obj.eventsChan <- msg:
-
-	case <-obj.exit.Signal():
+	case obj.resumeSignal <- struct{}{}:
 	}
+
+	obj.paused = false
 }
 
-// read is a helper function used inside the main select statement of resources.
-// If it returns an error, then this is a signal for the resource to exit.
-func (obj *State) read(msg *event.Msg) error {
-	switch msg.Kind {
-	case event.KindPoke:
-		return obj.event() // a poke needs to cause an event...
-	case event.KindStart:
-		return fmt.Errorf("unexpected start")
-	case event.KindPause:
-		// pass
-	case event.KindExit:
-		return engine.ErrSignalExit
+// event is a helper function to send an event to the CheckApply process loop.
+// It can be used for the initial `running` event, or any regular event. You
+// should instead use Poke() to "schedule" a new Process/CheckApply loop when
+// one might be needed. This method will block until we're unpaused and ready to
+// receive on the events channel.
+func (obj *State) event() {
+	obj.setDirty() // assume we're initially dirty
 
-	default:
-		return fmt.Errorf("unhandled event: %+v", msg.Kind)
-	}
-
-	// we're paused now
 	select {
-	case msg, ok := <-obj.eventsChan:
-		if !ok {
-			return engine.ErrWatchExit
-		}
-		switch msg.Kind {
-		case event.KindPoke:
-			return fmt.Errorf("unexpected poke")
-		case event.KindPause:
-			return fmt.Errorf("unexpected pause")
-		case event.KindStart:
-			// resumed
-			return nil
-		case event.KindExit:
-			return engine.ErrSignalExit
-
-		default:
-			return fmt.Errorf("unhandled event: %+v", msg.Kind)
-		}
+	case obj.eventsChan <- nil: // blocks! (this is unbuffered)
+		// send!
 	}
+
+	//return // implied
 }
 
-// event is a helper function to send an event from the resource Watch loop. It
-// can be used for the initial `running` event, or any regular event. If it
-// returns an error, then the Watch loop must return this error and shutdown.
-func (obj *State) event() error {
-	// loop until we sent on obj.outputChan or exit with error
-	for {
-		select {
-		// send "activity" event
-		case obj.outputChan <- nil:
-			return nil // sent event!
-
-		// make sure to keep handling incoming
-		case msg, ok := <-obj.eventsChan:
-			if !ok {
-				return engine.ErrWatchExit
-			}
-			switch msg.Kind {
-			case event.KindPoke:
-				// we're trying to send an event, so swallow the
-				// poke: it's what we wanted to have happen here
-				continue
-			case event.KindStart:
-				return fmt.Errorf("unexpected start")
-			case event.KindPause:
-				// pass
-			case event.KindExit:
-				return engine.ErrSignalExit
-
-			default:
-				return fmt.Errorf("unhandled event: %+v", msg.Kind)
-			}
-		}
-
-		// we're paused now
-		select {
-		case msg, ok := <-obj.eventsChan:
-			if !ok {
-				return engine.ErrWatchExit
-			}
-			switch msg.Kind {
-			case event.KindPoke:
-				return fmt.Errorf("unexpected poke")
-			case event.KindPause:
-				return fmt.Errorf("unexpected pause")
-			case event.KindStart:
-				// resumed
-			case event.KindExit:
-				return engine.ErrSignalExit
-
-			default:
-				return fmt.Errorf("unhandled event: %+v", msg.Kind)
-			}
-		}
-	}
-}
-
-// varDir returns the path to a working directory for the resource. It will try
-// and create the directory first, and return an error if this failed. The dir
-// should be cleaned up by the resource on Close if it wishes to discard the
-// contents. If it does not, then a future resource with the same kind and name
-// may see those contents in that directory. The resource should clean up the
-// contents before use if it is important that nothing exist. It is always
-// possible that contents could remain after an abrupt crash, so do not store
-// overly sensitive data unless you're aware of the risks.
-func (obj *State) varDir(extra string) (string, error) {
-	// Using extra adds additional dirs onto our namespace. An empty extra
-	// adds no additional directories.
-	if obj.Prefix == "" { // safety
-		return "", fmt.Errorf("the VarDir prefix is empty")
-	}
-
-	// an empty string at the end has no effect
-	p := fmt.Sprintf("%s/", path.Join(obj.Prefix, extra))
-	if err := os.MkdirAll(p, 0770); err != nil {
-		return "", errwrap.Wrapf(err, "can't create prefix in: %s", p)
-	}
-
-	// returns with a trailing slash as per the mgmt file res convention
-	return p, nil
+// setDirty marks the resource state as dirty. This signals to the engine that
+// CheckApply will have some work to do in order to converge it.
+func (obj *State) setDirty() {
+	obj.tuid.StopTimer()
+	//obj.mutex.Lock()
+	obj.isStateOK.Store(false) // concurrent write
+	//obj.mutex.Unlock()
 }
 
 // poll is a replacement for Watch when the Poll metaparameter is used.
-func (obj *State) poll(interval uint32) error {
+func (obj *State) poll(ctx context.Context, interval uint32) error {
 	// create a time.Ticker for the given interval
 	ticker := time.NewTicker(time.Duration(interval) * time.Second)
 	defer ticker.Stop()
 
-	// notify engine that we're running
-	if err := obj.init.Running(); err != nil {
-		return err // exit if requested
-	}
+	obj.init.Running() // when started, notify engine that we're running
 
-	var send = false // send event?
 	for {
 		select {
 		case <-ticker.C: // received the timer event
 			obj.init.Logf("polling...")
-			send = true
-			obj.init.Dirty() // dirty
 
-		case event, ok := <-obj.init.Events:
-			if !ok {
-				return nil
-			}
-			if err := obj.init.Read(event); err != nil {
-				return err
-			}
+		case <-ctx.Done(): // signal for shutdown request
+			return nil
 		}
 
-		// do all our event sending all together to avoid duplicate msgs
-		if send {
-			send = false
-			if err := obj.init.Event(); err != nil {
-				return err // exit if requested
-			}
-		}
+		obj.init.Event() // notify engine of an event (this can block)
 	}
 }

engine/graph/vardir.go

@@ -0,0 +1,64 @@
+// Mgmt
+// Copyright (C) James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+package graph
+
+import (
+	"fmt"
+	"os"
+	"path"
+
+	"github.com/purpleidea/mgmt/util/errwrap"
+)
+
+// varDir returns the path to a working directory for the resource. It will try
+// and create the directory first, and return an error if this failed. The dir
+// should be cleaned up by the resource on Close if it wishes to discard the
+// contents. If it does not, then a future resource with the same kind and name
+// may see those contents in that directory. The resource should clean up the
+// contents before use if it is important that nothing exist. It is always
+// possible that contents could remain after an abrupt crash, so do not store
+// overly sensitive data unless you're aware of the risks.
+func (obj *State) varDir(extra string) (string, error) {
+	// Using extra adds additional dirs onto our namespace. An empty extra
+	// adds no additional directories.
+	if obj.Prefix == "" { // safety
+		return "", fmt.Errorf("the VarDir prefix is empty")
+	}
+
+	// an empty string at the end has no effect
+	p := fmt.Sprintf("%s/", path.Join(obj.Prefix, extra))
+	// 0775 since we want children to be able to read this!
+	if err := os.MkdirAll(p, 0775); err != nil {
+		return "", errwrap.Wrapf(err, "can't create prefix in: %s", p)
+	}
+
+	// returns with a trailing slash as per the mgmt file res convention
+	return p, nil
+}

engine/graphqueryable.go

@@ -0,0 +1,82 @@
+// Mgmt
+// Copyright (C) James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+package engine
+
+// GraphQueryableRes is the interface that must be implemented if you want your
+// resource to be allowed to be queried from another resource in the graph. This
+// is done as a form of explicit authorization tracking so that we can consider
+// security aspects more easily. Ultimately, all resource code should be
+// trusted, but it's still a good idea to know if a particular resource is even
+// able to access information about another one, and if your resource doesn't
+// add the trait supporting this, then it won't be allowed.
+type GraphQueryableRes interface {
+	Res // implement everything in Res but add the additional requirements
+
+	// GraphQueryAllowed returns nil if you're allowed to query the graph.
+	GraphQueryAllowed(...GraphQueryableOption) error
+}
+
+// GraphQueryableOption is an option that can be used to specify the
+// authentication.
+type GraphQueryableOption func(*GraphQueryableOptions)
+
+// GraphQueryableOptions represents the different possible configurable options.
+type GraphQueryableOptions struct {
+	// Kind is the kind of the resource making the access.
+	Kind string
+	// Name is the name of the resource making the access.
+	Name string
+	// TODO: add more options if needed
+}
+
+// Apply is a helper function to apply a list of options to the struct. You
+// should initialize it with defaults you want, and then apply any you've
+// received like this.
+func (obj *GraphQueryableOptions) Apply(opts ...GraphQueryableOption) {
+	for _, optionFunc := range opts { // apply the options
+		optionFunc(obj)
+	}
+}
+
+// GraphQueryableOptionKind tells the GraphQueryAllowed function what the
+// resource kind is.
+func GraphQueryableOptionKind(kind string) GraphQueryableOption {
+	return func(gqo *GraphQueryableOptions) {
+		gqo.Kind = kind
+	}
+}
+
+// GraphQueryableOptionName tells the GraphQueryAllowed function what the
+// resource name is.
+func GraphQueryableOptionName(name string) GraphQueryableOption {
+	return func(gqo *GraphQueryableOptions) {
+		gqo.Name = name
+	}
+}