lib: Update the help text

Give some longer descriptions so they show up nicely for the user.

.github/FUNDING.yml

@@ -0,0 +1,5 @@
+# You can add one username per supported platform and one custom link.
+custom: "https://paypal.me/purpleidea"
+github: purpleidea
+liberapay: purpleidea
+patreon: purpleidea

.github/settings.yml

@@ -68,6 +68,8 @@ labels:
     color: e11d21
   - name: question
     color: cc317c
+  - name: needinfo
+    color: fbca04
   - name: wontfix
     color: ffffff
 #  - name: first-timers-only

.github/workflows/test.yaml

@@ -0,0 +1,70 @@
+# Docs: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+
+# If the name is omitted, it uses the filename instead.
+#name: Test
+on:
+  # Run on all pull requests.
+  pull_request:
+    #branches:
+    #- master
+  # Run on all pushes.
+  push:
+  # Run daily at 4am.
+  schedule:
+  - cron: 0 4 * * *
+
+jobs:
+  maketest:
+    name: Test (${{ matrix.test_block }}) on ${{ matrix.os }} with golang ${{ matrix.golang_version }}
+    runs-on: ${{ matrix.os }}
+    env:
+      GOPATH: /home/runner/work/mgmt/mgmt/go
+    strategy:
+      matrix:
+        # TODO: Add tip when it's supported: https://github.com/actions/setup-go/issues/21
+        os:
+          - ubuntu-latest
+          # macos tests are currently failing in CI
+          #- macos-latest
+        golang_version:
+          # TODO: add 1.15.x and tip
+          # minimum required and latest published go_version
+          #- 1.13
+          - 1.15
+        test_block:
+          - basic
+          - shell
+          - race
+      #fail-fast: false
+
+    steps:
+      # Do not shallow fetch, will fail when building bindata/
+      # The path can't be absolute, so we need to move it to the
+      # expected location later.
+      - name: Clone mgmt
+        uses: actions/checkout@v2
+        with:
+          submodules: recursive
+          fetch-depth: 0
+          path: ./go/src/github.com/purpleidea/mgmt
+
+      - name: Install Go ${{ matrix.golang_version }}
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.golang_version }}
+
+      # Install & configure ruby, fixes gem permissions error
+      - name: Install Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: head
+
+      - name: Install dependencies
+        working-directory: ./go/src/github.com/purpleidea/mgmt
+        run: |
+          make deps
+
+      - name: Run test
+        working-directory: ./go/src/github.com/purpleidea/mgmt
+        run: |
+          TEST_BLOCK="${{ matrix.test_block }}" make test

.gitignore

@@ -17,3 +17,5 @@ rpmbuild/
 releases/
 # vim swap files
 .*.sw[op]
+# prevent `echo foo 2>1` typo errors by making this file read-only
+1

.gitmodules

@@ -1,5 +1,5 @@
 [submodule "vendor/github.com/coreos/etcd"]
-	path = vendor/github.com/coreos/etcd
+	path = vendor/go.etcd.io/etcd
 	url = https://github.com/coreos/etcd/
 [submodule "vendor/google.golang.org/grpc"]
 	path = vendor/google.golang.org/grpc
@@ -28,6 +28,12 @@
 [submodule "vendor/github.com/purpleidea/distribution"]
 	path = vendor/github.com/docker/distribution
 	url = https://github.com/purpleidea/distribution
-[submodule "vendor/github.com/purpleidea/go-connections"]
-	path = vendor/github.com/docker/go-connections
-	url = https://github.com/docker/go-connections
+[submodule "vendor/github.com/hashicorp/go-multierror"]
+	path = vendor/github.com/hashicorp/go-multierror
+	url = https://github.com/hashicorp/go-multierror
+[submodule "vendor/github.com/containerd/containerd"]
+	path = vendor/github.com/containerd/containerd
+	url = https://github.com/purpleidea/containerd
+[submodule "vendor/github.com/hashicorp/consul"]
+	path = vendor/github.com/hashicorp/consul
+	url = https://github.com/hashicorp/consul/

.travis.yml

@@ -24,21 +24,21 @@ install: 'make deps'
 matrix:
   fast_finish: false
   allow_failures:
-  - go: 1.11.x
+  - go: 1.14.x
   - go: tip
   - os: osx
   # include only one build for osx for a quicker build as the nr. of these runners are sparse
   include:
   - name: "basic tests"
-    go: 1.10.x
+    go: 1.13.x
     env: TEST_BLOCK=basic
   - name: "shell tests"
-    go: 1.10.x
+    go: 1.13.x
     env: TEST_BLOCK=shell
   - name: "race tests"
-    go: 1.10.x
+    go: 1.13.x
     env: TEST_BLOCK=race
-  - go: 1.11.x
+  - go: 1.14.x
   - go: tip
   - os: osx
 script: 'TEST_BLOCK="$TEST_BLOCK" make test'
@@ -47,8 +47,8 @@ script: 'TEST_BLOCK="$TEST_BLOCK" make test'
 # with a value of: irc.freenode.net#mgmtconfig to eliminate noise from forks...
 notifications:
   irc:
-    channels:
-      - secure: htcuWAczm3C1zKC9vUfdRzhIXM1vtF+q0cLlQFXK1IQQlk693/pM30Mmf2L/9V2DVDeps+GyLdip0ARXD1DZEJV0lK+Ca1qbHdFP1r4Xv6l5+jaDb5Y88YU5LI8K758QShiZJojuQ1aO2j8xmmt9V0/5y5QwlpPeHbKYBOFPBX3HvlT9DhvwZNKGhBb4qJOEaPVOwq9IkN3DyQ456MHcJ3q3vF9Lb440uTuLsJNof2AbYZH8ZIHCSG2N8tBj2qhJOpWQboYtQJzE2pRaGkGBL4kYcHZSZMXX8sl4cBM1vx/IRUkvBxJUpLJz2gn/eRI+/gr59juZE2K0+FOLlx9dLnX626Y9xSViopBI6JsIoHJDqNC7aGaF2qaYulGYN65VNKVqmghjgt6JLmmiKeH10hYrJMMvt2rms8l4+5iwmCwXvhH/WU9edzk2p5wqERMnostJFEJib0zI3yzLoF0sdJs+veKtagzfayY2d2l7hlmt951IpqqVWldVgWUcQKVvi8gmRarbwFlK+5D7BEnkUDcLNly/cqf7BgEeX6YfF+FiR4pgfOhYvGCD+2q91NgWQXHBCxbyN0be1TVdkXD94f0Lkn94VyEJJ+PkPlG+rPgFwGcjqN4oEGkJeJmES2If05q2Ms1dJLwYQDL3+Py4lNMSdSWj24TzlFVhtwHepuw=
+    #channels:
+    #  - secure: htcuWAczm3C1zKC9vUfdRzhIXM1vtF+q0cLlQFXK1IQQlk693/pM30Mmf2L/9V2DVDeps+GyLdip0ARXD1DZEJV0lK+Ca1qbHdFP1r4Xv6l5+jaDb5Y88YU5LI8K758QShiZJojuQ1aO2j8xmmt9V0/5y5QwlpPeHbKYBOFPBX3HvlT9DhvwZNKGhBb4qJOEaPVOwq9IkN3DyQ456MHcJ3q3vF9Lb440uTuLsJNof2AbYZH8ZIHCSG2N8tBj2qhJOpWQboYtQJzE2pRaGkGBL4kYcHZSZMXX8sl4cBM1vx/IRUkvBxJUpLJz2gn/eRI+/gr59juZE2K0+FOLlx9dLnX626Y9xSViopBI6JsIoHJDqNC7aGaF2qaYulGYN65VNKVqmghjgt6JLmmiKeH10hYrJMMvt2rms8l4+5iwmCwXvhH/WU9edzk2p5wqERMnostJFEJib0zI3yzLoF0sdJs+veKtagzfayY2d2l7hlmt951IpqqVWldVgWUcQKVvi8gmRarbwFlK+5D7BEnkUDcLNly/cqf7BgEeX6YfF+FiR4pgfOhYvGCD+2q91NgWQXHBCxbyN0be1TVdkXD94f0Lkn94VyEJJ+PkPlG+rPgFwGcjqN4oEGkJeJmES2If05q2Ms1dJLwYQDL3+Py4lNMSdSWj24TzlFVhtwHepuw=
     template:
       - "%{repository} (%{commit}: %{author}): %{message}"
       - "More info : %{build_url}"

AUTHORS

@@ -6,6 +6,7 @@ This list is sorted alphabetically by first name.
 
 Felix Frank
 James Shubin
+Joe Groocock
 Johan Bloemberg
 Jonathan Gold
 Julien Pivotto

COPYRIGHT

@@ -1,5 +1,5 @@
 Mgmt
-Copyright (C) 2013-2018+ James Shubin and the project contributors
+Copyright (C) 2013-2021+ James Shubin and the project contributors
 Written by James Shubin <james@shubin.ca> and the project contributors
 
 This program is free software: you can redistribute it and/or modify

Makefile

@@ -1,5 +1,5 @@
 # Mgmt
-# Copyright (C) 2013-2018+ James Shubin and the project contributors
+# Copyright (C) 2013-2021+ James Shubin and the project contributors
 # Written by James Shubin <james@shubin.ca> and the project contributors
 #
 # This program is free software: you can redistribute it and/or modify
@@ -16,11 +16,16 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 SHELL = /usr/bin/env bash
-.PHONY: all art cleanart version program lang path deps run race bindata generate build build-debug crossbuild clean test gofmt yamlfmt format docs rpmbuild mkdirs rpm srpm spec tar upload upload-sources upload-srpms upload-rpms copr tag release funcgen
+.PHONY: all art cleanart version program lang path deps run race bindata generate build build-debug crossbuild clean test gofmt yamlfmt format docs
+.PHONY: rpmbuild mkdirs rpm srpm spec tar upload upload-sources upload-srpms upload-rpms upload-releases copr tag
+.PHONY: mkosi mkosi_fedora-30 mkosi_fedora-29 mkosi_centos-7 mkosi_debian-10 mkosi_ubuntu-bionic mkosi_archlinux
+.PHONY: release releases_path release_fedora-30 release_fedora-29 release_centos-7 release_debian-10 release_ubuntu-bionic release_archlinux
+.PHONY: funcgen
 .SILENT: clean bindata
 
 # a large amount of output from this `find`, can cause `make` to be much slower!
 GO_FILES := $(shell find * -name '*.go' -not -path 'old/*' -not -path 'tmp/*')
+MCL_FILES := $(shell find lang/funcs/ -name '*.mcl' -not -path 'old/*' -not -path 'tmp/*')
 
 SVERSION := $(or $(SVERSION),$(shell git describe --match '[0-9]*\.[0-9]*\.[0-9]*' --tags --dirty --always))
 VERSION := $(or $(VERSION),$(shell git describe --match '[0-9]*\.[0-9]*\.[0-9]*' --tags --abbrev=0))
@@ -48,9 +53,26 @@ GOOSARCHES ?= linux/amd64 linux/ppc64 linux/ppc64le linux/arm64 darwin/amd64
 GOHOSTOS = $(shell go env GOHOSTOS)
 GOHOSTARCH = $(shell go env GOHOSTARCH)
 
-RPM_PKG = releases/$(VERSION)/rpm/mgmt-$(VERSION)-1.x86_64.rpm
-DEB_PKG = releases/$(VERSION)/deb/mgmt_$(VERSION)_amd64.deb
-PACMAN_PKG = releases/$(VERSION)/pacman/mgmt-$(VERSION)-1-x86_64.pkg.tar.xz
+TOKEN_FEDORA-30 = fedora-30
+TOKEN_FEDORA-29 = fedora-29
+TOKEN_CENTOS-7 = centos-7
+TOKEN_DEBIAN-10 = debian-10
+TOKEN_UBUNTU-BIONIC = ubuntu-bionic
+TOKEN_ARCHLINUX = archlinux
+
+FILE_FEDORA-30 = mgmt-$(TOKEN_FEDORA-30)-$(VERSION)-1.x86_64.rpm
+FILE_FEDORA-29 = mgmt-$(TOKEN_FEDORA-29)-$(VERSION)-1.x86_64.rpm
+FILE_CENTOS-7 = mgmt-$(TOKEN_CENTOS-7)-$(VERSION)-1.x86_64.rpm
+FILE_DEBIAN-10 = mgmt_$(TOKEN_DEBIAN-10)_$(VERSION)_amd64.deb
+FILE_UBUNTU-BIONIC = mgmt_$(TOKEN_UBUNTU-BIONIC)_$(VERSION)_amd64.deb
+FILE_ARCHLINUX = mgmt-$(TOKEN_ARCHLINUX)-$(VERSION)-1-x86_64.pkg.tar.xz
+
+PKG_FEDORA-30 = releases/$(VERSION)/$(TOKEN_FEDORA-30)/$(FILE_FEDORA-30)
+PKG_FEDORA-29 = releases/$(VERSION)/$(TOKEN_FEDORA-29)/$(FILE_FEDORA-29)
+PKG_CENTOS-7 = releases/$(VERSION)/$(TOKEN_CENTOS-7)/$(FILE_CENTOS-7)
+PKG_DEBIAN-10 = releases/$(VERSION)/$(TOKEN_DEBIAN-10)/$(FILE_DEBIAN-10)
+PKG_UBUNTU-BIONIC = releases/$(VERSION)/$(TOKEN_UBUNTU-BIONIC)/$(FILE_UBUNTU-BIONIC)
+PKG_ARCHLINUX = releases/$(VERSION)/$(TOKEN_ARCHLINUX)/$(FILE_ARCHLINUX)
 
 SHA256SUMS = releases/$(VERSION)/SHA256SUMS
 SHA256SUMS_ASC = $(SHA256SUMS).asc
@@ -131,7 +153,7 @@ lang: ## generates the lexer/parser for the language frontend
 $(PROGRAM): build/mgmt-${GOHOSTOS}-${GOHOSTARCH} ## build an mgmt binary for current host os/arch
 	cp -a $< $@
 
-$(PROGRAM).static: $(GO_FILES)
+$(PROGRAM).static: $(GO_FILES) $(MCL_FILES)
 	@echo "Building: $(PROGRAM).static, version: $(SVERSION)..."
 	go generate
 	go build -a -installsuffix cgo -tags netgo -ldflags '-extldflags "-static" -X main.program=$(PROGRAM) -X main.version=$(SVERSION) -s -w' -o $(PROGRAM).static $(BUILD_FLAGS);
@@ -146,24 +168,20 @@ build-debug: $(PROGRAM)
 # extract os and arch from target pattern
 GOOS=$(firstword $(subst -, ,$*))
 GOARCH=$(lastword $(subst -, ,$*))
-build/mgmt-%: $(GO_FILES) | bindata lang funcgen
+build/mgmt-%: $(GO_FILES) $(MCL_FILES) | bindata lang funcgen
 	@echo "Building: $(PROGRAM), os/arch: $*, version: $(SVERSION)..."
-	@# reassigning GOOS and GOARCH to make build command copy/pastable
-	@# go 1.10 requires specifying the package for ldflags
-	@if go version | grep -qE 'go1.9'; then \
-		time env GOOS=${GOOS} GOARCH=${GOARCH} go build -i -ldflags "-X main.program=$(PROGRAM) -X main.version=$(SVERSION) ${LDFLAGS}" -o $@ $(BUILD_FLAGS); \
-	else \
-		time env GOOS=${GOOS} GOARCH=${GOARCH} go build -i -ldflags=$(PKGNAME)="-X main.program=$(PROGRAM) -X main.version=$(SVERSION) ${LDFLAGS}" -o $@ $(BUILD_FLAGS); \
-	fi
+	@time env GOOS=${GOOS} GOARCH=${GOARCH} go build -i -ldflags=$(PKGNAME)="-X main.program=$(PROGRAM) -X main.version=$(SVERSION) ${LDFLAGS}" -o $@ $(BUILD_FLAGS)
 
 # create a list of binary file names to use as make targets
 crossbuild_targets = $(addprefix build/mgmt-,$(subst /,-,${GOOSARCHES}))
 crossbuild: ${crossbuild_targets}
 
 clean: ## clean things up
+	$(MAKE) --quiet -C test clean
 	$(MAKE) --quiet -C bindata clean
 	$(MAKE) --quiet -C lang/funcs clean
 	$(MAKE) --quiet -C lang clean
+	$(MAKE) --quiet -C misc/mkosi clean
 	rm -f lang/funcs/core/generated_funcs.go || true
 	rm -f lang/funcs/core/generated_funcs_test.go || true
 	[ ! -e $(PROGRAM) ] || rm $(PROGRAM)
@@ -173,6 +191,8 @@ clean: ## clean things up
 	rm -f build/mgmt-*
 
 test: build ## run tests
+	@# recursively run make in child dir named test
+	@$(MAKE) --quiet -C test
 	./test.sh
 
 # create all test targets for make tab completion (eg: make test-gofmt)
@@ -188,8 +208,8 @@ $(addprefix test-shell-,${test_shell}): test-shell-%: build
 
 gofmt:
 	# TODO: remove gofmt once goimports has a -s option
-	find . -maxdepth 6 -type f -name '*.go' -not -path './old/*' -not -path './tmp/*' -not -path './vendor/*' -exec gofmt -s -w {} \;
-	find . -maxdepth 6 -type f -name '*.go' -not -path './old/*' -not -path './tmp/*' -not -path './vendor/*' -exec goimports -w {} \;
+	find . -maxdepth 9 -type f -name '*.go' -not -path './old/*' -not -path './tmp/*' -not -path './vendor/*' -exec gofmt -s -w {} \;
+	find . -maxdepth 9 -type f -name '*.go' -not -path './old/*' -not -path './tmp/*' -not -path './vendor/*' -exec goimports -w {} \;
 
 yamlfmt:
 	find . -maxdepth 3 -type f -name '*.yaml' -not -path './old/*' -not -path './tmp/*' -not -path './omv.yaml' -exec ruby -e "require 'yaml'; x=YAML.load_file('{}').to_yaml.each_line.map(&:rstrip).join(10.chr)+10.chr; File.open('{}', 'w').write x" \;
@@ -326,6 +346,10 @@ upload-rpms: rpmbuild/RPMS/ rpmbuild/RPMS/SHA256SUMS rpmbuild/RPMS/SHA256SUMS.as
 		rsync -avz --prune-empty-dirs rpmbuild/RPMS/ $(SERVER):$(REMOTE_PATH)/RPMS/; \
 	fi
 
+upload-releases:
+	echo Running releases/ upload...
+	rsync -avz --exclude '.mkdir' --exclude 'mgmt-release.url' releases/ $(SERVER):$(REMOTE_PATH)/releases/
+
 #
 #	copr build
 #
@@ -338,18 +362,63 @@ copr: upload-srpms ## build in copr
 tag: ## tags a new release
 	./misc/tag.sh
 
+#
+#	mkosi
+#
+mkosi: mkosi_fedora-30 mkosi_fedora-29 mkosi_centos-7 mkosi_debian-10 mkosi_ubuntu-bionic mkosi_archlinux ## builds distro packages via mkosi
+
+mkosi_fedora-30: releases/$(VERSION)/.mkdir
+	@title='$@' ; echo "Generating: $${title#'mkosi_'} via mkosi..."
+	@title='$@' ; distro=$${title#'mkosi_'} ; ./misc/mkosi/make.sh $${distro} `realpath "releases/$(VERSION)/"`
+
+mkosi_fedora-29: releases/$(VERSION)/.mkdir
+	@title='$@' ; echo "Generating: $${title#'mkosi_'} via mkosi..."
+	@title='$@' ; distro=$${title#'mkosi_'} ; ./misc/mkosi/make.sh $${distro} `realpath "releases/$(VERSION)/"`
+
+mkosi_centos-7: releases/$(VERSION)/.mkdir
+	@title='$@' ; echo "Generating: $${title#'mkosi_'} via mkosi..."
+	@title='$@' ; distro=$${title#'mkosi_'} ; ./misc/mkosi/make.sh $${distro} `realpath "releases/$(VERSION)/"`
+
+mkosi_debian-10: releases/$(VERSION)/.mkdir
+	@title='$@' ; echo "Generating: $${title#'mkosi_'} via mkosi..."
+	@title='$@' ; distro=$${title#'mkosi_'} ; ./misc/mkosi/make.sh $${distro} `realpath "releases/$(VERSION)/"`
+
+mkosi_ubuntu-bionic: releases/$(VERSION)/.mkdir
+	@title='$@' ; echo "Generating: $${title#'mkosi_'} via mkosi..."
+	@title='$@' ; distro=$${title#'mkosi_'} ; ./misc/mkosi/make.sh $${distro} `realpath "releases/$(VERSION)/"`
+
+mkosi_archlinux: releases/$(VERSION)/.mkdir
+	@title='$@' ; echo "Generating: $${title#'mkosi_'} via mkosi..."
+	@title='$@' ; distro=$${title#'mkosi_'} ; ./misc/mkosi/make.sh $${distro} `realpath "releases/$(VERSION)/"`
+
 #
 #	release
 #
 release: releases/$(VERSION)/mgmt-release.url ## generates and uploads a release
 
-releases/$(VERSION)/mgmt-release.url: $(RPM_PKG) $(DEB_PKG) $(PACMAN_PKG) $(SHA256SUMS_ASC)
+releases_path:
+	@#Don't put any other output or dependencies in here or they'll show!
+	@echo "releases/$(VERSION)/"
+
+release_fedora-30: $(PKG_FEDORA-30)
+release_fedora-29: $(PKG_FEDORA-29)
+release_centos-7: $(PKG_CENTOS-7)
+release_debian-10: $(PKG_DEBIAN-10)
+release_ubuntu-bionic: $(PKG_UBUNTU-BIONIC)
+release_archlinux: $(PKG_ARCHLINUX)
+
+releases/$(VERSION)/mgmt-release.url: $(PKG_FEDORA-30) $(PKG_FEDORA-29) $(PKG_CENTOS-7) $(PKG_DEBIAN-10) $(PKG_UBUNTU-BIONIC) $(PKG_ARCHLINUX) $(SHA256SUMS_ASC)
+	@echo "Pushing git tag $(VERSION) to origin..."
+	git push origin $(VERSION)
 	@echo "Creating github release..."
 	hub release create \
 		-F <( echo -e "$(VERSION)\n";echo "Verify the signatures of all packages before you use them. The signing key can be downloaded from https://purpleidea.com/contact/#pgp-key to verify the release." ) \
-		-a $(RPM_PKG) \
-		-a $(DEB_PKG) \
-		-a $(PACMAN_PKG) \
+		-a $(PKG_FEDORA-30) \
+		-a $(PKG_FEDORA-29) \
+		-a $(PKG_CENTOS-7) \
+		-a $(PKG_DEBIAN-10) \
+		-a $(PKG_UBUNTU-BIONIC) \
+		-a $(PKG_ARCHLINUX) \
 		-a $(SHA256SUMS_ASC) \
 		$(VERSION) \
 		> releases/$(VERSION)/mgmt-release.url \
@@ -357,32 +426,56 @@ releases/$(VERSION)/mgmt-release.url: $(RPM_PKG) $(DEB_PKG) $(PACMAN_PKG) $(SHA2
 		|| rm -f releases/$(VERSION)/mgmt-release.url
 
 releases/$(VERSION)/.mkdir:
-	mkdir -p releases/$(VERSION)/{deb,rpm,pacman}/ && touch releases/$(VERSION)/.mkdir
+	mkdir -p releases/$(VERSION)/{$(TOKEN_FEDORA-30),$(TOKEN_FEDORA-29),$(TOKEN_CENTOS-7),$(TOKEN_DEBIAN-10),$(TOKEN_UBUNTU-BIONIC),$(TOKEN_ARCHLINUX)}/ && touch releases/$(VERSION)/.mkdir
 
-releases/$(VERSION)/rpm/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir
-	@echo "Generating: rpm changelog..."
-	./misc/make-rpm-changelog.sh $(VERSION)
+releases/$(VERSION)/$(TOKEN_FEDORA-30)/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Generating: $${distro} changelog..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/make-rpm-changelog.sh "$${distro}" $(VERSION)
 
-$(RPM_PKG): releases/$(VERSION)/rpm/changelog
-	@echo "Building: rpm package..."
-	./misc/fpm-pack.sh rpm $(VERSION) libvirt-devel augeas-devel
+$(PKG_FEDORA-30): releases/$(VERSION)/$(TOKEN_FEDORA-30)/changelog
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Building: $${distro} package..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/fpm-pack.sh $${distro} $(VERSION) "$(FILE_FEDORA-30)" libvirt-devel augeas-devel
 
-releases/$(VERSION)/deb/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir
-	@echo "Generating: deb changelog..."
-	./misc/make-deb-changelog.sh $(VERSION)
+releases/$(VERSION)/$(TOKEN_FEDORA-29)/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Generating: $${distro} changelog..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/make-rpm-changelog.sh "$${distro}" $(VERSION)
 
-$(DEB_PKG): releases/$(VERSION)/deb/changelog
-	@echo "Building: deb package..."
-	./misc/fpm-pack.sh deb $(VERSION) libvirt-dev libaugeas-dev
+$(PKG_FEDORA-29): releases/$(VERSION)/$(TOKEN_FEDORA-29)/changelog
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Building: $${distro} package..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/fpm-pack.sh $${distro} $(VERSION) "$(FILE_FEDORA-29)" libvirt-devel augeas-devel
 
-$(PACMAN_PKG): $(PROGRAM) releases/$(VERSION)/.mkdir
-	@echo "Building: pacman package..."
-	./misc/fpm-pack.sh pacman $(VERSION) libvirt augeas
+releases/$(VERSION)/$(TOKEN_CENTOS-7)/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Generating: $${distro} changelog..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/make-rpm-changelog.sh "$${distro}" $(VERSION)
 
-$(SHA256SUMS): $(RPM_PKG) $(DEB_PKG) $(PACMAN_PKG)
+$(PKG_CENTOS-7): releases/$(VERSION)/$(TOKEN_CENTOS-7)/changelog
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Building: $${distro} package..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/fpm-pack.sh $${distro} $(VERSION) "$(FILE_CENTOS-7)" libvirt-devel augeas-devel
+
+releases/$(VERSION)/$(TOKEN_DEBIAN-10)/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Generating: $${distro} changelog..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/make-deb-changelog.sh "$${distro}" $(VERSION)
+
+$(PKG_DEBIAN-10): releases/$(VERSION)/$(TOKEN_DEBIAN-10)/changelog
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Building: $${distro} package..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/fpm-pack.sh $${distro} $(VERSION) "$(FILE_DEBIAN-10)" libvirt-dev libaugeas-dev
+
+releases/$(VERSION)/$(TOKEN_UBUNTU-BIONIC)/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Generating: $${distro} changelog..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/make-deb-changelog.sh "$${distro}" $(VERSION)
+
+$(PKG_UBUNTU-BIONIC): releases/$(VERSION)/$(TOKEN_UBUNTU-BIONIC)/changelog
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Building: $${distro} package..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/fpm-pack.sh $${distro} $(VERSION) "$(FILE_UBUNTU-BIONIC)" libvirt-dev libaugeas-dev
+
+$(PKG_ARCHLINUX): $(PROGRAM) releases/$(VERSION)/.mkdir
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Building: $${distro} package..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/fpm-pack.sh $${distro} $(VERSION) "$(FILE_ARCHLINUX)" libvirt augeas
+
+$(SHA256SUMS): $(PKG_FEDORA-30) $(PKG_FEDORA-29) $(PKG_CENTOS-7) $(PKG_DEBIAN-10) $(PKG_UBUNTU-BIONIC) $(PKG_ARCHLINUX)
 	@# remove the directory separator in the SHA256SUMS file
 	@echo "Generating: sha256 sum..."
-	sha256sum $(RPM_PKG) $(DEB_PKG) $(PACMAN_PKG) | awk -F '/| ' '{print $$1"  "$$6}' > $(SHA256SUMS)
+	sha256sum $(PKG_FEDORA-30) $(PKG_FEDORA-29) $(PKG_CENTOS-7) $(PKG_DEBIAN-10) $(PKG_UBUNTU-BIONIC) $(PKG_ARCHLINUX) | awk -F '/| ' '{print $$1"  "$$6}' > $(SHA256SUMS)
 
 $(SHA256SUMS_ASC): $(SHA256SUMS)
 	@echo "Signing sha256 sum..."
@@ -408,14 +501,10 @@ help: ## show this help screen
 		awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
 	@echo ''
 
-funcgen: lang/funcs/core/generated_funcs_test.go lang/funcs/core/generated_funcs.go
-
-lang/funcs/core/generated_funcs_test.go: lang/funcs/funcgen/*.go lang/funcs/core/funcgen.yaml lang/funcs/funcgen/templates/generated_funcs_test.go.tpl
-	@echo "Generating: funcs test..."
-	@go run lang/funcs/funcgen/*.go -templates lang/funcs/funcgen/templates/generated_funcs_test.go.tpl 2>/dev/null
+funcgen: lang/funcs/core/generated_funcs.go
 
 lang/funcs/core/generated_funcs.go: lang/funcs/funcgen/*.go lang/funcs/core/funcgen.yaml lang/funcs/funcgen/templates/generated_funcs.go.tpl
 	@echo "Generating: funcs..."
-	@go run lang/funcs/funcgen/*.go -templates lang/funcs/funcgen/templates/generated_funcs.go.tpl 2>/dev/null
+	@go run `find lang/funcs/funcgen/ -maxdepth 1 -type f -name '*.go' -not -name '*_test.go'` -templates=lang/funcs/funcgen/templates/generated_funcs.go.tpl >/dev/null
 
 # vim: ts=8

README.md

@@ -4,18 +4,69 @@
 
 [![Go Report Card](https://goreportcard.com/badge/github.com/purpleidea/mgmt?style=flat-square)](https://goreportcard.com/report/github.com/purpleidea/mgmt)
 [![Build Status](https://img.shields.io/travis/purpleidea/mgmt/master.svg?style=flat-square)](http://travis-ci.org/purpleidea/mgmt)
+[![Build Status](https://github.com/purpleidea/mgmt/workflows/.github/workflows/test.yaml/badge.svg)](https://github.com/purpleidea/mgmt/actions/)
 [![GoDoc](https://img.shields.io/badge/godoc-reference-5272B4.svg?style=flat-square)](https://godoc.org/github.com/purpleidea/mgmt)
-[![IRC](https://img.shields.io/badge/irc-%23mgmtconfig-orange.svg?style=flat-square)](https://webchat.freenode.net/?channels=#mgmtconfig)
+[![IRC](https://img.shields.io/badge/irc-%23mgmtconfig-orange.svg?style=flat-square)](https://web.libera.chat/?channels=#mgmtconfig)
 [![Patreon](https://img.shields.io/badge/patreon-donate-yellow.svg?style=flat-square)](https://www.patreon.com/purpleidea)
 [![Liberapay](https://img.shields.io/badge/liberapay-donate-yellow.svg?style=flat-square)](https://liberapay.com/purpleidea/donate)
 
+## About:
+
+`Mgmt` is a real-time automation tool. It is familiar to existing configuration
+management software, but is drastically more powerful as it can allow you to
+build real-time, closed-loop feedback systems, in a very safe way, and with a
+surprisingly small amout of our `mcl` code. For example, the following code will
+ensure that your file server is set to read-only when it's friday.
+
+```mcl
+import "datetime"
+$is_friday = datetime.weekday(datetime.now()) == "friday"
+file "/srv/files/" {
+	state => $const.res.file.state.exists,
+	mode => if $is_friday { # this updates the mode, the instant it changes!
+		"0550"
+	} else {
+		"0770"
+	},
+}
+```
+
+It can run continuously, intermittently, or on-demand, and in the first case, it
+will guarantee that your system is always in the desired state for that instant!
+In this mode it can run as a decentralized cluster of agents across your
+network, each exchanging information with the others in real-time, to respond to
+your changing needs. For example, if you want to ensure that some resource runs
+on a maximum of two hosts in your cluster, you can specify that as well:
+
+```mcl
+import "sys"
+import "world"
+
+# we'll set a few scheduling options:
+$opts = struct{strategy => "rr", max => 2, ttl => 10,}
+
+# schedule in a particular namespace with options:
+$set = world.schedule("xsched", $opts)
+
+if sys.hostname() in $set {
+	# use your imagination to put something more complex right here...
+	print "i got scheduled" {} # this will run on the chosen machines
+}
+```
+
+As you add and remove hosts from the cluster, the real-time `schedule` function
+will dynamically pick up to two hosts from the available pool. These specific
+functions aren't intrinsic to the core design, and new ones can be easily added.
+
+Please read on if you'd like to learn more...
+
 ## Community:
 
 Come join us in the `mgmt` community!
 
 | Medium | Link |
 |---|---|
-| IRC | [#mgmtconfig](https://webchat.freenode.net/?channels=#mgmtconfig) on Freenode |
+| IRC | [#mgmtconfig](https://web.libera.chat/?channels=#mgmtconfig) on Libera.Chat |
 | Twitter | [@mgmtconfig](https://twitter.com/mgmtconfig) & [#mgmtconfig](https://twitter.com/hashtag/mgmtconfig) |
 | Mailing list | [mgmtconfig-list@redhat.com](https://www.redhat.com/mailman/listinfo/mgmtconfig-list) |
 | Patreon | [purpleidea](https://www.patreon.com/purpleidea) on Patreon |
@@ -30,7 +81,7 @@ approach. The project contains an engine and a language.
 
 Mgmt is a fairly new project. It is usable today, but not yet feature complete.
 With your help you'll be able to influence our design and get us to 1.0 sooner!
-Interested developers should read the [quick start guide](docs/quick-start-guide.md).
+Interested users should read the [quick start guide](docs/quick-start-guide.md).
 
 ## Documentation:
 
@@ -38,7 +89,7 @@ Please read, enjoy and help improve our documentation!
 
 | Documentation | Additional Notes |
 |---|---|
-| [quick start guide](docs/quick-start-guide.md) | for mgmt developers |
+| [quick start guide](docs/quick-start-guide.md) | for everyone |
 | [frequently asked questions](docs/faq.md) | for everyone |
 | [general documentation](docs/documentation.md) | for everyone |
 | [language guide](docs/language-guide.md) | for everyone |
@@ -57,22 +108,18 @@ If you have a well phrased question that might benefit others, consider asking
 it by sending a patch to the [FAQ](docs/faq.md) section. I'll merge your
 question, and a patch with the answer!
 
-## Roadmap:
+## Get involved:
 
 Feel free to grab one of the straightforward [#mgmtlove](https://github.com/purpleidea/mgmt/labels/mgmtlove)
 issues if you're a first time contributor to the project or if you're unsure
-about what to hack on!
-Please see: [TODO.md](TODO.md) for a list of upcoming work and TODO items.
-Please get involved by working on one of these items or by suggesting something
-else!
+about what to hack on! Please get involved by working on one of these items or
+by suggesting something else! There are some lower priority issues and harder
+issues available in our [TODO](TODO.md) file. Please have a look.
 
 ## Bugs:
 
 Please set the `DEBUG` constant in [main.go](https://github.com/purpleidea/mgmt/blob/master/main.go)
 to `true`, and post the logs when you report the [issue](https://github.com/purpleidea/mgmt/issues).
-Bonus points if you provide a [shell](https://github.com/purpleidea/mgmt/tree/master/test/shell)
-or [OMV](https://github.com/purpleidea/mgmt/tree/master/test/omv) reproducible
-test case.
 Feel free to read my article on [debugging golang programs](https://purpleidea.com/blog/2016/02/15/debugging-golang-programs/).
 
 ## Patches:

TODO.md

@@ -1,10 +1,18 @@
 # TODO
 
-If you're looking for something to do, look here!
-Let us know if you're working on one of the items.
-If you'd like something to work on, ping @purpleidea and I'll create an issue
-tailored especially for you! Just let me know your approximate golang skill
-level and how many hours you'd like to spend on the patch.
+Here is a TODO list of longstanding items that are either lower-priority, or
+more involved in terms of time, skill-level, and/or motivation.
+
+Please have a look, and let us know if you're working on one of the items. It's
+best to open an issue to track your progress and to discuss any implementation
+questions you might have.
+
+Lastly, if you'd like something different to work on, please ping @purpleidea
+and I'll create an issue tailored especially for your approximate golang skill
+level and available time commitment in terms of hours you'd need to spend on the
+patch.
+
+Happy Hacking!
 
 ## Package resource
 
@@ -19,7 +27,7 @@ level and how many hours you'd like to spend on the patch.
 
 ## Svc resource
 
-- [ ] base resource improvements
+- [ ] refreshonly support [:heart:](https://github.com/purpleidea/mgmt/issues/464)
 
 ## Exec resource
 
@@ -33,33 +41,14 @@ level and how many hours you'd like to spend on the patch.
 
 - [ ] automatic edges to file resource [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
 
-## Virt (libvirt) resource
-
-- [ ] base resource improvements [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
-
-## Net (systemd-networkd) resource
-
-- [ ] base resource
-
-## Nspawn (systemd-nspawn) resource
-
-- [ ] base resource [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
-
-## Mount (systemd-mount) resource
-
-- [ ] base resource [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
-
-## Cron (systemd-timer) resource
-
-- [ ] base resource [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
-
 ## Http resource
 
 - [ ] base resource [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
 
 ## Etcd improvements
 
-- [ ] fix embedded etcd master race
+- [ ] fix etcd race bug that only happens during CI testing (intermittently
+failing test case issue)
 
 ## Torrent/dht file transfer
 
@@ -69,17 +58,33 @@ level and how many hours you'd like to spend on the patch.
 
 - [ ] base plumbing
 
+## Resource improvements
+
+- [ ] more reversible resources implemented
+- [ ] more "cloud" resources
+
 ## Language improvements
 
 - [ ] more core functions
 - [ ] automatic language formatter, ala `gofmt`
 - [ ] gedit/gnome-builder/gtksourceview syntax highlighting
 - [ ] vim syntax highlighting
-- [x] emacs syntax highlighting: see `misc/emacs/`
+- [ ] emacs syntax highlighting: see `misc/emacs/` (needs updating)
+- [ ] exposed $error variable for feedback in the language
+- [ ] improve the printf function to add %[]s, %[]f ([]str, []float) and map,
+struct, nested etc... %v would be nice too!
+- [ ] add line/col/file annotations to AST so we can get locations of errors
+that the parser finds
+- [ ] add more error messages with the `%error` pattern in parser.y
+- [ ] we should have helper functions or language sugar to pull a field out of a
+struct, or a value out of a map, or an index out of a list, etc...
+
+## Engine improvements
+
+- [ ] add a "waiting for func" message in the func engine to notify the user
+about slow functions...
 
 ## Other
 
-- [ ] better error/retry handling
-- [ ] deb package target in Makefile
 - [ ] reproducible builds
 - [ ] add your suggestions!

Vagrantfile

@@ -6,7 +6,7 @@ Vagrant.configure(2) do |config|
 	config.vm.synced_folder ".", "/vagrant", disabled: true
 
 	config.vm.define "mgmt-dev" do |instance|
-		instance.vm.box = "fedora/28-cloud-base"
+		instance.vm.box = "bento/fedora-31"
 	end
 
 	config.vm.provider "virtualbox" do |v|
@@ -23,8 +23,7 @@ Vagrant.configure(2) do |config|
 	config.vm.provision "file", source: "vagrant/mgmt.bashrc", destination: ".mgmt.bashrc"
 	config.vm.provision "file", source: "~/.gitconfig", destination: ".gitconfig"
 
-	# copied from make-deps.sh (with added git)
-	config.vm.provision "shell", inline: "dnf install -y libvirt-devel golang golang-googlecode-tools-stringer hg git make gem"
+	config.vm.provision "shell", inline: "dnf install -y golang git make"
 
 	# set up packagekit
 	config.vm.provision "shell" do |shell|
@@ -39,8 +38,10 @@ Vagrant.configure(2) do |config|
 	script = <<-SCRIPT
 		grep -q 'mgmt\.bashrc' ~/.bashrc || echo '. ~/.mgmt.bashrc' >>~/.bashrc
 		. ~/.mgmt.bashrc
-		go get -u github.com/purpleidea/mgmt
-		cd ~/gopath/src/github.com/purpleidea/mgmt
+		mkdir -p ~/gopath/src/github.com/purpleidea
+		cd ~/gopath/src/github.com/purpleidea
+		git clone https://github.com/purpleidea/mgmt --recursive
+		cd mgmt
 		make deps
 	SCRIPT
 	config.vm.provision "shell" do |shell|

bindata/Makefile

@@ -1,5 +1,5 @@
 # Mgmt
-# Copyright (C) 2013-2018+ James Shubin and the project contributors
+# Copyright (C) 2013-2021+ James Shubin and the project contributors
 # Written by James Shubin <james@shubin.ca> and the project contributors
 #
 # This program is free software: you can redistribute it and/or modify

converger/converger.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -25,8 +25,7 @@ import (
 	"time"
 
 	"github.com/purpleidea/mgmt/util"
-
-	multierr "github.com/hashicorp/go-multierror"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 // New builds a new converger coordinator.
@@ -323,9 +322,8 @@ func (obj *Coordinator) runStateFns(converged bool) error {
 	for _, name := range keys { // run in deterministic order
 		fn := obj.stateFns[name]
 		// call an arbitrary function
-		if e := fn(converged); e != nil {
-			err = multierr.Append(err, e) // list of errors
-		}
+		e := fn(converged)
+		err = errwrap.Append(err, e) // list of errors
 	}
 	return err
 }

converger/converger_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

debian/copyright

@@ -3,7 +3,7 @@ Upstream-Name: mgmt
 Source: <https://github.com/purpleidea/mgmt>
 
 Files: *
-Copyright: Copyright (C) 2013-2018+ James Shubin and the project contributors
+Copyright: Copyright (C) 2013-2021+ James Shubin and the project contributors
 License: GPL-3.0
 
 License: GPL-3.0

doc.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

docker/Dockerfile

@@ -1,10 +1,10 @@
-FROM golang:1.9
+FROM golang:1.13
 
 MAINTAINER Michał Czeraszkiewicz <contact@czerasz.com>
 
 # Set the reset cache variable
 # Read more here: http://czerasz.com/2014/11/13/docker-tip-and-tricks/#use-refreshedat-variable-for-better-cache-control
-ENV REFRESHED_AT 2017-11-16
+ENV REFRESHED_AT 2020-09-23
 
 # Update the package list to be able to use required packages
 RUN apt-get update

docker/Dockerfile.development

@@ -1,4 +1,4 @@
-FROM golang:1.11
+FROM golang:1.13
 
 MAINTAINER Michał Czeraszkiewicz <contact@czerasz.com>
 

docs/conf.py

@@ -51,7 +51,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'mgmt'
-copyright = u'2013-2018+ James Shubin and the project contributors'
+copyright = u'2013-2021+ James Shubin and the project contributors'
 author = u'James Shubin'
 
 # The version info for the project you're documenting, acts as replacement for

docs/development.md

@@ -3,7 +3,119 @@
 This document contains some additional information and help regarding
 developing `mgmt`. Useful tools, conventions, etc.
 
-Be sure to read [quick start guide](docs/quick-start-guide.md) first.
+Be sure to read [quick start guide](quick-start-guide.md) first.
+
+## Vagrant
+
+If you would like to avoid doing the above steps manually, we have prepared a
+[Vagrant](https://www.vagrantup.com/) environment for your convenience. From the
+project directory, run a `vagrant up`, and then a `vagrant status`. From there,
+you can `vagrant ssh` into the `mgmt` machine. The `MOTD` will explain the rest.
+This environment isn't commonly used by the `mgmt` developers, so it might not
+be working properly.
+
+## Using Docker
+
+Alternatively, you can check out the [docker-guide](docker-guide.md) in order to
+develop or deploy using docker. This method is not endorsed or supported, so use
+at your own risk, as it might not be working properly.
+
+## Information about dependencies
+
+Software projects have a few different kinds of dependencies. There are _build_
+dependencies, _runtime_ dependencies, and additionally, a few extra dependencies
+required for running the _test_ suite.
+
+### Build
+
+* `golang` 1.13 or higher (required, available in some distros and distributed
+as a binary officially by [golang.org](https://golang.org/dl/))
+
+### Runtime
+
+A relatively modern GNU/Linux system should be able to run `mgmt` without any
+problems. Since `mgmt` runs as a single statically compiled binary, all of the
+library dependencies are included. It is expected, that certain advanced
+resources require host specific facilities to work. These requirements are
+listed below:
+
+| Resource | Dependency        | Version                     | Check version with                                        |
+|----------|-------------------|-----------------------------|-----------------------------------------------------------|
+| augeas   | augeas-devel      | `augeas 1.6` or greater     | `dnf info augeas-devel` or `apt-cache show libaugeas-dev` |
+| file     | inotify           | `Linux 2.6.27` or greater   | `uname -a`                                                |
+| hostname | systemd-hostnamed | `systemd 25` or greater     | `systemctl --version`                                     |
+| nspawn   | systemd-nspawn    | `systemd ???` or greater    | `systemctl --version`                                     |
+| pkg      | packagekitd       | `packagekit 1.x` or greater | `pkcon --version`                                         |
+| svc      | systemd           | `systemd ???` or greater    | `systemctl --version`                                     |
+| virt     | libvirt-devel     | `libvirt 1.2.0` or greater  | `dnf info libvirt-devel` or `apt-cache show libvirt-dev`  |
+| virt     | libvirtd          | `libvirt 1.2.0` or greater  | `libvirtd --version`                                      |
+
+For building a visual representation of the graph, `graphviz` is required.
+
+To build `mgmt` without augeas support please run:
+`GOTAGS='noaugeas' make build`
+
+To build `mgmt` without libvirt support please run:
+`GOTAGS='novirt' make build`
+
+To build `mgmt` without docker support please run:
+`GOTAGS='nodocker' make build`
+
+To build `mgmt` without augeas, libvirt or docker support please run:
+`GOTAGS='noaugeas novirt nodocker' make build`
+
+## OSX/macOS/Darwin development
+
+Developing and running `mgmt` on macOS is currently not supported (but not
+discouraged either). Meaning it might work but in the case it doesn't you would
+have to provide your own patches to fix problems (the project maintainer and
+community are glad to assist where needed).
+
+There are currently some issues that make `mgmt` less suitable to run for
+provisioning macOS. But as a client to provision remote servers it should run
+fine.
+
+Since the primary supported systems are Linux and these are the environments
+tested, it is wise to run these suites during macOS development as well. To ease
+this, Docker can be leveraged ([Docker for Mac](https://docs.docker.com/docker-for-mac/)).
+
+Before running any of the commands below create the development Docker image:
+
+```
+docker/scripts/build-development
+```
+
+This image requires updating every time dependencies (`make-deps.sh`) changes.
+
+Then to run the test suite:
+
+```
+docker run --rm -ti \
+	-v $PWD:/go/src/github.com/purpleidea/mgmt/ \
+	-w /go/src/github.com/purpleidea/mgmt/ \
+	purpleidea/mgmt:development \
+	make test
+```
+
+For convenience this command is wrapped in `docker/scripts/exec-development`.
+
+Basically any command can be executed this way. Because the repository source is
+mounted into the Docker container invocation will be quick and allow rapid
+testing, for example:
+
+```
+docker/scripts/exec-development test/test-shell.sh load0.sh
+```
+
+Other examples:
+
+```
+docker/scripts/exec-development make build
+docker/scripts/exec-development ./mgmt run --tmp-prefix lang examples/lang/load0.mcl
+```
+
+Be advised that this method is not supported and it might not be working
+properly.
 
 ## Testing
 
@@ -45,5 +157,6 @@ individual tests to run.
 
 ### IDE/Editor support
 
-- Emacs: see `misc/emacs/`
-- [Textmate](https://github.com/aequitas/mgmt.tmbundle)
+* Emacs: see `misc/emacs/`
+* [Textmate](https://github.com/aequitas/mgmt.tmbundle)
+* [VSCode](https://github.com/aequitas/mgmt.vscode)

docs/documentation.md

@@ -122,6 +122,10 @@ entire set of running mgmt agents will need to all simultaneously converge for
 the group to exit. This is particularly useful for bootstrapping new clusters
 which need to exchange information that is only available at run time.
 
+This existed in earlier versions of mgmt as a `--remote` option, but it has been
+removed and is being ported to a more powerful variant where you can remote
+execute via a `remote` resource.
+
 #### Blog post
 
 You can read the introductory blog post about this topic here:
@@ -147,7 +151,7 @@ Invoke `mgmt` with the `--puppet` switch, which supports 3 variants:
 
 	`mgmt run puppet --puppet 'file { "/etc/ntp.conf": ensure => file }'`
 
-For more details and caveats see [Puppet.md](Puppet.md).
+For more details and caveats see [puppet-guide.md](puppet-guide.md).
 
 #### Blog post
 
@@ -250,6 +254,43 @@ integer, then that value is the max size for that semaphore. Valid semaphore
 id's include: `some_id`, `hello:42`, `not:smart:4` and `:13`. It is expected
 that the last bare example be only used by the engine to add a global semaphore.
 
+#### Rewatch
+
+Boolean. Rewatch specifies whether we re-run the Watch worker during a graph
+swap if it has errored. When doing a graph compare to swap the graphs, if this
+is true, and this particular worker has errored, then we'll remove it and add it
+back as a new vertex, thus causing it to run again. This is different from the
+`Retry` metaparam which applies during the normal execution. It is only when
+this is exhausted that we're in permanent worker failure, and only then can we
+rely on this metaparam.
+
+#### Realize
+
+Boolean. Realize ensures that the resource is guaranteed to converge at least
+once before a potential graph swap removes or changes it. This guarantee is
+useful for fast changing graphs, to ensure that the brief creation of a resource
+is seen. This guarantee does not prevent against the engine quitting normally,
+and it can't guarantee it if the resource is blocked because of a failed
+pre-requisite resource.
+*XXX: This is currently not implemented!*
+
+#### Reverse
+
+Boolean. Reverse is a property that some resources can implement that specifies
+that some "reverse" operation should happen when that resource "disappears". A
+disappearance happens when a resource is defined in one instance of the graph,
+and is gone in the subsequent one. This disappearance can happen if it was
+previously in an if statement that then becomes false.
+
+This is helpful for building robust programs with the engine. The engine adds a
+"reversed" resource to that subsequent graph to accomplish the desired "reverse"
+mechanics. The specifics of what this entails is a property of the particular
+resource that is being "reversed".
+
+It might be wise to combine the use of this meta parameter with the use of the
+`realize` meta parameter to ensure that your reversed resource actually runs at
+least once, if there's a chance that it might be gone for a while.
+
 ### Lang metadata file
 
 Any module *must* have a metadata file in its root. It must be named
@@ -298,10 +339,6 @@ recommended that you use this, since it's preferable to write code in the
 The main interface to the `mgmt` tool is the command line. For the most recent
 documentation, please run `mgmt --help`.
 
-#### `--yaml <graph.yaml>`
-
-Point to a graph file to run.
-
 #### `--converged-timeout <seconds>`
 
 Exit if the machine has converged for approximately this many seconds.
@@ -327,12 +364,6 @@ collision with this globally defined semaphore. The size value must be greater
 than zero at this time. The traditional non-parallel execution found in config
 management tools such as `Puppet` can be obtained with `--sema 1`.
 
-#### `--remote <graph.yaml>`
-
-Point to a graph file to run on the remote host specified within. This parameter
-can be used multiple times if you'd like to remotely run on multiple hosts in
-parallel.
-
 #### `--allow-interactive`
 
 Allow interactive prompting for SSH passwords if there is no authentication
@@ -371,8 +402,8 @@ default prefix. This can't be combined with the `--prefix` option.
 If this option is specified, we will attempt to fall back to a temporary prefix
 if the primary prefix couldn't be created. This is useful for avoiding failures
 in environments where the primary prefix may or may not be available, but you'd
-like to try. The canonical example is when running `mgmt` with `--remote` there
-might be a cached copy of the binary in the primary prefix, but in case there's
+like to try. The canonical example is when running `mgmt` with remote execution
+there might be a cached copy of the binary in the primary prefix, but if there's
 no binary available continue working in a temporary directory to avoid failure.
 
 ### Compilation options
@@ -455,7 +486,7 @@ To report any bugs, please file a ticket at: [https://github.com/purpleidea/mgmt
 
 ## Authors
 
-Copyright (C) 2013-2018+ James Shubin and the project contributors
+Copyright (C) 2013-2021+ James Shubin and the project contributors
 
 Please see the
 [AUTHORS](https://github.com/purpleidea/mgmt/tree/master/AUTHORS) file

docs/faq.md

@@ -9,6 +9,18 @@ I wanted a next generation config management solution that didn't have all of
 the design flaws or limitations that the current generation of tools do, and no
 tool existed!
 
+### Why did you choose `golang` for the project?
+
+When I started working on the project, I needed to choose a language that
+already had an implementation of a distributed consensus algorithm available.
+That meant [Paxos](https://en.wikipedia.org/wiki/Paxos_(computer_science)) or
+[Raft](https://en.wikipedia.org/wiki/Raft_(computer_science)). Golang was one
+language that actually had two different Raft implementations, `etcd`, and
+`consul`. Other design requirements included something that was reasonably fast,
+typed and memory-safe, and suited for systems engineering. After a reasonably
+extensive search, I chose `golang`. I think it was the right decision. There are
+a number of other features of the language which helped influence the decision.
+
 ### How do I contribute to the project if I don't know `golang`?
 
 There are many different ways you can contribute to the project. They can be
@@ -41,10 +53,11 @@ find a number of tutorials online.
 3. Spend between four to six hours with the [golang tour](https://tour.golang.org/).
 Skip over the longer problems, but try and get a solid overview of everything.
 If you forget something, you can always go back and repeat those parts.
-4. Connect to our [#mgmtconfig](https://webchat.freenode.net/?channels=#mgmtconfig)
-IRC channel on the [Freenode](https://freenode.net/) network. You can use any
-IRC client that you'd like, but the [hosted web portal](https://webchat.freenode.net/?channels=#mgmtconfig)
-will suffice if you don't know what else to use.
+4. Connect to our [#mgmtconfig](https://web.libera.chat/?channels=#mgmtconfig)
+IRC channel on the [Libera.Chat](https://libera.chat/) network. You can use any
+IRC client that you'd like, but the [hosted web portal](https://web.libera.chat/?channels=#mgmtconfig)
+will suffice if you don't know what else to use. [Here are a few suggestions for
+alternative clients.](https://libera.chat/guides/clients)
 5. Now it's time to try and starting writing a patch! We have tagged a bunch of
 [open issues as #mgmtlove](https://github.com/purpleidea/mgmt/issues?q=is%3Aissue+is%3Aopen+label%3Amgmtlove)
 for new users to have somewhere to get involved. Look through them to see if
@@ -125,6 +138,58 @@ The downside to this approach is that you won't benefit from the automatic
 elastic nature of the embedded etcd servers, and that you're responsible if you
 accidentally break your etcd cluster, or if you use an unsupported version.
 
+### In `mgmt` you talk about events. What is this referring to?
+
+Mgmt has two main concepts that involve "events":
+1. Events in the [resource primitive](resource-guide.md).
+2. Events in the [reactive language](language-guide.md).
+
+Each resource primitive in mgmt can test (check) and set (apply) the desired
+state that was requested of it. This is familiar to what is common with existing
+tools such as `Puppet`, `Ansible`, `Chef`, `Terraform`, etc... In addition,
+`mgmt` can also **watch** the state and detect changes. As a result, it never
+has to waste time and cpu resources by polling to test and set state, leading to
+a design which is algorithmically much faster than the existing generation of
+tools.
+
+To describe the set of resources to apply, mgmt describes this collection with a
+language. In order to model the time component of infrastructure, we use a
+special kind of language called an [FRP](https://en.wikipedia.org/wiki/Functional_reactive_programming).
+This language has a built-in concept that we call "events", and which means that
+we re-evaluate the relevant portions of the code whenever a value or function
+has an event that tells us that it changed. The `R` in `FRP` stands for
+reactive. This is similar to how a spreadsheet updates dependent cells when a
+pre-requisite value is modified. [This article](https://en.wikipedia.org/wiki/Reactive_programming)
+provides a bit more background.
+
+Whenever any of the streams of values in the language change, the program is
+partially re-evaluated. The output of any mgmt program is a [DAG](https://en.wikipedia.org/wiki/Directed_acyclic_graph)
+of resources, or more precisely, a stream of resource graphs. Since we have
+events per-resource, we can efficiently switch from one desired-state resource
+graph to the next without re-checking their individual states, since we've been
+monitoring them all along.
+
+One side-effect of all this, is that if a rogue systems administrator manually
+changes the state of any managed resource, mgmt will detect this and attempt to
+revert the change. This makes for excellent live demos, but is not the primary
+design goal. It is a consequence of tracking state so that graph changes are
+efficient. We implement the event detection via an intentional per-resource
+[main loop](https://en.wikipedia.org/wiki/Event_loop) which can enable other
+interesting functionality too!
+
+Make sure to get rid of your rogue sysadmin! ;)
+
+### Do I need to run `mgmt` as `root`?
+
+No and yes. It depends. Nothing in mgmt explicitly requires root in the design,
+however mgmt will require root only if the changes to your system that you want
+it to make require root.
+
+For example, if you use it to manage files that require root access to modify,
+then you'll need root. If you only use it to manage files and resources
+elsewhere, then it shouldn't need root. Many resources are perfectly usable
+without root, and virtually all of my live demos are done without root.
+
 ### How can I run `mgmt` on-demand, or in `cron`, instead of continuously?
 
 By default, `mgmt` will run continuously in an attempt to keep your machine in a
@@ -148,43 +213,94 @@ requires a number of seconds as an argument.
 #### Example:
 
 ```
-./mgmt run lang --lang examples/lang/hello0.mcl --converged-timeout=5
+./mgmt run lang examples/lang/hello0.mcl --converged-timeout=5
 ```
 
-### What does the error message about an inconsistent dataDir mean?
+### When I try to build `mgmt` I see: `no Go files in $GOPATH/src/github.com/purpleidea/mgmt/bindata`.
+
+Due to the arcane way that `golang` designed its `$GOPATH`, the main project
+directory must be inside your `$GOPATH`, and at the appropriate FQDN. This is:
+`$GOPATH/src/github.com/purpleidea/mgmt/`. If you have your project root outside
+of that directory, then you may get this error when you try to build it. In this
+case there is likely a `go get` version of the project at this location. Remove
+it and replace it with your git cloned directory. In my case, I like to work on
+things in `~/code/mgmt/`, so that path is a symlink that points to the long
+project directory.
+
+### Why does my file resource error with `no such file or directory`?
+
+If you create a file resource and only specify the content like this:
+
+```
+file "/tmp/foo" {
+	content => "hello world\n",
+}
+```
+
+Then this will attempt to set the contents of that file to the desired string,
+but *only* if that file already exists. If you'd like to ensure that it also
+gets created in case it is not present, then you must also specify the state:
+
+```
+file "/tmp/foo" {
+	state => $const.res.file.state.exists,
+	content => "hello world\n",
+}
+```
+
+Similar logic applies for situations when you only specify the `mode` parameter.
+
+This all turns out to be more safe and "correct", in that it would error and
+prevent masking an error for a situation when you expected a file to already be
+at that location. It also turns out to simplify the internals significantly, and
+remove an ambiguous scenario with the reversable file resource.
+
+### Why do function names inside of templates include underscores?
+
+The golang template library which we use to implement the template() function
+doesn't support the dot notation, so we import all our normal functions, and
+just replace dots with underscores. As an example, the standard `datetime.print`
+function is shown within mcl scripts as datetime_print after being imported.
+
+### On startup `mgmt` hangs after: `etcd: server: starting...`.
 
 If you get an error message similar to:
 
 ```
-Etcd: Connect: CtxError...
-Etcd: CtxError: Reason: CtxDelayErr(5s): No endpoints available yet!
-Etcd: Connect: Endpoints: []
-Etcd: The dataDir (/var/lib/mgmt/etcd) might be inconsistent or corrupt.
+etcd: server: starting...
+etcd: server: start timeout of 1m0s reached
+etcd: server: close timeout of 15s reached
 ```
 
-This happens when there are a series of fatal connect errors in a row. This can
-happen when you start `mgmt` using a dataDir that doesn't correspond to the
-current cluster view. As a result, the embedded etcd server never finishes
-starting up, and as a result, a default endpoint never gets added. The solution
-is to either reconcile the mistake, and if there is no important data saved, you
-can remove the etcd dataDir. This is typically `/var/lib/mgmt/etcd/member/`.
+But nothing happens afterwards, this can be due to a corrupt etcd storage
+directory. Each etcd server embedded in mgmt must have a special directory where
+it stores local state. It must not be shared by more than one individual member.
+This dir is typically `/var/lib/mgmt/etcd/member/`. If you accidentally use it
+(for example during testing) with a different cluster view, then you can corrupt
+it. This can happen if you use it with more than one different hostname.
 
-### Why do resources have both a `Cmp` method and an `IFF` (on the UID) method?
+The solution is to avoid making this mistake, and if there is no important data
+saved, you can remove the etcd member dir and start over.
 
-The `Cmp()` methods are for determining if two resources are effectively the
-same, which is used to make graph change delta's efficient. This is when we want
-to change from the current running graph to a new graph, but preserve the common
-vertices. Since we want to make this process efficient, we only update the parts
-that are different, and leave everything else alone. This `Cmp()` method can
-tell us if two resources are the same. In case it is not obvious, `cmp` is an
-abbrev. for compare.
+### On running `make` to build a new version, it errors with: `Text file busy`.
 
-The `IFF()` method is part of the whole UID system, which is for discerning if a
-resource meets the requirements another expects for an automatic edge. This is
-because the automatic edge system assumes a unified UID pattern to test for
-equality. In the future it might be helpful or sane to merge the two similar
-comparison functions although for now they are separate because they are
-actually answer different questions.
+If you get an error like:
+
+```
+cp: cannot create regular file 'mgmt': Text file busy
+```
+
+This can happen if you ran `make build` (or just `make`) when there was already
+an instance of mgmt running, or if a related file locking issue occurred. To
+solve this, shutdown and running mgmt process, run `rm mgmt` to remove the file,
+and then get a new one by running `make` again.
+
+### The docs speaks of `--remote` but the CLI errors out?
+
+The `--remote` flag existed in an earlier version of mgmt. It was removed and
+will be replaced with a more powerful version, which is a "remote" resource. The
+code is mostly ready but it's not finished. If you'd like to help finish it or
+sponsor the work, please let me know.
 
 ### Does this support Windows? OSX? GNU Hurd?
 
@@ -193,7 +309,7 @@ serious automation workloads. Support for non-Linux operating systems isn't a
 high priority of mine, but we're happy to accept patches for missing features
 or resources that you think would make sense on your favourite platform.
 
-### Why aren't you using `glide` or `godep` for dependency management?
+### Why aren't you using `glide`, `godep` or `go mod` for dependency management?
 
 Vendoring dependencies means that as the git master branch of each dependency
 marches on, you're left behind using an old version. As a result, bug fixes and
@@ -261,10 +377,9 @@ which definitely existed before the band did.
 
 ### You didn't answer my question, or I have a question!
 
-It's best to ask on [IRC](https://webchat.freenode.net/?channels=#mgmtconfig)
-to see if someone can help you. Once we get a big enough community going, we'll
-add a mailing list. If you don't get any response from the above, you can
-contact me through my [technical blog](https://purpleidea.com/contact/)
-and I'll do my best to help. If you have a good question, please add it as a
-patch to this documentation. I'll merge your question, and add a patch with the
-answer!
+It's best to ask on [IRC](https://web.libera.chat/?channels=#mgmtconfig)
+to see if someone can help you. If you don't get a response from IRC, you can
+contact me through my [technical blog](https://purpleidea.com/contact/) and I'll
+do my best to help. If you have a good question, please add it as a patch to
+this documentation. I'll merge your question, and add a patch with the answer!
+For news and updates, subscribe to the [mailing list](https://www.redhat.com/mailman/listinfo/mgmtconfig-list).

docs/function-guide.md

@@ -37,8 +37,10 @@ available types and values in the mgmt language. It is very easy to use, and
 should be fairly intuitive. Most of what you'll need to know can be inferred
 from looking at example code.
 
-To implement a function, you'll need to create a file in
-[`lang/funcs/simple/`](https://github.com/purpleidea/mgmt/tree/master/lang/funcs/simple/).
+To implement a function, you'll need to create a file that imports the
+[`lang/funcs/simple/`](https://github.com/purpleidea/mgmt/tree/master/lang/funcs/simple/)
+module. It should probably get created in the correct directory inside of:
+[`lang/funcs/core/`](https://github.com/purpleidea/mgmt/tree/master/lang/funcs/core/).
 The function should be implemented as a `FuncValue` in our type system. It is
 then registered with the engine during `init()`. An example explains it best:
 
@@ -50,14 +52,15 @@ package simple
 import (
 	"fmt"
 
+	"github.com/purpleidea/mgmt/lang/funcs/simple"
 	"github.com/purpleidea/mgmt/lang/types"
 )
 
 // you must register your functions in init when the program starts up
 func init() {
 	// Example function that squares an int and prints out answer as an str.
-	Register("talkingsquare", &types.FuncValue{
-		T: types.NewType("func(a int) str"), // declare the signature
+	simple.ModuleRegister(ModuleName, "talkingsquare", &types.FuncValue{
+		T: types.NewType("func(int) str"), // declare the signature
 		V: func(input []types.Value) (types.Value, error) {
 			i := input[0].Int() // get first arg as an int64
 			// must return the above specified value
@@ -109,15 +112,20 @@ As with the simple, non-polymorphic API, you can only implement [pure](https://e
 functions, without writing too much boilerplate code. They will be automatically
 re-evaluated as needed when their input values change.
 
-To implement a function, you'll need to create a file in
-[`lang/funcs/simplepoly/`](https://github.com/purpleidea/mgmt/tree/master/lang/funcs/simplepoly/).
+To implement a function, you'll need to create a file that imports the
+[`lang/funcs/simplepoly/`](https://github.com/purpleidea/mgmt/tree/master/lang/funcs/simplepoly/)
+module. It should probably get created in the correct directory inside of:
+[`lang/funcs/core/`](https://github.com/purpleidea/mgmt/tree/master/lang/funcs/core/).
 The function should be implemented as a list of `FuncValue`'s in our type
 system. It is then registered with the engine during `init()`. You may also use
 the `variant` type in your type definitions. This special type will never be
 seen inside a running program, and will get converted to a concrete type if a
 suitable match to this signature can be found. Be warned that signatures which
 contain too many variants, or which are very general, might be hard for the
-compiler to match, and ambiguous type graphs make for user compiler errors.
+compiler to match, and ambiguous type graphs make for user compiler errors. The
+top-level type must still be a function type, it may only contain variants as
+part of its signature. It is probably more difficult to unify a function if its
+return type is a variant, as opposed to if one of its args was.
 
 An example explains it best:
 
@@ -127,11 +135,13 @@ An example explains it best:
 import (
 	"fmt"
 
-	"github.com/purpleidea/mgmt/lang/types"
 	"github.com/purpleidea/mgmt/lang/funcs/simplepoly"
+	"github.com/purpleidea/mgmt/lang/types"
 )
 
 func init() {
+	// You may use the simplepoly.ModuleRegister method to register your
+	// function if it's in a module, as seen in the simple function example.
 	simplepoly.Register("len", []*types.FuncValue{
 		{
 			T: types.NewType("func([]variant) int"),
@@ -190,7 +200,7 @@ if it meets your needs. Most functions will be able to use that API. If you
 really need something more powerful, then you can use the regular function API.
 What follows are each of the method signatures and a description of each.
 
-### Default
+### Info
 
 ```golang
 Info() *interfaces.Info
@@ -219,7 +229,7 @@ Init(init *interfaces.Init) error
 
 This is called to initialize the function. If something goes wrong, it should
 return an error. It is passed a struct that contains all the important
-information and poiinters that it might need to work with throughout its
+information and pointers that it might need to work with throughout its
 lifetime. As a result, it will need to save a copy to that pointer for future
 use in the other methods.
 
@@ -377,9 +387,9 @@ might be different ways you would want to call `printf`, such as:
 `printf("the %s is %d", "answer", 42)` or `printf("3 * 2 = %d", 3 * 2)`. Since
 you couldn't implement the infinite number of possible signatures, this API lets
 you write code which can be coerced into different forms. This makes
-implementing what would appear to be generic or polymorphic, instead something
-that is actually static and that still has the static type safety properties
-that were guaranteed by the mgmt language.
+implementing what would appear to be generic or polymorphic, instead of
+something that is actually static and that still has the static type safety
+properties that were guaranteed by the mgmt language.
 
 Since this is an advanced topic, it is not described in full at this time. For
 more information please have a look at the source code comments, some of the
@@ -435,6 +445,11 @@ generator to build your `FuncValue` implementations, and pass in the unique
 signature to each one as you are building them. Using a generator is a common
 technique which was mentioned previously.
 
+One obvious situation where this might occur is if your function doesn't take
+any inputs! An example `math.fortytwo()` function was implemented that
+demonstrates the use of function generators to pass the type signatures into the
+implementations.
+
 ### Where can I find more information about mgmt?
 
 Additional blog posts, videos and other material [is available!](https://github.com/purpleidea/mgmt/blob/master/docs/on-the-web.md).

docs/language-guide.md

@@ -54,7 +54,7 @@ can be impossible to infer the item's type.
 
 An unordered set of unique keys of the same type and corresponding value pairs
 of another type, eg:
-`{"boiling" => 100, "freezing" => 0, "room" => "25", "house" => 22, "canada" => -30,}`.
+`{"boiling" => 100, "freezing" => 0, "room" => 25, "house" => 22, "canada" => -30,}`.
 That is to say, all of the keys must have the same type, and all of the values
 must have the same type. You can use any type for either, although it is
 probably advisable to avoid using very complex types as map keys.
@@ -206,7 +206,7 @@ value to use if that boolean is true. You can do this with the resource-specific
 $b = true # change me to false and then try editing the file manually
 file "/tmp/mgmt-elvis" {
 	content => $b ?: "hello world\n",
-	state => "exists",
+	state => $const.res.file.state.exists,
 }
 ```
 
@@ -293,7 +293,7 @@ send notifications. You may have multiples of these per resource, including
 multiple `Depend` lines if necessary. Each of these properties also supports the
 conditional inclusion `elvis` operator as well.
 
-For example, you may write is:
+For example, you may write:
 
 ```mcl
 $b = true # for example purposes
@@ -511,6 +511,9 @@ without making any changes. The `ExprVar` node naturally consumes scope's and
 the `StmtProg` node cleverly passes the scope through in the order expected for
 the out-of-order bind logic to work.
 
+This step typically calls the ordering algorithm to determine the correct order
+of statements in a program.
+
 #### Type unification
 
 Each expression must have a known type. The unpleasant option is to force the

docs/on-the-web.md

@@ -44,3 +44,15 @@ if we missed something that you think is relevant!
 | James Shubin | blog | [Mgmt Configuration Language](https://purpleidea.com/blog/2018/02/05/mgmt-configuration-language/) |
 | James Shubin | video | [Recording from CfgMgmtCamp.eu 2018](https://www.youtube.com/watch?v=NxObmwZDyrI) |
 | Jonathan Gold | blog | [Go Netlink and Select](https://jonathangold.ca/blog/go-netlink-and-select/) |
+| James Shubin | video | [Recording from DevOpsDays Montreal 2018](https://www.youtube.com/watch?v=1i38c5cooHo) |
+| James Shubin | video | [Recording from FOSDEM Minimalistic Languages Devroom 2019](https://video.fosdem.org/2019/K.4.201/mgmtconfig.webm) |
+| James Shubin | video | [Recording from FOSDEM Infra Management Devroom 2019](https://video.fosdem.org/2019/UB2.252A/mgmt.webm) |
+| James Shubin | video | [Recording from FOSDEM Graph Processing Devroom 2019](https://video.fosdem.org/2019/H.1308/graph_mgmt_config.webm) |
+| James Shubin | video | [Recording from FOSDEM Virtualization Devroom 2019](https://video.fosdem.org/2019/H.2213/vai_real_time_virtualization_automation.webm) |
+| James Shubin | video | [Recording from FOSDEM Containers Devroom 2019](https://video.fosdem.org/2019/UA2.114/containers_mgmt.webm) |
+| James Shubin | video | [Recording from FOSDEM Monitoring Devroom 2019](https://video.fosdem.org/2019/UB2.252A/real_time_merging_of_config_management_and_monitoring.webm) |
+| James Shubin | blog | [Mgmt Configuration Language: Class and Include](https://purpleidea.com/blog/2019/07/26/class-and-include-in-mgmt/) |
+| James Shubin | video | [Recording from FOSDEM 2020, Main Track (History)](https://video.fosdem.org/2020/Janson/automation.webm) |
+| James Shubin | video | [Recording from FOSDEM 2020, Infra Management Devroom](https://video.fosdem.org/2020/UA2.120/mgmt.webm) |
+| James Shubin | video | [Recording from FOSDEM 2020, Minimalistic Languages Devroom](https://video.fosdem.org/2020/AW1.125/mgmtconfigmore.webm) |
+| James Shubin | video | [Recording from CfgMgmtCamp.eu 2020](https://www.youtube.com/watch?v=Kd7FAORFtsc) |

docs/puppet-guide.md

@@ -173,7 +173,7 @@ useful when you are in the process of replacing Puppet with mgmt. You
 can translate your custom modules into mgmt's language one by one,
 and let mgmt run the current mix.
 
-Instead of the usual `--puppet`, `--puppet-conf`, and `--lang` for mcl,
+Instead of the usual `--puppet-conf` flag and argv for `puppet` and `mcl` input,
 you need to use alternative flags to make this work:
 
 * `--lp-lang` to specify the mcl input

docs/quick-start-guide.md

@@ -2,65 +2,108 @@
 
 ## Introduction
 
-This guide is intended for developers. Once `mgmt` is minimally viable, we'll
-publish a quick start guide for users too. If you're brand new to `mgmt`, it's
-probably a good idea to start by reading the
-[introductory article](https://purpleidea.com/blog/2016/01/18/next-generation-configuration-mgmt/)
-or to watch an [introductory video](https://www.youtube.com/watch?v=LkEtBVLfygE&html5=1).
-Once you're familiar with the general idea, please start hacking...
+This guide is intended for users and developers. If you're brand new to `mgmt`,
+it's probably a good idea to start by reading an
+[introductory article about the engine](https://purpleidea.com/blog/2016/01/18/next-generation-configuration-mgmt/)
+and an [introductory article about the language](https://purpleidea.com/blog/2018/02/05/mgmt-configuration-language/).
+[There are other articles and videos available](on-the-web.md) if you'd like to
+learn more or prefer different formats. Once you're familiar with the general
+idea, or if you prefer a hands-on approach, please start hacking...
 
-## Quick start
+## Getting mgmt
 
-### Installing golang
+You can either build `mgmt` from source, or you can download a pre-built
+release. There are also some distro repositories available, but they may not be
+up to date. A pre-built release is the fastest option if there's one that's
+available for your platform. If you are developing or testing a new patch to
+`mgmt`, or there is not a release available for your platform, then you'll have
+to build your own.
 
-* You need golang version 1.10 or greater installed.
+### Downloading a pre-built release:
+
+The latest releases can be found [here](https://github.com/purpleidea/mgmt/releases/).
+An alternate mirror is available [here](https://dl.fedoraproject.org/pub/alt/purpleidea/mgmt/releases/).
+
+Make sure to verify the signatures of all packages before you use them. The
+signing key can be downloaded from [https://purpleidea.com/contact/#pgp-key](https://purpleidea.com/contact/#pgp-key)
+to verify the release.
+
+If you've decided to install a pre-build release, you can skip to the
+[Running mgmt](#running-mgmt) section below!
+
+### Building a release:
+
+You'll need some dependencies, including `golang`, and some associated tools.
+
+#### Installing golang
+
+* You need golang version 1.13 or greater installed.
 	* To install on rpm style systems: `sudo dnf install golang`
 	* To install on apt style systems: `sudo apt install golang`
 	* To install on macOS systems install [Homebrew](https://brew.sh)
 	and run: `brew install go`
 * You can run `go version` to check the golang version.
-* If your distro is tool old, you may need to [download](https://golang.org/dl/)
+* If your distro is too old, you may need to [download](https://golang.org/dl/)
 a newer golang version.
 
-### Setting up golang
+#### Setting up golang
 
-* If you do not have a GOPATH yet, create one and export it:
+* You can skip this step, as your installation will default to using `~/go/`,
+but if you do not have a `GOPATH` yet and want one in a custom location, create
+one and export it:
 
-```
+```shell
 mkdir $HOME/gopath
 export GOPATH=$HOME/gopath
 ```
 
 * You might also want to add the GOPATH to your `~/.bashrc` or `~/.profile`.
-* For more information you can read the [GOPATH documentation](https://golang.org/cmd/go/#hdr-GOPATH_environment_variable).
+* For more information you can read the
+[GOPATH documentation](https://golang.org/cmd/go/#hdr-GOPATH_environment_variable).
 
-### Getting the mgmt code and dependencies
+#### Getting the mgmt code and associated dependencies
 
-* Download the `mgmt` code into the GOPATH, and switch to that directory:
+* Download the `mgmt` code into the `GOPATH`, and switch to that directory:
 
-```
-mkdir -p $GOPATH/src/github.com/purpleidea/
-cd $GOPATH/src/github.com/purpleidea/
+```shell
+[ -z "$GOPATH" ] && mkdir ~/go/ || mkdir -p $GOPATH/src/github.com/purpleidea/
+cd $GOPATH/src/github.com/purpleidea/ || cd ~/go/
 git clone --recursive https://github.com/purpleidea/mgmt/
-cd $GOPATH/src/github.com/purpleidea/mgmt
+cd $GOPATH/src/github.com/purpleidea/mgmt/ || cd ~/go/src/github.com/purpleidea/mgmt/
 ```
 
-* Add $GOPATH/bin to $PATH
+* Add `$GOPATH/bin` to `$PATH`
 
-```
+```shell
 export PATH=$PATH:$GOPATH/bin
 ```
 
 * Run `make deps` to install system and golang dependencies. Take a look at
-`misc/make-deps.sh` for details.
-* Run `make build` to get a freshly built `mgmt` binary.
+`misc/make-deps.sh` if you want to see the details of what it does.
 
-### Running mgmt
+#### Building mgmt
 
-* Run `time ./mgmt run --tmp-prefix lang --lang examples/lang/hello0.mcl` to try
-out a very simple example!
+* Now run `make` to get a freshly built `mgmt` binary. If this succeeds, you can
+proceed to the [Running mgmt](#running-mgmt) section below!
+
+### Installing a distro release
+
+Installation of `mgmt` from distribution packages currently needs improvement.
+They are not always up-to-date with git master and as such are not recommended.
+At the moment we have:
+* [COPR](https://copr.fedoraproject.org/coprs/purpleidea/mgmt/) (currently dead)
+* [Arch](https://aur.archlinux.org/packages/mgmt/) (currently stale)
+
+Please contribute more and help improve these! We'd especially like to see a
+Debian package!
+
+## Running mgmt
+
+* Run `mgmt run --tmp-prefix lang examples/lang/hello0.mcl` to try out a very
+simple example! If you built it from source, you'll need to use `./mgmt` from
+the project directory.
 * Look in that example file that you ran to see if you can figure out what it
-did!
+did! You can press `^C` to exit `mgmt`.
 * Have fun hacking on our future technology and get involved to shape the
 project!
 
@@ -68,118 +111,3 @@ project!
 
 Please look in the [examples/lang/](../examples/lang/) folder for some more
 examples!
-
-## Vagrant
-
-If you would like to avoid doing the above steps manually, we have prepared a
-[Vagrant](https://www.vagrantup.com/) environment for your convenience. From the
-project directory, run a `vagrant up`, and then a `vagrant status`. From there,
-you can `vagrant ssh` into the `mgmt` machine. The MOTD will explain the rest.
-
-## Using Docker
-
-Alternatively, you can check out the [docker-guide](docs/docker-guide.md) in
-order to develop or deploy using docker.
-
-## Information about dependencies
-
-Software projects have a few different kinds of dependencies. There are _build_
-dependencies, _runtime_ dependencies, and additionally, a few extra dependencies
-required for running the _test_ suite.
-
-### Build
-
-* `golang` 1.10 or higher (required, available in some distros and distributed
-as a binary officially by [golang.org](https://golang.org/dl/))
-
-### Runtime
-
-A relatively modern GNU/Linux system should be able to run `mgmt` without any
-problems. Since `mgmt` runs as a single statically compiled binary, all of the
-library dependencies are included. It is expected, that certain advanced
-resources require host specific facilities to work. These requirements are
-listed below:
-
-| Resource | Dependency        | Version                     | Check version with                                        |
-|----------|-------------------|-----------------------------|-----------------------------------------------------------|
-| augeas   | augeas-devel      | `augeas 1.6` or greater     | `dnf info augeas-devel` or `apt-cache show libaugeas-dev` |
-| file     | inotify           | `Linux 2.6.27` or greater   | `uname -a`                                                |
-| hostname | systemd-hostnamed | `systemd 25` or greater     | `systemctl --version`                                     |
-| nspawn   | systemd-nspawn    | `systemd ???` or greater    | `systemctl --version`                                     |
-| pkg      | packagekitd       | `packagekit 1.x` or greater | `pkcon --version`                                         |
-| svc      | systemd           | `systemd ???` or greater    | `systemctl --version`                                     |
-| virt     | libvirt-devel     | `libvirt 1.2.0` or greater  | `dnf info libvirt-devel` or `apt-cache show libvirt-dev`  |
-| virt     | libvirtd          | `libvirt 1.2.0` or greater  | `libvirtd --version`                                      |
-
-For building a visual representation of the graph, `graphviz` is required.
-
-To build `mgmt` without augeas support please run:
-`GOTAGS='noaugeas' make build`
-
-To build `mgmt` without libvirt support please run:
-`GOTAGS='novirt' make build`
-
-To build `mgmt` without docker support please run:
-`GOTAGS='nodocker' make build`
-
-To build `mgmt` without augeas, libvirt or docker support please run:
-`GOTAGS='noaugeas novirt nodocker' make build`
-
-## Binary Package Installation
-
-Installation of `mgmt` from distribution packages currently needs improvement.
-They are not always up-to-date with git master and as such are not recommended.
-At the moment we have:
-* [COPR](https://copr.fedoraproject.org/coprs/purpleidea/mgmt/)
-* [Arch](https://aur.archlinux.org/packages/mgmt/)
-
-Please contribute more! We'd especially like to see a Debian package!
-
-## OSX/macOS/Darwin development
-
-Developing and running `mgmt` on macOS is currently not supported (but not
-discouraged either). Meaning it might work but in the case it doesn't you would
-have to provide your own patches to fix problems (the project maintainer and
-community are glad to assist where needed).
-
-There are currently some issues that make `mgmt` less suitable to run for provisioning
-macOS. But as a client to provision remote servers it should run fine.
-
-Since the primary supported systems are Linux and these are the environments
-tested for it is wise to run these suites during macOS development as well. To
-ease this Docker can be leveraged ([Docker for Mac](https://docs.docker.com/docker-for-mac/)).
-
-Before running any of the commands below create the development Docker image:
-
-```
-docker/scripts/build-development
-```
-
-This image requires updating every time dependencies (`make-deps.sh`) change.
-
-Then to run the test suite:
-
-```
-docker run --rm -ti \
-	-v $PWD:/go/src/github.com/purpleidea/mgmt/ \
-	-w /go/src/github.com/purpleidea/mgmt/ \
-	purpleidea/mgmt:development \
-	make test
-```
-
-For convenience this command is wrapped in `docker/scripts/exec-development`.
-
-Basically any command can be executed this way. Because the repository source is
-mounted into the Docker container invocation will be quick and allow rapid
-testing, example:
-
-```
-docker/scripts/exec-development test/test-shell.sh load0.sh
-```
-
-Other examples:
-
-```
-docker/scripts/exec-development make build
-docker/scripts/exec-development ./mgmt run --tmp-prefix lang --lang examples/lang/load0.mcl
-```

docs/resource-guide.md

@@ -96,14 +96,16 @@ Default() engine.Res
 ```
 
 This returns a populated resource struct as a `Res`. It shouldn't populate any
-values which already have the correct default as the golang zero value. In
+values which already get a good default as the respective golang zero value. In
 general it is preferable if the zero values make for the correct defaults.
+(This is to say, resources are designed to behave safely and intuitively
+when parameters take a zero value, whenever this is possible.)
 
 #### Example
 
 ```golang
 // Default returns some sensible defaults for this resource.
-func (obj *FooRes) Default() Res {
+func (obj *FooRes) Default() engine.Res {
 	return &FooRes{
 		Answer: 42, // sometimes, defaults shouldn't be the zero value
 	}
@@ -640,8 +642,8 @@ The signature intentionally matches what is required to satisfy the `go-yaml`
 #### Example
 
 ```golang
-// UnmarshalYAML is the custom unmarshal handler for this struct.
-// It is primarily useful for setting the defaults.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
+// primarily useful for setting the defaults.
 func (obj *FooRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes FooRes // indirection to avoid infinite recursion
 
@@ -754,6 +756,23 @@ Feel free to use this pattern if you're convinced it's necessary. Alternatively,
 if you think I got the `Res` API wrong and you have an improvement, please let
 us know!
 
+### Why do resources have both a `Cmp` method and an `IFF` (on the UID) method?
+
+The `Cmp()` methods are for determining if two resources are effectively the
+same, which is used to make graph change delta's efficient. This is when we want
+to change from the current running graph to a new graph, but preserve the common
+vertices. Since we want to make this process efficient, we only update the parts
+that are different, and leave everything else alone. This `Cmp()` method can
+tell us if two resources are the same. In case it is not obvious, `cmp` is an
+abbrev. for compare.
+
+The `IFF()` method is part of the whole UID system, which is for discerning if a
+resource meets the requirements another expects for an automatic edge. This is
+because the automatic edge system assumes a unified UID pattern to test for
+equality. In the future it might be helpful or sane to merge the two similar
+comparison functions although for now they are separate because they are
+actually answer different questions.
+
 ### What new resource primitives need writing?
 
 There are still many ideas for new resources that haven't been written yet. If

docs/resources.md

@@ -17,6 +17,7 @@ You might want to look at the [generated documentation](https://godoc.org/github
 for more up-to-date information about these resources.
 
 * [Augeas](#Augeas): Manipulate files using augeas.
+* [Consul:KV](#ConsulKV): Set keys in a Consul datastore.
 * [Docker](#Docker):[Container](#Container) Manage docker containers.
 * [Exec](#Exec): Execute shell commands on the system.
 * [File](#File): Manage files and directories.
@@ -32,6 +33,8 @@ for more up-to-date information about these resources.
 * [Print](#Print): Print messages to the console.
 * [Svc](#Svc): Manage system systemd services.
 * [Test](#Test): A mostly harmless resource that is used for internal testing.
+* [Tftp:File](#TftpFile): Add files to the small embedded embedded tftp server.
+* [Tftp:Server](#TftpServer): Run a small embedded tftp server.
 * [Timer](#Timer): Manage system systemd services.
 * [User](#User): Manage system users.
 * [Virt](#Virt): Manage virtual machines with libvirt.
@@ -69,9 +72,9 @@ identified by a trailing slash in their path name. File have no such slash.
 It has the following properties:
 
 * `path`: absolute file path (directories have a trailing slash here)
+* `state`: either `exists`, `absent`, or undefined
 * `content`: raw file content
-* `state`: either `exists` (the default value) or `absent`
-* `mode`: octal unix file permissions
+* `mode`: octal unix file permissions or symbolic string
 * `owner`: username or uid for the file owner
 * `group`: group name or gid for the file group
 
@@ -79,6 +82,16 @@ It has the following properties:
 
 The path property specifies the file or directory that we are managing.
 
+### State
+
+The state property describes the action we'd like to apply for the resource. The
+possible values are: `exists` and `absent`. If you do not specify either of
+these, it is undefined. Without specifying this value as `exists`, another param
+cannot cause a file to get implicitly created. When specifying this value as
+`absent`, you should not specify any other params that would normally change the
+file. For example, if you specify `content` and this param is `absent`, then you
+will get an engine validation error.
+
 ### Content
 
 The content property is a string that specifies the desired file contents.
@@ -88,10 +101,12 @@ The content property is a string that specifies the desired file contents.
 The source property points to a source file or directory path that we wish to
 copy over and use as the desired contents for our resource.
 
-### State
+### Fragments
 
-The state property describes the action we'd like to apply for the resource. The
-possible values are: `exists` and `absent`.
+The fragments property lets you specify a list of files to concatenate together
+to make up the contents of this file. They will be combined in the order that
+they are listed in. If one of the files specified is a directory, then the
+files in that top-level directory will be themselves combined together and used.
 
 ### Recurse
 
@@ -104,6 +119,12 @@ The force property is required if we want the file resource to be able to change
 a file into a directory or vice-versa. If such a change is needed, but the force
 property is not set to `true`, then this file resource will error.
 
+### Purge
+
+The purge property is used when this file represents a directory, and we'd like
+to remove any unmanaged files from within it. Please note that any unmanaged
+files in a directory with this flag set will be irreversibly deleted.
+
 ## Group
 
 The group resource manages the system groups from `/etc/group`.
@@ -206,6 +227,16 @@ The service resource is still very WIP. Please help us by improving it!
 
 The test resource is mostly harmless and is used for internal tests.
 
+## Tftp:File
+
+This adds files to the running tftp server. It's useful because it allows you to
+add individual files without needing to create them on disk.
+
+## Tftp:Server
+
+Run a small embedded tftp server. This doesn't apply any state, but instead runs
+a pure golang tftp server in the Watch loop.
+
 ## Timer
 
 This resource needs better documentation. Please help us by improving it!

docs/style-guide.md

@@ -61,6 +61,12 @@ Occasionally inline, two line source code comments are used within a function.
 These should usually be balanced so that you don't have one line with 78
 characters and the second with only four. Split the comment between the two.
 
+### Default values
+
+Whenever a constant or function parameter is defined, try and have the safer or
+default value be the `zero` value. For example, instead of `const NoDanger`, use
+`const AllowDanger` so that the `false` value is the safe scenario.
+
 ### Method receiver naming
 
 [Contrary](https://github.com/golang/go/wiki/CodeReviewComments#receiver-names)
@@ -84,6 +90,57 @@ func (obj *Foo) Bar(baz string) int {
 }
 ```
 
+### Variable naming
+
+We prefer shorter, scoped variables rather than `unnecessarilyLongIdentifiers`.
+Remember the scoping rules and feel free to use new variables where appropriate.
+For example, in a short string snippet you can use `s` instead of `myString`, as
+well as other common choices. `i` is a common `int` counter, `f` for files, `fn`
+for functions, `x` for something else and so on.
+
+### Variable re-use
+
+Feel free to create and use new variables instead of attempting to re-use the
+same string. For example, if a function input arg is named `s`, you can use a
+new variable to receive the first computation result on `s` instead of storing
+it back into the original `s`. This avoids confusion if a different part of the
+code wants to read the original input, and it avoids any chance of edit by
+reference of the original callers copy of the variable.
+
+#### Example
+
+```golang
+MyNotIdealFunc(s string, b bool) string {
+	if !b {
+		return s + "hey"
+	}
+	s = strings.Replace(s, "blah", "", -1) // not ideal (re-use of `s` var)
+	return s
+}
+
+MyOkayFunc(s string, b bool) string {
+	if !b {
+		return s + "hey"
+	}
+	s2 := strings.Replace(s, "blah", "", -1) // doesn't re-use `s` variable
+	return s2
+}
+
+MyGreatFunc(s string, b bool) string {
+	if !b {
+		return s + "hey"
+	}
+	return strings.Replace(s, "blah", "", -1) // even cleaner
+}
+```
+
+### Constants in code
+
+If a function takes a specifier (often a bool) it's sometimes better to name
+that variable (often with a `const`) rather than leaving a naked `bool` in the
+code. For example, `x := MyFoo("blah", false)` is less clear than
+`const useMagic = false; x := MyFoo("blah", useMagic)`.
+
 ### Consistent ordering
 
 In general we try to preserve a logical ordering in source files which usually
@@ -96,6 +153,23 @@ declared in the interface.
 When implementing code for the various types in the language, please follow this
 order: `bool`, `str`, `int`, `float`, `list`, `map`, `struct`, `func`.
 
+For other aspects where you have a set of items, try to be internally consistent
+as well. For example, if you have two switch statements with `A`, `B`, and `C`,
+please use the same ordering for these elements elsewhere that they appear in
+the code and in the commentary if it is not illogical to do so.
+
+### Product identifiers
+
+Try to avoid references in the code to `mgmt` or a specific program name string
+if possible. This makes it easier to rename code if we ever pick a better name
+or support `libmgmt` better if we embed it. You can use the `Program` variable
+which is available in numerous places if you want a string to put in the logs.
+
+It is also recommended to avoid the `go` (programming language name) string if
+possible. Try to use `golang` if required, since the word `go` is already
+overloaded, and in particular it was even already used by the
+[`go!`](https://en.wikipedia.org/wiki/Go!_(programming_language)).
+
 ## Overview for mcl code
 
 The `mcl` language is quite new, so this guide will probably change over time as

engine/autoedge.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -68,7 +68,8 @@ type AutoEdge interface {
 	Test([]bool) bool // call until false
 }
 
-// ResUID is a unique identifier for a resource, namely it's name, and the kind ("type").
+// ResUID is a unique identifier for a resource, namely it's name, and the kind
+// ("type").
 type ResUID interface {
 	fmt.Stringer // String() string
 
@@ -104,9 +105,9 @@ func (obj *BaseUID) String() string {
 }
 
 // IFF looks at two UID's and if and only if they are equivalent, returns true.
-// If they are not equivalent, it returns false.
-// Most resources will want to override this method, since it does the important
-// work of actually discerning if two resources are identical in function.
+// If they are not equivalent, it returns false. Most resources will want to
+// override this method, since it does the important work of actually discerning
+// if two resources are identical in function.
 func (obj *BaseUID) IFF(uid ResUID) bool {
 	res, ok := uid.(*BaseUID)
 	if !ok {

engine/autoedge_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/autogroup.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -39,7 +39,7 @@ type GroupableRes interface {
 	SetAutoGroupMeta(*AutoGroupMeta)
 
 	// GroupCmp compares two resources and decides if they're suitable for
-	//grouping. This usually needs to be unique to your resource.
+	// grouping. This usually needs to be unique to your resource.
 	GroupCmp(res GroupableRes) error
 
 	// GroupRes groups resource argument (res) into self.

engine/cmp.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -152,6 +152,18 @@ func ResCmp(r1, r2 Res) error {
 		}
 	}
 
+	// compare meta params for resources with reversible traits
+	r1v, ok1 := r1.(ReversibleRes)
+	r2v, ok2 := r2.(ReversibleRes)
+	if ok1 != ok2 {
+		return fmt.Errorf("reversible differs") // they must be different (optional)
+	}
+	if ok1 && ok2 {
+		if r1v.ReversibleMeta().Cmp(r2v.ReversibleMeta()) != nil {
+			return fmt.Errorf("reversible differs")
+		}
+	}
+
 	return nil
 }
 
@@ -280,6 +292,18 @@ func AdaptCmp(r1, r2 CompatibleRes) error {
 		}
 	}
 
+	// compare meta params for resources with reversible traits
+	r1v, ok1 := r1.(ReversibleRes)
+	r2v, ok2 := r2.(ReversibleRes)
+	if ok1 != ok2 {
+		return fmt.Errorf("reversible differs") // they must be different (optional)
+	}
+	if ok1 && ok2 {
+		if r1v.ReversibleMeta().Cmp(r2v.ReversibleMeta()) != nil {
+			return fmt.Errorf("reversible differs")
+		}
+	}
+
 	return nil
 }
 

engine/copy.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -20,7 +20,7 @@ package engine
 import (
 	"fmt"
 
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 // ResCopy copies a resource. This is the main entry point for copying a
@@ -106,6 +106,16 @@ func ResCopy(r CopyableRes) (CopyableRes, error) {
 		}
 	}
 
+	// copy meta params for resources with reversible traits
+	if x, ok := r.(ReversibleRes); ok {
+		dst, ok := res.(ReversibleRes)
+		if !ok {
+			// programming error
+			panic("reversible interfaces are illogical")
+		}
+		dst.SetReversibleMeta(x.ReversibleMeta()) // no need to copy atm
+	}
+
 	return res, nil
 }
 

engine/edge.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/error.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/fs.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/actions.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -25,9 +25,8 @@ import (
 
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/util/errwrap"
 
-	multierr "github.com/hashicorp/go-multierror"
-	errwrap "github.com/pkg/errors"
 	"golang.org/x/time/rate"
 )
 
@@ -377,8 +376,8 @@ Loop:
 			// we then save so we can return it to the caller of us.
 			if err != nil {
 				failed = true
-				close(obj.state[vertex].watchDone)    // causes doneChan to close
-				reterr = multierr.Append(reterr, err) // permanent failure
+				close(obj.state[vertex].watchDone)   // causes doneChan to close
+				reterr = errwrap.Append(reterr, err) // permanent failure
 				continue
 			}
 			if obj.Debug {
@@ -458,8 +457,8 @@ Loop:
 					}
 					if e != nil {
 						failed = true
-						close(obj.state[vertex].limitDone)  // causes doneChan to close
-						reterr = multierr.Append(reterr, e) // permanent failure
+						close(obj.state[vertex].limitDone) // causes doneChan to close
+						reterr = errwrap.Append(reterr, e) // permanent failure
 						break LimitWait
 					}
 					if obj.Debug {
@@ -498,8 +497,8 @@ Loop:
 						}
 						if e != nil {
 							failed = true
-							close(obj.state[vertex].limitDone)  // causes doneChan to close
-							reterr = multierr.Append(reterr, e) // permanent failure
+							close(obj.state[vertex].limitDone) // causes doneChan to close
+							reterr = errwrap.Append(reterr, e) // permanent failure
 							break RetryWait
 						}
 						if obj.Debug {
@@ -546,8 +545,8 @@ Loop:
 			// this dies. If Process fails permanently, we ask it
 			// to exit right here... (It happens when we loop...)
 			failed = true
-			close(obj.state[vertex].processDone)  // causes doneChan to close
-			reterr = multierr.Append(reterr, err) // permanent failure
+			close(obj.state[vertex].processDone) // causes doneChan to close
+			reterr = errwrap.Append(reterr, err) // permanent failure
 			continue
 
 		} // retry loop

engine/graph/autoedge.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/autoedge/autoedge.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -22,9 +22,7 @@ import (
 
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/pgraph"
-
-	multierr "github.com/hashicorp/go-multierror"
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 // AutoEdge adds the automatic edges to the graph.
@@ -49,7 +47,7 @@ func AutoEdge(graph *pgraph.Graph, debug bool, logf func(format string, v ...int
 	for _, res := range sorted { // for each vertexes autoedges
 		autoEdgeObj, e := res.AutoEdges()
 		if e != nil {
-			err = multierr.Append(err, e) // collect all errors
+			err = errwrap.Append(err, e) // collect all errors
 			continue
 		}
 		if autoEdgeObj == nil {
@@ -91,6 +89,9 @@ func AutoEdge(graph *pgraph.Graph, debug bool, logf func(format string, v ...int
 			}
 		}
 	}
+
+	// It would be great to ensure we didn't add any graph cycles here, but
+	// instead of checking now, we'll move the check into the main loop.
 	return nil
 }
 

engine/graph/autogroup.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -24,8 +24,7 @@ import (
 	"github.com/purpleidea/mgmt/engine/graph/autogroup"
 	"github.com/purpleidea/mgmt/pgraph"
 	"github.com/purpleidea/mgmt/util"
-
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 // AutoGroup runs the auto grouping on the loaded graph.
@@ -75,10 +74,10 @@ func (obj *wrappedGrouper) VertexCmp(v1, v2 pgraph.Vertex) error {
 		return fmt.Errorf("v2 is not a GroupableRes")
 	}
 
-	if r1.Kind() != r2.Kind() { // we must group similar kinds
-		// TODO: maybe future resources won't need this limitation?
-		return fmt.Errorf("the two resources aren't the same kind")
-	}
+	// Some resources of different kinds can now group together!
+	//if r1.Kind() != r2.Kind() { // we must group similar kinds
+	//	return fmt.Errorf("the two resources aren't the same kind")
+	//}
 	// someone doesn't want to group!
 	if r1.AutoGroupMeta().Disabled || r2.AutoGroupMeta().Disabled {
 		return fmt.Errorf("one of the autogroup flags is false")

engine/graph/autogroup/autogroup.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -22,8 +22,7 @@ import (
 
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/pgraph"
-
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 // AutoGroup is the mechanical auto group "runner" that runs the interface spec.
@@ -55,7 +54,7 @@ func AutoGroup(ag engine.AutoGrouper, g *pgraph.Graph, debug bool, logf func(for
 			logf("!VertexMerge for: %s into: %s", wStr, vStr)
 
 		} else { // success!
-			logf("success for: %s into: %s", wStr, vStr)
+			logf("%s into %s", wStr, vStr)
 			merged = true // woo
 		}
 
@@ -67,5 +66,8 @@ func AutoGroup(ag engine.AutoGrouper, g *pgraph.Graph, debug bool, logf func(for
 		}
 	}
 
+	// It would be great to ensure we didn't add any graph cycles here, but
+	// instead of checking now, we'll move the check into the main loop.
+
 	return nil
 }

engine/graph/autogroup/autogroup_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -31,8 +31,7 @@ import (
 	"github.com/purpleidea/mgmt/engine/traits"
 	"github.com/purpleidea/mgmt/pgraph"
 	"github.com/purpleidea/mgmt/util"
-
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 func init() {
@@ -596,10 +595,11 @@ func TestPgraphGrouping11(t *testing.T) {
 	runGraphCmp(t, g1, g2)
 }
 
-// simple merge 1
+/* simple merge 1
 // a1   a2         a1,a2
 //   \ /     >>>     |     (arrows point downwards)
 //    b              b
+*/
 func TestPgraphGrouping12(t *testing.T) {
 	g1, _ := pgraph.NewGraph("g1") // original graph
 	{
@@ -621,10 +621,11 @@ func TestPgraphGrouping12(t *testing.T) {
 	runGraphCmp(t, g1, g2)
 }
 
-// simple merge 2
+/* simple merge 2
 //    b              b
 //   / \     >>>     |     (arrows point downwards)
 // a1   a2         a1,a2
+*/
 func TestPgraphGrouping13(t *testing.T) {
 	g1, _ := pgraph.NewGraph("g1") // original graph
 	{
@@ -646,10 +647,11 @@ func TestPgraphGrouping13(t *testing.T) {
 	runGraphCmp(t, g1, g2)
 }
 
-// triple merge
+/* triple merge
 // a1 a2  a3         a1,a2,a3
 //   \ | /     >>>       |      (arrows point downwards)
 //     b                 b
+*/
 func TestPgraphGrouping14(t *testing.T) {
 	g1, _ := pgraph.NewGraph("g1") // original graph
 	{
@@ -674,12 +676,13 @@ func TestPgraphGrouping14(t *testing.T) {
 	runGraphCmp(t, g1, g2)
 }
 
-// chain merge
+/* chain merge
 //    a1             a1
 //   /  \             |
 // b1    b2   >>>   b1,b2   (arrows point downwards)
 //   \  /             |
 //    c1             c1
+*/
 func TestPgraphGrouping15(t *testing.T) {
 	g1, _ := pgraph.NewGraph("g1") // original graph
 	{
@@ -709,7 +712,7 @@ func TestPgraphGrouping15(t *testing.T) {
 	runGraphCmp(t, g1, g2)
 }
 
-// re-attach 1 (outer)
+/* re-attach 1 (outer)
 // technically the second possibility is valid too, depending on which order we
 // merge edges in, and if we don't filter out any unnecessary edges afterwards!
 // a1    a2         a1,a2        a1,a2
@@ -717,6 +720,7 @@ func TestPgraphGrouping15(t *testing.T) {
 // b1  /      >>>    b1     OR    b1  /   (arrows point downwards)
 //  | /               |            | /
 // c1                c1           c1
+*/
 func TestPgraphGrouping16(t *testing.T) {
 	g1, _ := pgraph.NewGraph("g1") // original graph
 	{
@@ -744,12 +748,13 @@ func TestPgraphGrouping16(t *testing.T) {
 	runGraphCmp(t, g1, g2)
 }
 
-// re-attach 2 (inner)
+/* re-attach 2 (inner)
 // a1    b2          a1
 //  |   /             |
 // b1  /      >>>   b1,b2   (arrows point downwards)
 //  | /               |
 // c1                c1
+*/
 func TestPgraphGrouping17(t *testing.T) {
 	g1, _ := pgraph.NewGraph("g1") // original graph
 	{
@@ -777,13 +782,14 @@ func TestPgraphGrouping17(t *testing.T) {
 	runGraphCmp(t, g1, g2)
 }
 
-// re-attach 3 (double)
+/* re-attach 3 (double)
 // similar to "re-attach 1", technically there is a second possibility for this
 // a2   a1    b2         a1,a2
 //   \   |   /             |
 //    \ b1  /      >>>   b1,b2   (arrows point downwards)
 //     \ | /               |
 //      c1                c1
+*/
 func TestPgraphGrouping18(t *testing.T) {
 	g1, _ := pgraph.NewGraph("g1") // original graph
 	{
@@ -814,10 +820,11 @@ func TestPgraphGrouping18(t *testing.T) {
 	runGraphCmp(t, g1, g2)
 }
 
-// connected merge 0, (no change!)
+/* connected merge 0, (no change!)
 // a1            a1
 //   \     >>>     \     (arrows point downwards)
 //    a2            a2
+*/
 func TestPgraphGroupingConnected0(t *testing.T) {
 	g1, _ := pgraph.NewGraph("g1") // original graph
 	{
@@ -836,12 +843,13 @@ func TestPgraphGroupingConnected0(t *testing.T) {
 	runGraphCmp(t, g1, g2)
 }
 
-// connected merge 1, (no change!)
+/* connected merge 1, (no change!)
 // a1              a1
 //   \               \
 //    b      >>>      b      (arrows point downwards)
 //     \               \
 //      a2              a2
+*/
 func TestPgraphGroupingConnected1(t *testing.T) {
 	g1, _ := pgraph.NewGraph("g1") // original graph
 	{

engine/graph/autogroup/base.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -89,6 +89,19 @@ func (ag *baseGrouper) VertexNext() (v1, v2 pgraph.Vertex, err error) {
 			ag.done = true
 		}
 	}
+	// TODO: is this index swap better or even valid?
+	//if ag.i < l {
+	//	ag.i++
+	//}
+	//if ag.i == l {
+	//	ag.i = 0
+	//	if ag.j < l {
+	//		ag.j++
+	//	}
+	//	if ag.j == l {
+	//		ag.done = true
+	//	}
+	//}
 
 	return
 }
@@ -110,7 +123,7 @@ func (ag *baseGrouper) VertexMerge(v1, v2 pgraph.Vertex) (v pgraph.Vertex, err e
 	return nil, fmt.Errorf("vertexMerge needs to be overridden")
 }
 
-// EdgeMerge can be overridden, since it just simple returns the first edge.
+// EdgeMerge can be overridden, since it just simply returns the first edge.
 func (ag *baseGrouper) EdgeMerge(e1, e2 pgraph.Edge) pgraph.Edge {
 	return e1 // noop
 }

engine/graph/autogroup/nonreachability.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -19,8 +19,7 @@ package autogroup
 
 import (
 	"github.com/purpleidea/mgmt/pgraph"
-
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 // NonReachabilityGrouper is the most straight-forward algorithm for grouping.

engine/graph/autogroup/util.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -18,14 +18,16 @@
 package autogroup
 
 import (
-	"github.com/purpleidea/mgmt/pgraph"
+	"fmt"
 
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/engine"
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
-// VertexMerge merges v2 into v1 by reattaching the edges where appropriate,
-// and then by deleting v2 from the graph. Since more than one edge between two
-// vertices is not allowed, duplicate edges are merged as well. an edge merge
+// VertexMerge merges v2 into v1 by reattaching the edges where appropriate, and
+// then by deleting v2 from the graph. Since more than one edge between two
+// vertices is not allowed, duplicate edges are merged as well. An edge merge
 // function can be provided if you'd like to control how you merge the edges!
 func VertexMerge(g *pgraph.Graph, v1, v2 pgraph.Vertex, vertexMergeFn func(pgraph.Vertex, pgraph.Vertex) (pgraph.Vertex, error), edgeMergeFn func(pgraph.Edge, pgraph.Edge) pgraph.Edge) error {
 	// methodology
@@ -113,8 +115,17 @@ func VertexMerge(g *pgraph.Graph, v1, v2 pgraph.Vertex, vertexMergeFn func(pgrap
 			// note: This branch isn't used if the vertexMergeFn
 			// decides to just merge logically on its own instead
 			// of actually returning something that we then merge.
-			v1 = v // TODO: ineffassign?
+			v1 = v // XXX: ineffassign?
 			//*v1 = *v
+
+			// Ensure that everything still validates. (For safety!)
+			r, ok := v1.(engine.Res) // TODO: v ?
+			if !ok {
+				return fmt.Errorf("not a Res")
+			}
+			if err := engine.Validate(r); err != nil {
+				return errwrap.Wrapf(err, "the Res did not Validate")
+			}
 		}
 	}
 	g.DeleteVertex(v2) // remove grouped vertex

engine/graph/engine.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -25,11 +25,16 @@ import (
 
 	"github.com/purpleidea/mgmt/converger"
 	"github.com/purpleidea/mgmt/engine"
+	engineUtil "github.com/purpleidea/mgmt/engine/util"
 	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/util/errwrap"
 	"github.com/purpleidea/mgmt/util/semaphore"
+)
 
-	multierr "github.com/hashicorp/go-multierror"
-	errwrap "github.com/pkg/errors"
+const (
+	// StateDir is the name of the sub directory where all the local
+	// resource state is stored.
+	StateDir = "state"
 )
 
 // Engine encapsulates a generic graph and manages its operations.
@@ -50,6 +55,7 @@ type Engine struct {
 	nextGraph *pgraph.Graph
 	state     map[pgraph.Vertex]*State
 	waits     map[pgraph.Vertex]*sync.WaitGroup // wg for the Worker func
+	wlock     *sync.Mutex                       // lock around waits map
 
 	slock *sync.Mutex // semaphore lock
 	semas map[string]*semaphore.Semaphore
@@ -64,6 +70,13 @@ type Engine struct {
 // If the struct does not validate, or it cannot initialize, then this errors.
 // Initially it will contain an empty graph.
 func (obj *Engine) Init() error {
+	if obj.Program == "" {
+		return fmt.Errorf("the Program is empty")
+	}
+	if obj.Hostname == "" {
+		return fmt.Errorf("the Hostname is empty")
+	}
+
 	var err error
 	if obj.graph, err = pgraph.NewGraph("graph"); err != nil {
 		return err
@@ -78,6 +91,7 @@ func (obj *Engine) Init() error {
 
 	obj.state = make(map[pgraph.Vertex]*State)
 	obj.waits = make(map[pgraph.Vertex]*sync.WaitGroup)
+	obj.wlock = &sync.Mutex{}
 
 	obj.slock = &sync.Mutex{}
 	obj.semas = make(map[string]*semaphore.Semaphore)
@@ -167,9 +181,9 @@ func (obj *Engine) Commit() error {
 			return errwrap.Wrapf(err, "the Res did not Validate")
 		}
 
-		// FIXME: is res.Name() sufficiently unique to use as a UID here?
-		pathUID := fmt.Sprintf("%s-%s", res.Kind(), res.Name())
-		statePrefix := fmt.Sprintf("%s/", path.Join(obj.Prefix, "state", pathUID))
+		pathUID := engineUtil.ResPathUID(res)
+		statePrefix := fmt.Sprintf("%s/", path.Join(obj.statePrefix(), pathUID))
+
 		// don't create this unless it *will* be used
 		//if err := os.MkdirAll(statePrefix, 0770); err != nil {
 		//	return errwrap.Wrapf(err, "can't create state prefix")
@@ -177,7 +191,7 @@ func (obj *Engine) Commit() error {
 
 		obj.waits[vertex] = &sync.WaitGroup{}
 		obj.state[vertex] = &State{
-			//Graph: obj.graph, // TODO: what happens if we swap the graph?
+			Graph:  obj.graph, // Update if we swap the graph!
 			Vertex: vertex,
 
 			Program:  obj.Program,
@@ -199,10 +213,19 @@ func (obj *Engine) Commit() error {
 		fn := func() error {
 			// start the Worker
 			obj.wg.Add(1)
+			obj.wlock.Lock()
 			obj.waits[vertex].Add(1)
+			obj.wlock.Unlock()
 			go func(v pgraph.Vertex) {
 				defer obj.wg.Done()
-				defer obj.waits[v].Done()
+				defer func() {
+					// we need this lock, because this go
+					// routine could run when the next fn
+					// function above here is running...
+					obj.wlock.Lock()
+					obj.waits[v].Done()
+					obj.wlock.Unlock()
+				}()
 
 				obj.Logf("Worker(%s)", v)
 				// contains the Watch and CheckApply loops
@@ -306,14 +329,14 @@ func (obj *Engine) Commit() error {
 	// the changes that we'd made to the previously primary graph. This is
 	// because this function is meant to atomically swap the graphs safely.
 
-	// TODO: update all the `State` structs with the new Graph pointer
-	//for _, vertex := range obj.graph.Vertices() {
-	//	state, exists := obj.state[vertex]
-	//	if !exists {
-	//		continue
-	//	}
-	//	state.Graph = obj.graph // update pointer to graph
-	//}
+	// Update all the `State` structs with the new Graph pointer.
+	for _, vertex := range obj.graph.Vertices() {
+		state, exists := obj.state[vertex]
+		if !exists {
+			continue
+		}
+		state.Graph = obj.graph // update pointer to graph
+	}
 
 	return nil
 }
@@ -380,20 +403,16 @@ func (obj *Engine) Pause(fastPause bool) error {
 
 // Close triggers a shutdown. Engine must be already paused before this is run.
 func (obj *Engine) Close() error {
-	var reterr error
-
-	emptyGraph, err := pgraph.NewGraph("empty")
-	if err != nil {
-		reterr = multierr.Append(reterr, err) // list of errors
-	}
+	emptyGraph, reterr := pgraph.NewGraph("empty")
 
 	// this is a graph switch (graph sync) that switches to an empty graph!
 	if err := obj.Load(emptyGraph); err != nil { // copy in empty graph
-		reterr = multierr.Append(reterr, err)
+		reterr = errwrap.Append(reterr, err)
 	}
+	// FIXME: Do we want to run commit if Load failed? Does this even work?
 	// the commit will cause the graph sync to shut things down cleverly...
 	if err := obj.Commit(); err != nil {
-		reterr = multierr.Append(reterr, err)
+		reterr = errwrap.Append(reterr, err)
 	}
 
 	obj.wg.Wait() // for now, this doesn't need to be a separate Wait() method
@@ -404,3 +423,8 @@ func (obj *Engine) Close() error {
 func (obj *Engine) Graph() *pgraph.Graph {
 	return obj.graph
 }
+
+// statePrefix returns the dir where all the resource state is stored locally.
+func (obj *Engine) statePrefix() string {
+	return fmt.Sprintf("%s/", path.Join(obj.Prefix, StateDir))
+}

engine/graph/graph_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -23,13 +23,13 @@ import (
 	"fmt"
 	"testing"
 
-	multierr "github.com/hashicorp/go-multierror"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 func TestMultiErr(t *testing.T) {
 	var err error
 	e := fmt.Errorf("some error")
-	err = multierr.Append(err, e) // build an error from a nil base
+	err = errwrap.Append(err, e) // build an error from a nil base
 	// ensure that this lib allows us to append to a nil
 	if err == nil {
 		t.Errorf("missing error")

engine/graph/refresh.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/reverse.go

@@ -0,0 +1,300 @@
+// Mgmt
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package graph
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"sort"
+
+	"github.com/purpleidea/mgmt/engine"
+	engineUtil "github.com/purpleidea/mgmt/engine/util"
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/util/errwrap"
+)
+
+const (
+	// ReverseFile is the file name in the resource state dir where any
+	// reversal information is stored.
+	ReverseFile = "reverse"
+
+	// ReversePerm is the permissions mode used to create the ReverseFile.
+	ReversePerm = 0600
+)
+
+// Reversals adds the reversals onto the loaded graph. This should happen last,
+// and before Commit.
+func (obj *Engine) Reversals() error {
+	if obj.nextGraph == nil {
+		return fmt.Errorf("there is no active graph to add reversals to")
+	}
+
+	// Initially get all of the reversals to seek out all possible errors.
+	// XXX: The engine needs to know where data might have been stored if we
+	// XXX: want to potentially allow alternate read/write paths, like etcd.
+	// XXX: In this scenario, we'd have to store a token somewhere to let us
+	// XXX: know to look elsewhere for the special ReversalList read method.
+	data, err := obj.ReversalList() // (map[string]string, error)
+	if err != nil {
+		return errwrap.Wrapf(err, "the reversals had errors")
+	}
+
+	if len(data) == 0 {
+		return nil // end early
+	}
+
+	resMatch := func(r1, r2 engine.Res) bool { // simple match on UID only!
+		if r1.Kind() != r2.Kind() {
+			return false
+		}
+		if r1.Name() != r2.Name() {
+			return false
+		}
+		return true
+	}
+	resInList := func(needle engine.Res, haystack []engine.Res) bool {
+		for _, res := range haystack {
+			if resMatch(needle, res) {
+				return true
+			}
+		}
+		return false
+	}
+
+	if obj.Debug {
+		obj.Logf("decoding %d reversals...", len(data))
+	}
+	resources := []engine.Res{}
+
+	// do this in a sorted order so that it errors deterministically
+	sorted := []string{}
+	for key := range data {
+		sorted = append(sorted, key)
+	}
+	sort.Strings(sorted)
+	for _, key := range sorted {
+		val := data[key]
+		// XXX: replace this ResToB64 method with one that stores it in
+		// a human readable format, in case someone wants to hack and
+		// edit it manually.
+		// XXX: we probably want this to be YAML, it works with the diff
+		// too...
+		r, err := engineUtil.B64ToRes(val)
+		if err != nil {
+			return errwrap.Wrapf(err, "error decoding res with UID: `%s`", key)
+		}
+
+		res, ok := r.(engine.ReversibleRes)
+		if !ok {
+			// this requirement is here to keep things simpler...
+			return errwrap.Wrapf(err, "decoded res with UID: `%s` was not reversible", key)
+		}
+
+		matchFn := func(vertex pgraph.Vertex) (bool, error) {
+			r, ok := vertex.(engine.Res)
+			if !ok {
+				return false, fmt.Errorf("not a Res")
+			}
+			if !resMatch(r, res) {
+				return false, nil
+			}
+			return true, nil
+		}
+
+		// FIXME: not efficient, we could build a cache-map first
+		vertex, err := obj.nextGraph.VertexMatchFn(matchFn) // (Vertex, error)
+		if err != nil {
+			return errwrap.Wrapf(err, "error searching graph for match")
+		}
+		if vertex != nil { // found one!
+			continue // it doesn't need reversing yet
+		}
+
+		// TODO: check for (incompatible?) duplicates instead
+		if resInList(res, resources) { // we've already got this one...
+			continue
+		}
+
+		// We set this in two different places to be safe. It ensures
+		// that we erase the reversal state file after we've used it.
+		res.ReversibleMeta().Reversal = true // set this for later...
+
+		resources = append(resources, res)
+	}
+
+	if len(resources) == 0 {
+		return nil // end early
+	}
+
+	// Now that we've passed the chance of any errors, we modify the graph.
+	obj.Logf("adding %d reversals...", len(resources))
+	for _, res := range resources {
+		obj.nextGraph.AddVertex(res)
+	}
+	// TODO: Do we want a way for stored reversals to add edges too?
+
+	// It would be great to ensure we didn't add any graph cycles here, but
+	// instead of checking now, we'll move the check into the main loop.
+	return nil
+}
+
+// ReversalList returns all the available pending reversal data on this host. It
+// can then be decoded by whatever method is appropriate for.
+func (obj *Engine) ReversalList() (map[string]string, error) {
+	result := make(map[string]string) // some key to contents
+
+	dir := obj.statePrefix() // loop through this dir...
+	files, err := ioutil.ReadDir(dir)
+	if err != nil && !os.IsNotExist(err) {
+		return nil, errwrap.Wrapf(err, "error reading list of state dirs")
+	} else if err != nil {
+		return result, nil // nothing found, no state dir exists yet
+	}
+
+	for _, x := range files {
+		key := x.Name() // some uid for the resource
+		file := path.Join(dir, key, ReverseFile)
+		content, err := ioutil.ReadFile(file)
+		if err != nil && !os.IsNotExist(err) {
+			return nil, errwrap.Wrapf(err, "could not read reverse file: %s", file)
+		} else if err != nil {
+			continue // file does not exist, skip
+		}
+
+		// file exists!
+		str := string(content)
+		result[key] = str // save
+	}
+
+	return result, nil
+}
+
+// ReversalInit performs the reversal initialization steps if necessary for this
+// resource.
+func (obj *State) ReversalInit() error {
+	res, ok := obj.Vertex.(engine.ReversibleRes)
+	if !ok {
+		return nil // nothing to do
+	}
+
+	if res.ReversibleMeta().Disabled {
+		return nil // nothing to do, reversal isn't enabled
+	}
+
+	// If the reversal is enabled, but we are the result of a previous
+	// reversal, then this will overwrite that older reversal request, and
+	// our resource should be designed to deal with that. This happens if we
+	// return a reversible resource as the reverse of a resource that was
+	// reversed. It's probably fairly rare.
+	if res.ReversibleMeta().Reversal {
+		obj.Logf("triangle reversal") // warn!
+	}
+
+	r, err := res.Reversed()
+	if err != nil {
+		return errwrap.Wrapf(err, "could not reverse: %s", res.String())
+	}
+	if r == nil {
+		return nil // this can't be reversed, or isn't implemented here
+	}
+
+	// We set this in two different places to be safe. It ensures that we
+	// erase the reversal state file after we've used it.
+	r.ReversibleMeta().Reversal = true // set this for later...
+
+	// XXX: replace this ResToB64 method with one that stores it in a human
+	// readable format, in case someone wants to hack and edit it manually.
+	// XXX: we probably want this to be YAML, it works with the diff too...
+	str, err := engineUtil.ResToB64(r)
+	if err != nil {
+		return errwrap.Wrapf(err, "could not encode: %s", res.String())
+	}
+
+	// TODO: put this method on traits.Reversible as part of the interface?
+	return obj.ReversalWrite(str, res.ReversibleMeta().Overwrite) // Store!
+}
+
+// ReversalClose performs the reversal shutdown steps if necessary for this
+// resource.
+func (obj *State) ReversalClose() error {
+	res, ok := obj.Vertex.(engine.ReversibleRes)
+	if !ok {
+		return nil // nothing to do
+	}
+
+	// Don't check res.ReversibleMeta().Disabled because we're removing the
+	// previous one. That value only applies if we're doing a new reversal.
+
+	if !res.ReversibleMeta().Reversal {
+		return nil // nothing to erase, we're not a reversal resource
+	}
+
+	if !obj.isStateOK { // did we successfully reverse?
+		obj.Logf("did not complete reversal") // warn
+		return nil
+	}
+
+	// TODO: put this method on traits.Reversible as part of the interface?
+	return obj.ReversalDelete() // Erase our reversal instructions.
+}
+
+// ReversalWrite stores the reversal state information for this resource.
+func (obj *State) ReversalWrite(str string, overwrite bool) error {
+	dir, err := obj.varDir("") // private version
+	if err != nil {
+		return errwrap.Wrapf(err, "could not get VarDir for reverse")
+	}
+	file := path.Join(dir, ReverseFile) // return a unique file
+
+	content, err := ioutil.ReadFile(file)
+	if err != nil && !os.IsNotExist(err) {
+		return errwrap.Wrapf(err, "could not read reverse file: %s", file)
+	}
+
+	// file exists and we shouldn't overwrite if different
+	if err == nil && !overwrite {
+		// compare to existing file
+		oldStr := string(content)
+		if str != oldStr {
+			obj.Logf("existing, pending, reversible resource exists")
+			//obj.Logf("diff:")
+			//obj.Logf("") // TODO: print the diff w/o and secret values
+			return fmt.Errorf("existing, pending, reversible resource exists")
+		}
+	}
+
+	return ioutil.WriteFile(file, []byte(str), ReversePerm)
+}
+
+// ReversalDelete removes the reversal state information for this resource.
+func (obj *State) ReversalDelete() error {
+	dir, err := obj.varDir("") // private version
+	if err != nil {
+		return errwrap.Wrapf(err, "could not get VarDir for reverse")
+	}
+	file := path.Join(dir, ReverseFile) // return a unique file
+
+	// FIXME: why do we see these removals when there isn't a state file?
+	if err = os.Remove(file); os.IsNotExist(err) {
+		return nil // ignore missing files
+	}
+
+	return errwrap.Wrapf(err, "could not remove reverse state file")
+}

engine/graph/semaphore.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -23,9 +23,8 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/purpleidea/mgmt/util/errwrap"
 	"github.com/purpleidea/mgmt/util/semaphore"
-
-	multierr "github.com/hashicorp/go-multierror"
 )
 
 // SemaSep is the trailing separator to split the semaphore id from the size.
@@ -46,9 +45,8 @@ func (obj *Engine) semaLock(semas []string) error {
 		}
 		obj.slock.Unlock()
 
-		if err := sema.P(1); err != nil { // lock!
-			reterr = multierr.Append(reterr, err) // list of errors
-		}
+		err := sema.P(1)                     // lock!
+		reterr = errwrap.Append(reterr, err) // list of errors
 	}
 	return reterr
 }
@@ -65,9 +63,8 @@ func (obj *Engine) semaUnlock(semas []string) error {
 			panic(fmt.Sprintf("graph: sema: %s does not exist", id))
 		}
 
-		if err := sema.V(1); err != nil { // unlock!
-			reterr = multierr.Append(reterr, err) // list of errors
-		}
+		err := sema.V(1)                     // unlock!
+		reterr = errwrap.Append(reterr, err) // list of errors
 	}
 	return reterr
 }

engine/graph/semaphore_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/sendrecv.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -22,9 +22,8 @@ import (
 	"reflect"
 
 	"github.com/purpleidea/mgmt/engine"
-
-	multierr "github.com/hashicorp/go-multierror"
-	errwrap "github.com/pkg/errors"
+	engineUtil "github.com/purpleidea/mgmt/engine/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 // SendRecv pulls in the sent values into the receive slots. It is called by the
@@ -47,16 +46,51 @@ func (obj *Engine) SendRecv(res engine.RecvableRes) (map[string]bool, error) {
 			st = v.Res.Sent()
 		}
 
+		if st == nil {
+			e := fmt.Errorf("received nil value from: %s", v.Res)
+			err = errwrap.Append(err, e) // list of errors
+			continue
+		}
+
+		if e := engineUtil.StructFieldCompat(st, v.Key, res, k); e != nil {
+			err = errwrap.Append(err, e) // list of errors
+			continue
+		}
+
 		// send
+		m1, e := engineUtil.StructTagToFieldName(st)
+		if e != nil {
+			err = errwrap.Append(err, e) // list of errors
+			continue
+		}
+		key1, exists := m1[v.Key]
+		if !exists {
+			e := fmt.Errorf("requested key of `%s` not found in send struct", v.Key)
+			err = errwrap.Append(err, e) // list of errors
+			continue
+		}
+
 		obj1 := reflect.Indirect(reflect.ValueOf(st))
 		type1 := obj1.Type()
-		value1 := obj1.FieldByName(v.Key)
+		value1 := obj1.FieldByName(key1)
 		kind1 := value1.Kind()
 
 		// recv
+		m2, e := engineUtil.StructTagToFieldName(res)
+		if e != nil {
+			err = errwrap.Append(err, e) // list of errors
+			continue
+		}
+		key2, exists := m2[k]
+		if !exists {
+			e := fmt.Errorf("requested key of `%s` not found in recv struct", k)
+			err = errwrap.Append(err, e) // list of errors
+			continue
+		}
+
 		obj2 := reflect.Indirect(reflect.ValueOf(res)) // pass in full struct
 		type2 := obj2.Type()
-		value2 := obj2.FieldByName(k)
+		value2 := obj2.FieldByName(key2)
 		kind2 := value2.Kind()
 
 		if obj.Debug {
@@ -67,7 +101,7 @@ func (obj *Engine) SendRecv(res engine.RecvableRes) (map[string]bool, error) {
 		// i think we probably want the same kind, at least for now...
 		if kind1 != kind2 {
 			e := fmt.Errorf("kind mismatch between %s: %s and %s: %s", v.Res, kind1, res, kind2)
-			err = multierr.Append(err, e) // list of errors
+			err = errwrap.Append(err, e) // list of errors
 			continue
 		}
 
@@ -75,21 +109,21 @@ func (obj *Engine) SendRecv(res engine.RecvableRes) (map[string]bool, error) {
 		// FIXME: do we want to relax this for string -> *string ?
 		if e := TypeCmp(value1, value2); e != nil {
 			e := errwrap.Wrapf(e, "type mismatch between %s and %s", v.Res, res)
-			err = multierr.Append(err, e) // list of errors
+			err = errwrap.Append(err, e) // list of errors
 			continue
 		}
 
 		// if we can't set, then well this is pointless!
 		if !value2.CanSet() {
 			e := fmt.Errorf("can't set %s.%s", res, k)
-			err = multierr.Append(err, e) // list of errors
+			err = errwrap.Append(err, e) // list of errors
 			continue
 		}
 
 		// if we can't interface, we can't compare...
 		if !value1.CanInterface() || !value2.CanInterface() {
 			e := fmt.Errorf("can't interface %s.%s", res, k)
-			err = multierr.Append(err, e) // list of errors
+			err = errwrap.Append(err, e) // list of errors
 			continue
 		}
 

engine/graph/state.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -26,14 +26,13 @@ import (
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/pgraph"
 	"github.com/purpleidea/mgmt/util"
-
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 // State stores some state about the resource it is mapped to.
 type State struct {
 	// Graph is a pointer to the graph that this vertex is part of.
-	//Graph pgraph.Graph
+	Graph *pgraph.Graph
 
 	// Vertex is the pointer in the graph that this state corresponds to. It
 	// can be converted to a `Res` if necessary.
@@ -170,25 +169,63 @@ func (obj *State) Init() error {
 			}
 			return res.Refresh()
 		},
-		Send: func(st interface{}) error {
-			res, ok := obj.Vertex.(engine.SendableRes)
-			if !ok {
-				panic("res does not support the Sendable trait")
-			}
-			// XXX: type check this
-			//expected := res.Sends()
-			//if err := XXX_TYPE_CHECK(expected, st); err != nil {
-			//	return err
-			//}
 
-			return res.Send(st) // send the struct
-		},
-		Recv: func() map[string]*engine.Send { // TODO: change this API?
-			res, ok := obj.Vertex.(engine.RecvableRes)
-			if !ok {
-				panic("res does not support the Recvable trait")
+		Send: engine.GenerateSendFunc(res),
+		Recv: engine.GenerateRecvFunc(res),
+
+		// FIXME: pass in a safe, limited query func instead?
+		// TODO: not implemented, use FilteredGraph
+		//Graph: func() *pgraph.Graph {
+		//	_, ok := obj.Vertex.(engine.CanGraphQueryRes)
+		//	if !ok {
+		//		panic("res does not support the GraphQuery trait")
+		//	}
+		//	return obj.Graph // we return in a func so it's fresh!
+		//},
+
+		FilteredGraph: func() (*pgraph.Graph, error) {
+			graph, err := pgraph.NewGraph("filtered")
+			if err != nil {
+				return nil, errwrap.Wrapf(err, "could not create graph")
 			}
-			return res.Recv()
+
+			// filter graph and build a new one...
+			adjacency := obj.Graph.Adjacency()
+			for v1 := range adjacency {
+				// check we're allowed
+				r1, ok := v1.(engine.GraphQueryableRes)
+				if !ok {
+					continue
+				}
+				// pass in information on requestor...
+				if err := r1.GraphQueryAllowed(
+					engine.GraphQueryableOptionKind(res.Kind()),
+					engine.GraphQueryableOptionName(res.Name()),
+					// TODO: add more information...
+				); err != nil {
+					continue
+				}
+				graph.AddVertex(v1)
+
+				for v2, edge := range adjacency[v1] {
+					r2, ok := v2.(engine.GraphQueryableRes)
+					if !ok {
+						continue
+					}
+					// pass in information on requestor...
+					if err := r2.GraphQueryAllowed(
+						engine.GraphQueryableOptionKind(res.Kind()),
+						engine.GraphQueryableOptionName(res.Name()),
+						// TODO: add more information...
+					); err != nil {
+						continue
+					}
+					//graph.AddVertex(v2) // redundant
+					graph.AddEdge(v1, v2, edge)
+				}
+			}
+
+			return graph, nil // we return in a func so it's fresh!
 		},
 
 		World:  obj.World,
@@ -204,6 +241,12 @@ func (obj *State) Init() error {
 	if obj.Debug {
 		obj.Logf("Init(%s)", res)
 	}
+
+	// write the reverse request to the disk...
+	if err := obj.ReversalInit(); err != nil {
+		return err // TODO: test this code path...
+	}
+
 	err := res.Init(obj.init)
 	if obj.Debug {
 		obj.Logf("Init(%s): Return(%+v)", res, err)
@@ -237,12 +280,23 @@ func (obj *State) Close() error {
 	if obj.Debug {
 		obj.Logf("Close(%s)", res)
 	}
-	err := res.Close()
-	if obj.Debug {
-		obj.Logf("Close(%s): Return(%+v)", res, err)
+
+	var reverr error
+	// clear the reverse request from the disk...
+	if err := obj.ReversalClose(); err != nil {
+		// TODO: test this code path...
+		// TODO: should this be an error or a warning?
+		reverr = err
 	}
 
-	return err
+	reterr := res.Close()
+	if obj.Debug {
+		obj.Logf("Close(%s): Return(%+v)", res, reterr)
+	}
+
+	reterr = errwrap.Append(reterr, reverr)
+
+	return reterr
 }
 
 // Poke sends a notification on the poke channel. This channel is used to notify

engine/graph/vardir.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -22,7 +22,7 @@ import (
 	"os"
 	"path"
 
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 // varDir returns the path to a working directory for the resource. It will try

engine/graphqueryable.go

@@ -0,0 +1,70 @@
+// Mgmt
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package engine
+
+// GraphQueryableRes is the interface that must be implemented if you want your
+// resource to be allowed to be queried from another resource in the graph. This
+// is done as a form of explicit authorization tracking so that we can consider
+// security aspects more easily. Ultimately, all resource code should be
+// trusted, but it's still a good idea to know if a particular resource is even
+// able to access information about another one, and if your resource doesn't
+// add the trait supporting this, then it won't be allowed.
+type GraphQueryableRes interface {
+	Res // implement everything in Res but add the additional requirements
+
+	// GraphQueryAllowed returns nil if you're allowed to query the graph.
+	GraphQueryAllowed(...GraphQueryableOption) error
+}
+
+// GraphQueryableOption is an option that can be used to specify the
+// authentication.
+type GraphQueryableOption func(*GraphQueryableOptions)
+
+// GraphQueryableOptions represents the different possible configurable options.
+type GraphQueryableOptions struct {
+	// Kind is the kind of the resource making the access.
+	Kind string
+	// Name is the name of the resource making the access.
+	Name string
+	// TODO: add more options if needed
+}
+
+// Apply is a helper function to apply a list of options to the struct. You
+// should initialize it with defaults you want, and then apply any you've
+// received like this.
+func (obj *GraphQueryableOptions) Apply(opts ...GraphQueryableOption) {
+	for _, optionFunc := range opts { // apply the options
+		optionFunc(obj)
+	}
+}
+
+// GraphQueryableOptionKind tells the GraphQueryAllowed function what the
+// resource kind is.
+func GraphQueryableOptionKind(kind string) GraphQueryableOption {
+	return func(gqo *GraphQueryableOptions) {
+		gqo.Kind = kind
+	}
+}
+
+// GraphQueryableOptionName tells the GraphQueryAllowed function what the
+// resource name is.
+func GraphQueryableOptionName(name string) GraphQueryableOption {
+	return func(gqo *GraphQueryableOptions) {
+		gqo.Name = name
+	}
+}

engine/metaparams.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -22,8 +22,8 @@ import (
 	"strconv"
 
 	"github.com/purpleidea/mgmt/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
 
-	errwrap "github.com/pkg/errors"
 	"golang.org/x/time/rate"
 )
 

engine/metaparams_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/refresh.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/resources.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -21,15 +21,17 @@ import (
 	"encoding/gob"
 	"fmt"
 
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/util/errwrap"
+
 	"gopkg.in/yaml.v2"
 )
 
 // TODO: should each resource be a sub-package?
 var registeredResources = map[string]func() Res{}
 
-// RegisterResource registers a new resource by providing a constructor
-// function that returns a resource object ready to be unmarshalled from YAML.
+// RegisterResource registers a new resource by providing a constructor function
+// that returns a resource object ready to be unmarshalled from YAML.
 func RegisterResource(kind string, fn func() Res) {
 	f := fn()
 	if kind == "" {
@@ -119,6 +121,20 @@ type Init struct {
 
 	// Other functionality:
 
+	// Graph is a function that returns the current graph. The returned
+	// value won't be valid after a graphsync so make sure to call this when
+	// you are about to use it, and discard it right after.
+	// FIXME: it might be better to offer a safer, more limited, GraphQuery?
+	//Graph func() *pgraph.Graph // TODO: not implemented, use FilteredGraph
+
+	// FilteredGraph is a function that returns a filtered variant of the
+	// current graph. Only resource that have allowed themselves to be added
+	// into this graph will appear. If they did not consent, then those
+	// vertices and any associated edges, will not be present.
+	FilteredGraph func() (*pgraph.Graph, error)
+
+	// TODO: GraphQuery offers an interface to query the resource graph.
+
 	// World provides a connection to the outside world. This is most often
 	// used for communicating with the distributed database.
 	World World
@@ -226,8 +242,8 @@ func Validate(res Res) error {
 // the Interrupt method to shutdown the resource quickly. Running this method
 // may leave the resource in a partial state, however this may be desired if you
 // want a faster exit or if you'd prefer a partial state over letting the
-// resource complete in a situation where you made an error and you wish to
-// exit quickly to avoid data loss. It is usually triggered after multiple ^C
+// resource complete in a situation where you made an error and you wish to exit
+// quickly to avoid data loss. It is usually triggered after multiple ^C
 // signals.
 type InterruptableRes interface {
 	Res

engine/resources/augeas.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -27,8 +27,8 @@ import (
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/engine/traits"
 	"github.com/purpleidea/mgmt/recwatch"
+	"github.com/purpleidea/mgmt/util/errwrap"
 
-	errwrap "github.com/pkg/errors"
 	// FIXME: we vendor go/augeas because master requires augeas 1.6.0
 	// and libaugeas-dev-1.6.0 is not yet available in a PPA.
 	"honnef.co/go/augeas"
@@ -124,8 +124,8 @@ func (obj *AugeasRes) Close() error {
 	return nil
 }
 
-// Watch is the primary listener for this resource and it outputs events.
-// Taken from the File resource.
+// Watch is the primary listener for this resource and it outputs events. This
+// was taken from the File resource.
 // FIXME: DRY - This is taken from the file resource
 func (obj *AugeasRes) Watch() error {
 	var err error
@@ -301,8 +301,8 @@ func (obj *AugeasRes) UIDs() []engine.ResUID {
 	return []engine.ResUID{x}
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
-// It is primarily useful for setting the defaults.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
+// primarily useful for setting the defaults.
 func (obj *AugeasRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes AugeasRes // indirection to avoid infinite recursion
 

engine/resources/aws_ec2.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -34,6 +34,7 @@ import (
 
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/engine/traits"
+	"github.com/purpleidea/mgmt/util/errwrap"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
@@ -42,8 +43,6 @@ import (
 	cwe "github.com/aws/aws-sdk-go/service/cloudwatchevents"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/sns"
-	multierr "github.com/hashicorp/go-multierror"
-	errwrap "github.com/pkg/errors"
 )
 
 func init() {
@@ -122,8 +121,8 @@ const (
 )
 
 // AwsRegions is a list of all AWS regions generated using ec2.DescribeRegions.
-// cn-north-1 and us-gov-west-1 are not returned, probably due to security.
-// List available at http://docs.aws.amazon.com/general/latest/gr/rande.html
+// cn-north-1 and us-gov-west-1 are not returned, probably due to security. List
+// available at http://docs.aws.amazon.com/general/latest/gr/rande.html
 var AwsRegions = []string{
 	"ap-northeast-1",
 	"ap-northeast-2",
@@ -188,7 +187,8 @@ type AwsEc2Res struct {
 	InstanceID  string
 }
 
-// chanStruct defines the type for a channel used to pass events and errors to watch.
+// chanStruct defines the type for a channel used to pass events and errors to
+// watch.
 type chanStruct struct {
 	event awsEc2Event
 	state string
@@ -234,7 +234,8 @@ type ruleDetail struct {
 	State []string `json:"state"`
 }
 
-// postData is the format of the messages received and decoded by snsPostHandler().
+// postData is the format of the messages received and decoded by
+// snsPostHandler().
 type postData struct {
 	Type             string `json:"Type"`
 	MessageID        string `json:"MessageId"`
@@ -248,7 +249,8 @@ type postData struct {
 	SigningCertURL   string `json:"SigningCertURL"`
 }
 
-// postMsg is used to unmarshal the postData message if it's an event notification.
+// postMsg is used to unmarshal the postData message if it's an event
+// notification.
 type postMsg struct {
 	InstanceID string `json:"instance-id"`
 	State      string `json:"state"`
@@ -393,17 +395,14 @@ func (obj *AwsEc2Res) Close() error {
 	// clean up sns objects created by Init/snsWatch
 	if obj.snsClient != nil {
 		// delete the topic and associated subscriptions
-		if err := obj.snsDeleteTopic(obj.snsTopicArn); err != nil {
-			errList = multierr.Append(errList, err)
-		}
+		e1 := obj.snsDeleteTopic(obj.snsTopicArn)
+		errList = errwrap.Append(errList, e1)
 		// remove the target
-		if err := obj.cweRemoveTarget(CweTargetID, CweRuleName); err != nil {
-			errList = multierr.Append(errList, err)
-		}
+		e2 := obj.cweRemoveTarget(CweTargetID, CweRuleName)
+		errList = errwrap.Append(errList, e2)
 		// delete the cloudwatch rule
-		if err := obj.cweDeleteRule(CweRuleName); err != nil {
-			errList = multierr.Append(errList, err)
-		}
+		e3 := obj.cweDeleteRule(CweRuleName)
+		errList = errwrap.Append(errList, e3)
 	}
 
 	return errList
@@ -417,7 +416,8 @@ func (obj *AwsEc2Res) Watch() error {
 	return obj.longpollWatch()
 }
 
-// longpollWatch uses the ec2 api's built in methods to watch ec2 resource state.
+// longpollWatch uses the ec2 api's built in methods to watch ec2 resource
+// state.
 func (obj *AwsEc2Res) longpollWatch() error {
 	send := false
 
@@ -514,10 +514,10 @@ func (obj *AwsEc2Res) longpollWatch() error {
 }
 
 // snsWatch uses amazon's SNS and CloudWatchEvents APIs to get instance state-
-// change notifications pushed to the http endpoint (snsServer) set up below.
-// In Init() a CloudWatch rule is created along with a corresponding SNS topic
-// that it can publish to. snsWatch creates an http server which listens for
-// messages published to the topic and processes them accordingly.
+// change notifications pushed to the http endpoint (snsServer) set up below. In
+// Init() a CloudWatch rule is created along with a corresponding SNS topic that
+// it can publish to. snsWatch creates an http server which listens for messages
+// published to the topic and processes them accordingly.
 func (obj *AwsEc2Res) snsWatch() error {
 	send := false
 	defer obj.wg.Wait()
@@ -608,7 +608,7 @@ func (obj *AwsEc2Res) snsWatch() error {
 }
 
 // CheckApply method for AwsEc2 resource.
-func (obj *AwsEc2Res) CheckApply(apply bool) (checkOK bool, err error) {
+func (obj *AwsEc2Res) CheckApply(apply bool) (bool, error) {
 	obj.init.Logf("CheckApply(%t)", apply)
 
 	// find the instance we need to check
@@ -755,45 +755,37 @@ func (obj *AwsEc2Res) CheckApply(apply bool) (checkOK bool, err error) {
 
 // Cmp compares two resources and returns an error if they are not equivalent.
 func (obj *AwsEc2Res) Cmp(r engine.Res) error {
-	if !obj.Compare(r) {
-		return fmt.Errorf("did not compare")
-	}
-	return nil
-}
-
-// Compare two resources and return if they are equivalent.
-func (obj *AwsEc2Res) Compare(r engine.Res) bool {
 	// we can only compare AwsEc2Res to others of the same resource kind
 	res, ok := r.(*AwsEc2Res)
 	if !ok {
-		return false
+		return fmt.Errorf("not a %s", obj.Kind())
 	}
 
 	if obj.State != res.State {
-		return false
+		return fmt.Errorf("the State differs")
 	}
 	if obj.Region != res.Region {
-		return false
+		return fmt.Errorf("the Region differs")
 	}
 	if obj.Type != res.Type {
-		return false
+		return fmt.Errorf("the Type differs")
 	}
 	if obj.ImageID != res.ImageID {
-		return false
+		return fmt.Errorf("the ImageID differs")
 	}
 	if obj.WatchEndpoint != res.WatchEndpoint {
-		return false
+		return fmt.Errorf("the WatchEndpoint differs")
 	}
 	if obj.WatchListenAddr != res.WatchListenAddr {
-		return false
+		return fmt.Errorf("the WatchListenAddr differs")
 	}
 	if obj.ErrorOnMalformedPost != res.ErrorOnMalformedPost {
-		return false
+		return fmt.Errorf("the ErrorOnMalformedPost differs")
 	}
 	if obj.UserData != res.UserData {
-		return false
+		return fmt.Errorf("the UserData differs")
 	}
-	return true
+	return nil
 }
 
 func (obj *AwsEc2Res) prependName() string {
@@ -807,8 +799,8 @@ type AwsEc2UID struct {
 	name string
 }
 
-// UIDs includes all params to make a unique identification of this object.
-// Most resources only return one, although some resources can return multiple.
+// UIDs includes all params to make a unique identification of this object. Most
+// resources only return one, although some resources can return multiple.
 func (obj *AwsEc2Res) UIDs() []engine.ResUID {
 	x := &AwsEc2UID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
@@ -817,8 +809,8 @@ func (obj *AwsEc2Res) UIDs() []engine.ResUID {
 	return []engine.ResUID{x}
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
-// It is primarily useful for setting the defaults.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
+// primarily useful for setting the defaults.
 func (obj *AwsEc2Res) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes AwsEc2Res // indirection to avoid infinite recursion
 
@@ -954,8 +946,8 @@ func (obj *AwsEc2Res) snsVerifySignature(post postData) error {
 	return nil
 }
 
-// snsGetCert downloads and parses the signing certificate from the provided
-// URL for message verification.
+// snsGetCert downloads and parses the signing certificate from the provided URL
+// for message verification.
 func (obj *AwsEc2Res) snsGetCert(url string) (*x509.Certificate, error) {
 	// only download valid certificates from amazon
 	matchURL, err := regexp.MatchString(SnsCertURLRegex, url)
@@ -1029,7 +1021,7 @@ func (obj *AwsEc2Res) snsMakeTopic() (string, error) {
 	}
 	obj.init.Logf("Created SNS Topic")
 	if topic.TopicArn == nil {
-		return "", fmt.Errorf("TopicArn is nil")
+		return "", fmt.Errorf("the TopicArn is nil")
 	}
 	return *topic.TopicArn, nil
 }
@@ -1047,8 +1039,8 @@ func (obj *AwsEc2Res) snsDeleteTopic(topicArn string) error {
 	return nil
 }
 
-// snsSubscribe subscribes the endpoint to the sns topic.
-// Returning SubscriptionArn here is useless as it is still pending confirmation.
+// snsSubscribe subscribes the endpoint to the sns topic. Returning
+// SubscriptionArn here is useless as it is still pending confirmation.
 func (obj *AwsEc2Res) snsSubscribe(endpoint string, topicArn string) error {
 	// subscribe to the topic
 	subInput := &sns.SubscribeInput{
@@ -1064,8 +1056,8 @@ func (obj *AwsEc2Res) snsSubscribe(endpoint string, topicArn string) error {
 	return nil
 }
 
-// snsConfirmSubscription confirms the sns subscription.
-// Returning SubscriptionArn here is useless as it is still pending confirmation.
+// snsConfirmSubscription confirms the sns subscription. Returning
+// SubscriptionArn here is useless as it is still pending confirmation.
 func (obj *AwsEc2Res) snsConfirmSubscription(topicArn string, token string) error {
 	// confirm the subscription
 	csInput := &sns.ConfirmSubscriptionInput{
@@ -1117,7 +1109,8 @@ func (obj *AwsEc2Res) snsProcessEvent(message, instanceName string) (awsEc2Event
 	return awsEc2EventNone, nil
 }
 
-// snsAuthorize adds the necessary permission for cloudwatch to publish to the SNS topic.
+// snsAuthorize adds the necessary permission for cloudwatch to publish to the
+// SNS topic.
 func (obj *AwsEc2Res) snsAuthorizeCloudWatch(topicArn string) error {
 	// get the topic attributes, including the security policy
 	gaInput := &sns.GetTopicAttributesInput{

engine/resources/config_etcd.go

@@ -0,0 +1,250 @@
+// Mgmt
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/purpleidea/mgmt/engine"
+	"github.com/purpleidea/mgmt/engine/traits"
+	"github.com/purpleidea/mgmt/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
+)
+
+func init() {
+	engine.RegisterResource("config:etcd", func() engine.Res { return &ConfigEtcdRes{} })
+}
+
+const (
+	sizeCheckApplyTimeout = 5 * time.Second
+)
+
+// ConfigEtcdRes is a resource that sets mgmt's etcd configuration.
+type ConfigEtcdRes struct {
+	traits.Base // add the base methods without re-implementation
+
+	init *engine.Init
+
+	// IdealClusterSize is the requested minimum size of the cluster. If you
+	// set this to zero, it will cause a cluster wide shutdown if
+	// AllowSizeShutdown is true. If it's not true, then it will cause a
+	// validation error.
+	IdealClusterSize uint16 `lang:"idealclustersize"`
+	// AllowSizeShutdown is a required safety flag that you must set to true
+	// if you want to allow causing a cluster shutdown by setting
+	// IdealClusterSize to zero.
+	AllowSizeShutdown bool `lang:"allow_size_shutdown"`
+
+	// sizeFlag determines whether sizeCheckApply already ran or not.
+	sizeFlag bool
+
+	interruptChan chan struct{}
+	wg            *sync.WaitGroup
+}
+
+// Default returns some sensible defaults for this resource.
+func (obj *ConfigEtcdRes) Default() engine.Res {
+	return &ConfigEtcdRes{}
+}
+
+// Validate if the params passed in are valid data.
+func (obj *ConfigEtcdRes) Validate() error {
+	if obj.IdealClusterSize < 0 {
+		return fmt.Errorf("the IdealClusterSize param must be positive")
+	}
+
+	if obj.IdealClusterSize == 0 && !obj.AllowSizeShutdown {
+		return fmt.Errorf("the IdealClusterSize can't be zero if AllowSizeShutdown is false")
+	}
+
+	return nil
+}
+
+// Init runs some startup code for this resource.
+func (obj *ConfigEtcdRes) Init(init *engine.Init) error {
+	obj.init = init // save for later
+
+	obj.interruptChan = make(chan struct{})
+	obj.wg = &sync.WaitGroup{}
+
+	return nil
+}
+
+// Close is run by the engine to clean up after the resource is done.
+func (obj *ConfigEtcdRes) Close() error {
+	obj.wg.Wait() // bonus
+	return nil
+}
+
+// Watch is the primary listener for this resource and it outputs events.
+func (obj *ConfigEtcdRes) Watch() error {
+	obj.wg.Add(1)
+	defer obj.wg.Done()
+	// FIXME: add timeout to context
+	// The obj.init.Done channel is closed by the engine to signal shutdown.
+	ctx, cancel := util.ContextWithCloser(context.Background(), obj.init.Done)
+	defer cancel()
+	ch, err := obj.init.World.IdealClusterSizeWatch(util.CtxWithWg(ctx, obj.wg))
+	if err != nil {
+		return errwrap.Wrapf(err, "could not watch ideal cluster size")
+	}
+
+	obj.init.Running() // when started, notify engine that we're running
+
+Loop:
+	for {
+		select {
+		case event, ok := <-ch:
+			if !ok {
+				break Loop
+			}
+			if obj.init.Debug {
+				obj.init.Logf("event: %+v", event)
+			}
+			// pass through and send an event
+
+		case <-obj.init.Done: // closed by the engine to signal shutdown
+		}
+
+		obj.init.Event() // notify engine of an event (this can block)
+	}
+
+	return nil
+}
+
+// sizeCheckApply sets the IdealClusterSize parameter. If it sees a value change
+// to zero, then it *won't* try and change it away from zero, because it assumes
+// that someone has requested a shutdown. If the value is seen on first startup,
+// then it will change it, because it might be a zero from the previous cluster.
+func (obj *ConfigEtcdRes) sizeCheckApply(apply bool) (bool, error) {
+	wg := &sync.WaitGroup{}
+	defer wg.Wait() // this must be above the defer cancel() call
+	ctx, cancel := context.WithTimeout(context.Background(), sizeCheckApplyTimeout)
+	defer cancel()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		select {
+		case <-obj.interruptChan:
+			cancel()
+		case <-ctx.Done():
+			// let this exit
+		}
+	}()
+
+	val, err := obj.init.World.IdealClusterSizeGet(ctx)
+	if err != nil {
+		return false, errwrap.Wrapf(err, "could not get ideal cluster size")
+	}
+
+	// if we got a value of zero, and we've already run before, then it's ok
+	if obj.IdealClusterSize != 0 && val == 0 && obj.sizeFlag {
+		obj.init.Logf("impending cluster shutdown, not setting ideal cluster size")
+		return true, nil // impending shutdown, don't try and cancel it.
+	}
+	obj.sizeFlag = true
+
+	// must be done after setting the above flag
+	if obj.IdealClusterSize == val { // state is correct
+		return true, nil
+	}
+
+	if !apply {
+		return false, nil
+	}
+
+	// set!
+	// This is run as a transaction so we detect if we needed to change it.
+	changed, err := obj.init.World.IdealClusterSizeSet(ctx, obj.IdealClusterSize)
+	if err != nil {
+		return false, errwrap.Wrapf(err, "could not set ideal cluster size")
+	}
+	if !changed {
+		return true, nil // we lost a race, which means no change needed
+	}
+	obj.init.Logf("set dynamic cluster size to: %d", obj.IdealClusterSize)
+
+	return false, nil
+}
+
+// CheckApply method for Noop resource. Does nothing, returns happy!
+func (obj *ConfigEtcdRes) CheckApply(apply bool) (bool, error) {
+	checkOK := true
+
+	if c, err := obj.sizeCheckApply(apply); err != nil {
+		return false, err
+	} else if !c {
+		checkOK = false
+	}
+
+	// TODO: add more config settings management here...
+	//if c, err := obj.TODOCheckApply(apply); err != nil {
+	//	return false, err
+	//} else if !c {
+	//	checkOK = false
+	//}
+
+	return checkOK, nil // w00t
+}
+
+// Cmp compares two resources and returns an error if they are not equivalent.
+func (obj *ConfigEtcdRes) Cmp(r engine.Res) error {
+	// we can only compare ConfigEtcdRes to others of the same resource kind
+	res, ok := r.(*ConfigEtcdRes)
+	if !ok {
+		return fmt.Errorf("not a %s", obj.Kind())
+	}
+
+	if obj.IdealClusterSize != res.IdealClusterSize {
+		return fmt.Errorf("the IdealClusterSize param differs")
+	}
+	if obj.AllowSizeShutdown != res.AllowSizeShutdown {
+		return fmt.Errorf("the AllowSizeShutdown param differs")
+	}
+
+	return nil
+}
+
+// Interrupt is called to ask the execution of this resource to end early.
+func (obj *ConfigEtcdRes) Interrupt() error {
+	close(obj.interruptChan)
+	return nil
+}
+
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
+// primarily useful for setting the defaults.
+func (obj *ConfigEtcdRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type rawRes ConfigEtcdRes // indirection to avoid infinite recursion
+
+	def := obj.Default()            // get the default
+	res, ok := def.(*ConfigEtcdRes) // put in the right format
+	if !ok {
+		return fmt.Errorf("could not convert to ConfigEtcdRes")
+	}
+	raw := rawRes(*res) // convert; the defaults go here
+
+	if err := unmarshal(&raw); err != nil {
+		return err
+	}
+
+	*obj = ConfigEtcdRes(raw) // restore from indirection with type conversion!
+	return nil
+}

engine/resources/consul_kv.go

@@ -0,0 +1,283 @@
+// Mgmt
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"sync"
+
+	"github.com/purpleidea/mgmt/engine"
+	"github.com/purpleidea/mgmt/engine/traits"
+	"github.com/purpleidea/mgmt/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
+
+	"github.com/hashicorp/consul/api"
+)
+
+func init() {
+	engine.RegisterResource("consul:kv", func() engine.Res { return &ConsulKVRes{} })
+}
+
+// ConsulKVRes is a resource that writes a value into a Consul datastore. The
+// name of the resource can either be the key name, or the concatenation of the
+// server address and the key name: http://127.0.0.1:8500/my-key. If the param
+// keys are specified, then those are used. If the Name cannot be properly
+// parsed by url.Parse, then it will be considered as the Key's value. If the
+// Key is specified explicitly, then we won't use anything from the Name.
+type ConsulKVRes struct {
+	traits.Base
+	init *engine.Init
+
+	// Key is the name of the key. Defaults to the name of the resource.
+	Key string `lang:"key" yaml:"key"`
+
+	// Value is the value for the key.
+	Value string `lang:"value" yaml:"value"`
+
+	// Scheme is the URI scheme for the Consul server. Default: http.
+	Scheme string `lang:"scheme" yaml:"scheme"`
+
+	// Address is the address of the Consul server. Default: 127.0.0.1:8500.
+	Address string `lang:"address" yaml:"address"`
+
+	// Token is used to provide an ACL token to use for this resource.
+	Token string `lang:"token" yaml:"token"`
+
+	client *api.Client
+	config *api.Config // needed to close the idle connections
+	once   bool        // safety token
+	key    string      // cache the key name to avoid re-running the parser
+}
+
+// Default returns some sensible defaults for this resource.
+func (obj *ConsulKVRes) Default() engine.Res {
+	return &ConsulKVRes{}
+}
+
+// Validate if the params passed in are valid data.
+func (obj *ConsulKVRes) Validate() error {
+	s, _, k := obj.inputParser()
+	if k == "" {
+		return fmt.Errorf("the Key is empty")
+	}
+	if s != "" && s != "http" && s != "https" {
+		return fmt.Errorf("unknown Scheme")
+	}
+	return nil
+}
+
+// Init runs some startup code for this resource.
+func (obj *ConsulKVRes) Init(init *engine.Init) error {
+	obj.init = init // save for later
+
+	s, a, k := obj.inputParser()
+
+	obj.config = api.DefaultConfig()
+	if s != "" {
+		obj.config.Scheme = s
+	}
+	if a != "" {
+		obj.config.Address = obj.Address
+	}
+	obj.key = k // store the key
+	obj.init.Logf("using consul key: %s", obj.key)
+
+	if obj.Token != "" {
+		obj.config.Token = obj.Token
+	}
+
+	var err error
+	obj.client, err = api.NewClient(obj.config)
+	return errwrap.Wrapf(err, "could not create Consul client")
+}
+
+// Close is run by the engine to clean up after the resource is done.
+func (obj *ConsulKVRes) Close() error {
+	if obj.config != nil && obj.config.Transport != nil {
+		obj.config.Transport.CloseIdleConnections()
+	}
+	return nil
+}
+
+// Watch is the listener and main loop for this resource and it outputs events.
+func (obj *ConsulKVRes) Watch() error {
+	wg := &sync.WaitGroup{}
+	defer wg.Wait()
+
+	ch := make(chan error)
+	exit := make(chan struct{})
+
+	kv := obj.client.KV()
+
+	wg.Add(1)
+	go func() {
+		defer close(ch)
+		defer wg.Done()
+
+		opts := &api.QueryOptions{RequireConsistent: true}
+		ctx, cancel := util.ContextWithCloser(context.Background(), exit)
+		defer cancel()
+		opts = opts.WithContext(ctx)
+
+		for {
+			_, meta, err := kv.Get(obj.key, opts)
+			select {
+			case ch <- err: // send
+				if err != nil {
+					return
+				}
+
+				// WaitIndex = 0, which means that it is the
+				// first time we run the query, as we are about
+				// to change the WaitIndex to make a blocking
+				// query, we can consider the watch started.
+				opts.WaitIndex = meta.LastIndex
+				if opts.WaitIndex != 0 {
+					continue
+				}
+
+				if !obj.once {
+					obj.init.Running()
+					obj.once = true
+					continue
+				}
+
+				// Unexpected situation, bug in consul API...
+				select {
+				case ch <- fmt.Errorf("unexpected behaviour in Consul API"):
+				case <-obj.init.Done: // signal for shutdown request
+				}
+
+			case <-obj.init.Done: // signal for shutdown request
+			}
+			return
+		}
+	}()
+
+	defer close(exit)
+	for {
+		select {
+		case err, ok := <-ch:
+			if !ok { // channel shutdown
+				return nil
+			}
+			if err != nil {
+				return errwrap.Wrapf(err, "unknown %s watcher error", obj)
+			}
+			if obj.init.Debug {
+				obj.init.Logf("event!")
+			}
+			obj.init.Event()
+
+		case <-obj.init.Done: // signal for shutdown request
+			return nil
+		}
+	}
+}
+
+// CheckApply is run to check the state and, if apply is true, to apply the
+// necessary changes to reach the desired state. This is run before Watch and
+// again if Watch finds a change occurring to the state.
+func (obj *ConsulKVRes) CheckApply(apply bool) (bool, error) {
+	if obj.init.Debug {
+		obj.init.Logf("consul key: %s", obj.key)
+	}
+	kv := obj.client.KV()
+	pair, _, err := kv.Get(obj.key, nil)
+	if err != nil {
+		return false, err
+	}
+
+	if pair != nil && string(pair.Value) == obj.Value {
+		return true, nil
+	}
+
+	if !apply {
+		return false, nil
+	}
+
+	p := &api.KVPair{Key: obj.key, Value: []byte(obj.Value)}
+	_, err = kv.Put(p, nil)
+	return false, err
+}
+
+// Cmp compares two resources and return if they are equivalent.
+func (obj *ConsulKVRes) Cmp(r engine.Res) error {
+	res, ok := r.(*ConsulKVRes)
+	if !ok {
+		return fmt.Errorf("not a %s", obj.Kind())
+	}
+
+	if obj.Key != res.Key {
+		return fmt.Errorf("the Key param differs")
+	}
+	if obj.Value != res.Value {
+		return fmt.Errorf("the Value param differs")
+	}
+	if obj.Scheme != res.Scheme {
+		return fmt.Errorf("the Scheme param differs")
+	}
+	if obj.Address != res.Address {
+		return fmt.Errorf("the Address param differs")
+	}
+	if obj.Token != res.Token {
+		return fmt.Errorf("the Token param differs")
+	}
+
+	return nil
+}
+
+// inputParser parses the Name() of a resource and extracts the scheme, address,
+// and key name of a consul key. We don't have an error, because if we have one,
+// then it means the input must be a raw key. Output of this function is scheme,
+// address (includes hostname and port), and key. This also takes our parameters
+// in to account, and applies the correct overrides if they are specified there.
+func (obj *ConsulKVRes) inputParser() (string, string, string) {
+	// If the key is specified explicitly, then we're not going to parse the
+	// resource name for a pattern, and we use our given params as they are.
+	if obj.Key != "" {
+		return obj.Scheme, obj.Address, obj.Key
+	}
+
+	// Now we parse...
+	u, err := url.Parse(obj.Name())
+	if err != nil {
+		// If this didn't work, then we know it's explicitly a raw key.
+		return obj.Scheme, obj.Address, obj.Name()
+	}
+
+	// Otherwise, we use the parse result, and we overwrite any of the
+	// fields if we have an explicit param that was specified.
+	k := u.Path
+	s := u.Scheme
+	a := u.Host
+
+	//if obj.Key != "" { // this is now guaranteed to never happen
+	//	k = obj.Key
+	//}
+	if obj.Scheme != "" {
+		s = obj.Scheme
+	}
+	if obj.Address != "" {
+		a = obj.Address
+	}
+
+	return s, a, k
+}

engine/resources/consul_kv_test.go

@@ -0,0 +1,71 @@
+// Mgmt
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/purpleidea/mgmt/engine"
+)
+
+func createConsulRes(name string) *ConsulKVRes {
+	r, err := engine.NewNamedResource("consul:kv", name)
+	if err != nil {
+		panic(fmt.Sprintf("could not create resource: %+v", err))
+	}
+
+	res := r.(*ConsulKVRes) // if this panics, the test will panic
+	return res
+}
+
+func TestParseConsulName(t *testing.T) {
+	n1 := "test"
+	r1 := createConsulRes(n1)
+	if s, a, k := r1.inputParser(); s != "" || a != "" || k != "test" {
+		t.Errorf("unexpected output while parsing `%s`: %s, %s, %s", n1, s, a, k)
+	}
+
+	n2 := "http://127.0.0.1:8500/test"
+	r2 := createConsulRes(n2)
+	if s, a, k := r2.inputParser(); s != "http" || a != "127.0.0.1:8500" || k != "/test" {
+		t.Errorf("unexpected output while parsing `%s`: %s, %s, %s", n2, s, a, k)
+	}
+
+	n3 := "http://127.0.0.1:8500/test"
+	r3 := createConsulRes(n3)
+	r3.Scheme = "https"
+	r3.Address = "example.com"
+	if s, a, k := r3.inputParser(); s != "https" || a != "example.com" || k != "/test" {
+		t.Errorf("unexpected output while parsing `%s`: %s, %s, %s", n3, s, a, k)
+	}
+
+	n4 := "http:://127.0.0.1..5:8500/test" // wtf, url.Parse is on drugs...
+	r4 := createConsulRes(n4)
+	//if s, a, k := r4.inputParser(); s != "" || a != "" || k != n4 { // what i really expect
+	if s, a, k := r4.inputParser(); s != "http" || a != "" || k != "" { // what i get
+		t.Errorf("unexpected output while parsing `%s`: %s, %s, %s", n4, s, a, k)
+	}
+
+	n5 := "http://127.0.0.1:8500/test" // whatever, it's ignored
+	r5 := createConsulRes(n3)
+	r5.Key = "some key"
+	if s, a, k := r5.inputParser(); s != "" || a != "" || k != "some key" {
+		t.Errorf("unexpected output while parsing `%s`: %s, %s, %s", n5, s, a, k)
+	}
+}

engine/resources/cron.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -31,12 +31,12 @@ import (
 	engineUtil "github.com/purpleidea/mgmt/engine/util"
 	"github.com/purpleidea/mgmt/recwatch"
 	"github.com/purpleidea/mgmt/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
 
 	sdbus "github.com/coreos/go-systemd/dbus"
 	"github.com/coreos/go-systemd/unit"
 	systemdUtil "github.com/coreos/go-systemd/util"
 	"github.com/godbus/dbus"
-	errwrap "github.com/pkg/errors"
 )
 
 const (
@@ -139,9 +139,9 @@ func (obj *CronRes) Default() engine.Res {
 	}
 }
 
-// makeComposite creates a pointer to a FileRes. The pointer is used to
-// validate and initialize the nested file resource and to apply the file state
-// in CheckApply.
+// makeComposite creates a pointer to a FileRes. The pointer is used to validate
+// and initialize the nested file resource and to apply the file state in
+// CheckApply.
 func (obj *CronRes) makeComposite() (*FileRes, error) {
 	p, err := obj.UnitFilePath()
 	if err != nil {
@@ -310,28 +310,29 @@ func (obj *CronRes) Watch() error {
 // CheckApply is run to check the state and, if apply is true, to apply the
 // necessary changes to reach the desired state. This is run before Watch and
 // again if Watch finds a change occurring to the state.
-func (obj *CronRes) CheckApply(apply bool) (checkOK bool, err error) {
-	ok := true
+func (obj *CronRes) CheckApply(apply bool) (bool, error) {
+	checkOK := true
 	// use the embedded file resource to apply the correct state
 	if c, err := obj.file.CheckApply(apply); err != nil {
 		return false, errwrap.Wrapf(err, "nested file failed")
 	} else if !c {
-		ok = false
+		checkOK = false
 	}
 	// check timer state and apply the defined state if needed
 	if c, err := obj.unitCheckApply(apply); err != nil {
 		return false, errwrap.Wrapf(err, "unitCheckApply error")
 	} else if !c {
-		ok = false
+		checkOK = false
 	}
-	return ok, nil
+	return checkOK, nil
 }
 
 // unitCheckApply checks the state of the systemd-timer unit and, if apply is
 // true, applies the defined state.
-func (obj *CronRes) unitCheckApply(apply bool) (checkOK bool, err error) {
+func (obj *CronRes) unitCheckApply(apply bool) (bool, error) {
 	var conn *sdbus.Conn
 	var godbusConn *dbus.Conn
+	var err error
 
 	// this resource depends on systemd to ensure that it's running
 	if !systemdUtil.IsRunningSystemd() {
@@ -465,8 +466,8 @@ func (obj *CronRes) AutoEdges() (engine.AutoEdge, error) {
 	return nil, nil
 }
 
-// UIDs includes all params to make a unique identification of this object.
-// Most resources only return one although some resources can return multiple.
+// UIDs includes all params to make a unique identification of this object. Most
+// resources only return one although some resources can return multiple.
 func (obj *CronRes) UIDs() []engine.ResUID {
 	unit := fmt.Sprintf("%s.service", obj.Name())
 	if obj.Unit != "" {
@@ -485,8 +486,8 @@ func (obj *CronRes) UIDs() []engine.ResUID {
 	return uids
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
-// It is primarily useful for setting the defaults.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
+// primarily useful for setting the defaults.
 func (obj *CronRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes CronRes // indirection to avoid infinite recursion
 

engine/resources/docker_container.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -30,13 +30,13 @@ import (
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/engine/traits"
 	"github.com/purpleidea/mgmt/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
 	"github.com/docker/go-connections/nat"
-	errwrap "github.com/pkg/errors"
 )
 
 const (
@@ -50,8 +50,8 @@ const (
 	// initCtxTimeout is the length of time, in seconds, before requests are
 	// cancelled in Init.
 	initCtxTimeout = 20
-	// checkApplyCtxTimeout is the length of time, in seconds, before requests
-	// are cancelled in CheckApply.
+	// checkApplyCtxTimeout is the length of time, in seconds, before
+	// requests are cancelled in CheckApply.
 	checkApplyCtxTimeout = 120
 )
 
@@ -74,11 +74,12 @@ type DockerContainerRes struct {
 	Env []string `yaml:"env"`
 	// Ports is a map of port bindings. E.g. {"tcp" => {80 => 8080},}.
 	Ports map[string]map[int64]int64 `yaml:"ports"`
-	// APIVersion allows you to override the host's default client API version.
+	// APIVersion allows you to override the host's default client API
+	// version.
 	APIVersion string `yaml:"apiversion"`
 
-	// Force, if true, will destroy and redeploy the container if the image is
-	// incorrect.
+	// Force, if true, this will destroy and redeploy the container if the
+	// image is incorrect.
 	Force bool `yaml:"force"`
 
 	client *client.Client // docker api client
@@ -88,7 +89,9 @@ type DockerContainerRes struct {
 
 // Default returns some sensible defaults for this resource.
 func (obj *DockerContainerRes) Default() engine.Res {
-	return &DockerContainerRes{}
+	return &DockerContainerRes{
+		State: "running",
+	}
 }
 
 // Validate if the params passed in are valid data.
@@ -98,6 +101,11 @@ func (obj *DockerContainerRes) Validate() error {
 		return fmt.Errorf("state must be running, stopped or removed")
 	}
 
+	// make sure an image is specified
+	if obj.Image == "" {
+		return fmt.Errorf("image must be specified")
+	}
+
 	// validate env
 	for _, env := range obj.Env {
 		if !strings.Contains(env, "=") || strings.Contains(env, " ") {
@@ -140,7 +148,7 @@ func (obj *DockerContainerRes) Init(init *engine.Init) error {
 	defer cancel()
 
 	// Initialize the docker client.
-	obj.client, err = client.NewClient(client.DefaultDockerHost, obj.APIVersion, nil, nil)
+	obj.client, err = client.NewClientWithOpts(client.WithVersion(obj.APIVersion))
 	if err != nil {
 		return errwrap.Wrapf(err, "error creating docker client")
 	}
@@ -201,7 +209,7 @@ func (obj *DockerContainerRes) Watch() error {
 }
 
 // CheckApply method for Docker resource.
-func (obj *DockerContainerRes) CheckApply(apply bool) (checkOK bool, err error) {
+func (obj *DockerContainerRes) CheckApply(apply bool) (bool, error) {
 	var id string
 	var destroy bool
 
@@ -302,7 +310,7 @@ func (obj *DockerContainerRes) CheckApply(apply bool) (checkOK bool, err error)
 			}
 		}
 
-		c, err := obj.client.ContainerCreate(ctx, containerConfig, hostConfig, nil, obj.Name())
+		c, err := obj.client.ContainerCreate(ctx, containerConfig, hostConfig, nil, nil, obj.Name())
 		if err != nil {
 			return false, errwrap.Wrapf(err, "error creating container")
 		}
@@ -367,52 +375,105 @@ func (obj *DockerContainerRes) Cmp(r engine.Res) error {
 	if !ok {
 		return fmt.Errorf("error casting r to *DockerContainerRes")
 	}
-	if obj.Name() != res.Name() {
-		return fmt.Errorf("names differ")
+
+	if obj.State != res.State {
+		return fmt.Errorf("the State differs")
+	}
+	if obj.Image != res.Image {
+		return fmt.Errorf("the Image differs")
 	}
 	if err := util.SortedStrSliceCompare(obj.Cmd, res.Cmd); err != nil {
-		return errwrap.Wrapf(err, "cmd differs")
+		return errwrap.Wrapf(err, "the Cmd field differs")
 	}
 	if err := util.SortedStrSliceCompare(obj.Env, res.Env); err != nil {
-		return errwrap.Wrapf(err, "env differs")
+		return errwrap.Wrapf(err, "tne Env field differs")
 	}
 	if len(obj.Ports) != len(res.Ports) {
-		return fmt.Errorf("ports length differs")
+		return fmt.Errorf("the Ports length differs")
 	}
 	for k, v := range obj.Ports {
 		for p, q := range v {
 			if w, ok := res.Ports[k][p]; !ok || q != w {
-				return fmt.Errorf("ports differ")
+				return fmt.Errorf("the Ports field differs")
 			}
 		}
 	}
 	if obj.APIVersion != res.APIVersion {
-		return fmt.Errorf("apiversions differ")
+		return fmt.Errorf("the APIVersion differs")
 	}
 	if obj.Force != res.Force {
-		return fmt.Errorf("forces differ")
+		return fmt.Errorf("the Force field differs")
 	}
 	return nil
 }
 
-// DockerUID is the UID struct for DockerContainerRes.
-type DockerUID struct {
+// DockerContainerUID is the UID struct for DockerContainerRes.
+type DockerContainerUID struct {
 	engine.BaseUID
+
 	name string
 }
 
-// UIDs includes all params to make a unique identification of this object.
-// Most resources only return one, although some resources can return multiple.
+// DockerContainerResAutoEdges holds the state of the auto edge generator.
+type DockerContainerResAutoEdges struct {
+	UIDs    []engine.ResUID
+	pointer int
+}
+
+// AutoEdges returns edges to any docker:image resource that matches the image
+// specified in the docker:container resource definition.
+func (obj *DockerContainerRes) AutoEdges() (engine.AutoEdge, error) {
+	var result []engine.ResUID
+	var reversed bool
+	if obj.State != "removed" {
+		reversed = true
+	}
+	result = append(result, &DockerImageUID{
+		BaseUID: engine.BaseUID{
+			Reversed: &reversed,
+		},
+		image: dockerImageNameTag(obj.Image),
+	})
+	return &DockerContainerResAutoEdges{
+		UIDs:    result,
+		pointer: 0,
+	}, nil
+}
+
+// Next returnes the next automatic edge.
+func (obj *DockerContainerResAutoEdges) Next() []engine.ResUID {
+	if len(obj.UIDs) == 0 {
+		return nil
+	}
+	value := obj.UIDs[obj.pointer]
+	obj.pointer++
+	return []engine.ResUID{value}
+}
+
+// Test gets results of the earlier Next() call, & returns if we should
+// continue.
+func (obj *DockerContainerResAutoEdges) Test(input []bool) bool {
+	if len(obj.UIDs) <= obj.pointer {
+		return false
+	}
+	if len(input) != 1 { // in case we get given bad data
+		panic(fmt.Sprintf("Expecting a single value!"))
+	}
+	return true // keep going
+}
+
+// UIDs includes all params to make a unique identification of this object. Most
+// resources only return one, although some resources can return multiple.
 func (obj *DockerContainerRes) UIDs() []engine.ResUID {
-	x := &DockerUID{
+	x := &DockerContainerUID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
 		name:    obj.Name(),
 	}
 	return []engine.ResUID{x}
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
-// It is primarily useful for setting the defaults.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
+// primarily useful for setting the defaults.
 func (obj *DockerContainerRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes DockerContainerRes // indirection to avoid infinite recursion
 

engine/resources/docker_container_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -165,6 +165,7 @@ func setup() error {
 		},
 		&container.HostConfig{},
 		nil,
+		nil,
 		"mgmt-test",
 	)
 	if err != nil {

engine/resources/docker_image.go

@@ -0,0 +1,295 @@
+// Mgmt
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// +build !nodocker
+
+package resources
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/purpleidea/mgmt/engine"
+	"github.com/purpleidea/mgmt/engine/traits"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/client"
+	errwrap "github.com/pkg/errors"
+)
+
+const (
+	// dockerImageInitCtxTimeout is the length of time, in seconds, before
+	// requests are cancelled in Init.
+	dockerImageInitCtxTimeout = 20
+	// dockerImageCheckApplyCtxTimeout is the length of time, in seconds,
+	// before requests are cancelled in CheckApply.
+	dockerImageCheckApplyCtxTimeout = 120
+)
+
+func init() {
+	engine.RegisterResource("docker:image", func() engine.Res { return &DockerImageRes{} })
+}
+
+// DockerImageRes is a docker image resource. The resource's name must be a
+// docker image in any supported format (url, image, or image:tag).
+type DockerImageRes struct {
+	traits.Base // add the base methods without re-implementation
+	traits.Edgeable
+
+	// State of the image must be exists or absent.
+	State string `yaml:"state"`
+	// APIVersion allows you to override the host's default client API
+	// version.
+	APIVersion string `yaml:"apiversion"`
+
+	image  string         // full image:tag format
+	client *client.Client // docker api client
+
+	init *engine.Init
+}
+
+// Default returns some sensible defaults for this resource.
+func (obj *DockerImageRes) Default() engine.Res {
+	return &DockerImageRes{
+		// TODO: eventually if image supports other properties, this can
+		// be left out and we could have the state be "unmanaged".
+		State: "exists",
+	}
+}
+
+// Validate if the params passed in are valid data.
+func (obj *DockerImageRes) Validate() error {
+	// validate state
+	if obj.State != "exists" && obj.State != "absent" {
+		return fmt.Errorf("state must be exists or absent")
+	}
+
+	// validate APIVersion
+	if obj.APIVersion != "" {
+		verOK, err := regexp.MatchString(`^(v)[1-9]\.[0-9]\d*$`, obj.APIVersion)
+		if err != nil {
+			return errwrap.Wrapf(err, "error matching apiversion string")
+		}
+		if !verOK {
+			return fmt.Errorf("invalid apiversion: %s", obj.APIVersion)
+		}
+	}
+
+	return nil
+}
+
+// Init runs some startup code for this resource.
+func (obj *DockerImageRes) Init(init *engine.Init) error {
+	var err error
+	obj.init = init // save for later
+
+	// Save the full image name and tag.
+	obj.image = dockerImageNameTag(obj.Name())
+
+	ctx, cancel := context.WithTimeout(context.Background(), dockerImageInitCtxTimeout*time.Second)
+	defer cancel()
+
+	// Initialize the docker client.
+	obj.client, err = client.NewClientWithOpts(client.WithVersion(obj.APIVersion))
+	if err != nil {
+		return errwrap.Wrapf(err, "error creating docker client")
+	}
+
+	// Validate the image.
+	resp, err := obj.client.ImageSearch(ctx, obj.image, types.ImageSearchOptions{Limit: 1})
+	if err != nil {
+		return errwrap.Wrapf(err, "error searching for image")
+	}
+	if len(resp) == 0 {
+		return fmt.Errorf("image: %s not found", obj.image)
+	}
+	return nil
+}
+
+// Close is run by the engine to clean up after the resource is done.
+func (obj *DockerImageRes) Close() error {
+	return obj.client.Close() // close the docker client
+}
+
+// Watch is the primary listener for this resource and it outputs events.
+func (obj *DockerImageRes) Watch() error {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	eventChan, errChan := obj.client.Events(ctx, types.EventsOptions{})
+
+	// notify engine that we're running
+	obj.init.Running()
+
+	var send = false // send event?
+	for {
+		select {
+		case event, ok := <-eventChan:
+			if !ok { // channel shutdown
+				return nil
+			}
+			if obj.init.Debug {
+				obj.init.Logf("%+v", event)
+			}
+			send = true
+
+		case err, ok := <-errChan:
+			if !ok {
+				return nil
+			}
+			return err
+
+		case <-obj.init.Done: // closed by the engine to signal shutdown
+			return nil
+		}
+
+		// do all our event sending all together to avoid duplicate msgs
+		if send {
+			send = false
+			obj.init.Event() // notify engine of an event (this can block)
+		}
+	}
+}
+
+// CheckApply method for Docker resource.
+func (obj *DockerImageRes) CheckApply(apply bool) (checkOK bool, err error) {
+	ctx, cancel := context.WithTimeout(context.Background(), dockerImageCheckApplyCtxTimeout*time.Second)
+	defer cancel()
+
+	s, err := obj.client.ImageList(ctx, types.ImageListOptions{
+		Filters: filters.NewArgs(filters.Arg("reference", obj.image)),
+	})
+	if err != nil {
+		return false, errwrap.Wrapf(err, "error listing images")
+	}
+	if len(s) > 1 {
+		return false, fmt.Errorf("more than one image found")
+	}
+
+	if obj.State == "absent" && len(s) == 0 {
+		return true, nil
+	}
+	if obj.State == "exists" && len(s) == 1 {
+		return true, nil
+	}
+
+	if !apply {
+		return false, nil
+	}
+
+	if obj.State == "absent" {
+		// TODO: force? prune children?
+		if _, err := obj.client.ImageRemove(ctx, obj.image, types.ImageRemoveOptions{}); err != nil {
+			return false, errwrap.Wrapf(err, "error removing image")
+		}
+		return false, nil
+	}
+
+	// pull the image
+	p, err := obj.client.ImagePull(ctx, obj.image, types.ImagePullOptions{})
+	if err != nil {
+		return false, errwrap.Wrapf(err, "error pulling image")
+	}
+	// Wait for the image to download, EOF signals that it's done.
+	if _, err := ioutil.ReadAll(p); err != nil {
+		return false, errwrap.Wrapf(err, "error reading image pull result")
+	}
+
+	return false, nil
+}
+
+// Cmp compares two resources and returns an error if they are not equivalent.
+func (obj *DockerImageRes) Cmp(r engine.Res) error {
+	// we can only compare DockerImageRes to others of the same resource kind
+	res, ok := r.(*DockerImageRes)
+	if !ok {
+		return fmt.Errorf("error casting r to *DockerImageRes")
+	}
+	if obj.State != res.State {
+		return fmt.Errorf("the State differs")
+	}
+
+	if obj.APIVersion != res.APIVersion {
+		return fmt.Errorf("the APIVersion differs")
+	}
+	return nil
+}
+
+// DockerImageUID is the UID struct for DockerImageRes.
+type DockerImageUID struct {
+	engine.BaseUID
+
+	image string
+}
+
+// UIDs includes all params to make a unique identification of this object. Most
+// resources only return one, although some resources can return multiple.
+func (obj *DockerImageRes) UIDs() []engine.ResUID {
+	x := &DockerImageUID{
+		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
+		image:   dockerImageNameTag(obj.Name()),
+	}
+	return []engine.ResUID{x}
+}
+
+// AutoEdges returns the AutoEdge interface.
+func (obj *DockerImageRes) AutoEdges() (engine.AutoEdge, error) {
+	return nil, nil
+}
+
+// IFF aka if and only if they are equivalent, return true. If not, false.
+func (obj *DockerImageUID) IFF(uid engine.ResUID) bool {
+	res, ok := uid.(*DockerImageUID)
+	if !ok {
+		return false
+	}
+	return obj.image == res.image
+}
+
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
+// primarily useful for setting the defaults.
+func (obj *DockerImageRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type rawRes DockerImageRes // indirection to avoid infinite recursion
+
+	def := obj.Default()             // get the default
+	res, ok := def.(*DockerImageRes) // put in the right format
+	if !ok {
+		return fmt.Errorf("could not convert to DockerImageRes")
+	}
+	raw := rawRes(*res) // convert; the defaults go here
+
+	if err := unmarshal(&raw); err != nil {
+		return err
+	}
+
+	*obj = DockerImageRes(raw) // restore from indirection with type conversion!
+	return nil
+}
+
+// dockerImageNameTag does a naive check to see if the input includes a tag or
+// is a url, and if not, appends the `:latest` tag to ensure disambiguation.
+func dockerImageNameTag(image string) string {
+	if strings.Contains(image, ":") {
+		return image
+	}
+	return image + ":latest"
+}

engine/resources/exec.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -24,16 +24,16 @@ import (
 	"fmt"
 	"os/exec"
 	"os/user"
+	"sort"
 	"strings"
 	"sync"
 	"syscall"
+	"time"
 
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/engine/traits"
 	engineUtil "github.com/purpleidea/mgmt/engine/util"
-	"github.com/purpleidea/mgmt/util"
-
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 func init() {
@@ -44,26 +44,63 @@ func init() {
 type ExecRes struct {
 	traits.Base // add the base methods without re-implementation
 	traits.Edgeable
+	traits.Sendable
 
 	init *engine.Init
 
-	Cmd        string  `yaml:"cmd"`        // the command to run
-	Cwd        string  `yaml:"cwd"`        // the dir to run the command in (empty means use `pwd` of command)
-	Shell      string  `yaml:"shell"`      // the (optional) shell to use to run the cmd
-	Timeout    int     `yaml:"timeout"`    // the cmd timeout in seconds
-	WatchCmd   string  `yaml:"watchcmd"`   // the watch command to run
-	WatchCwd   string  `yaml:"watchcwd"`   // the dir to run the watch command in (empty means use `pwd` of command)
-	WatchShell string  `yaml:"watchshell"` // the (optional) shell to use to run the watch cmd
-	IfCmd      string  `yaml:"ifcmd"`      // the if command to run
-	IfCwd      string  `yaml:"ifcwd"`      // the dir to run the if command in (empty means use `pwd` of command)
-	IfShell    string  `yaml:"ifshell"`    // the (optional) shell to use to run the if cmd
-	User       string  `yaml:"user"`       // the (optional) user to use to execute the command
-	Group      string  `yaml:"group"`      // the (optional) group to use to execute the command
-	Output     *string // all cmd output, read only, do not set!
-	Stdout     *string // the cmd stdout, read only, do not set!
-	Stderr     *string // the cmd stderr, read only, do not set!
+	// Cmd is the command to run. If this is not specified, we use the name.
+	Cmd string `yaml:"cmd"`
+	// Args is a list of args to pass to Cmd. This can be used *instead* of
+	// passing the full command and args as a single string to Cmd. It can
+	// only be used when a Shell is *not* specified. The advantage of this
+	// is that you don't have to worry about escape characters.
+	Args []string `yaml:"args"`
+	// Cwd is the dir to run the command in. If empty, then this will use
+	// the working directory of the calling process. (This process is mgmt,
+	// not the process being run here.)
+	Cwd string `yaml:"cwd"`
+	// Shell is the (optional) shell to use to run the cmd. If you specify
+	// this, then you can't use the Args parameter.
+	Shell string `yaml:"shell"`
+	// Timeout is the number of seconds to wait before sending a Kill to the
+	// running command. If the Kill is received before the process exits,
+	// then this be treated as an error.
+	Timeout uint64 `yaml:"timeout"`
+	// Env allows the user to specify environment variables for script
+	// execution. These are taken using a map of format of VAR_NAME -> value.
+	Env map[string]string `yaml:"env"`
 
-	wg *sync.WaitGroup
+	// Watch is the command to run to detect event changes. Each line of
+	// output from this command is treated as an event.
+	WatchCmd string `yaml:"watchcmd"`
+	// WatchCwd is the Cwd for the WatchCmd. See the docs for Cwd.
+	WatchCwd string `yaml:"watchcwd"`
+	// WatchShell is the Shell for the WatchCmd. See the docs for Shell.
+	WatchShell string `yaml:"watchshell"`
+
+	// IfCmd is the command that runs to guard against running the Cmd. If
+	// this command succeeds, then Cmd *will* be run. If this command
+	// returns a non-zero result, then the Cmd will not be run. Any error
+	// scenario or timeout will cause the resource to error.
+	IfCmd string `yaml:"ifcmd"`
+	// IfCwd is the Cwd for the IfCmd. See the docs for Cwd.
+	IfCwd string `yaml:"ifcwd"`
+	// IfShell is the Shell for the IfCmd. See the docs for Shell.
+	IfShell string `yaml:"ifshell"`
+
+	// User is the (optional) user to use to execute the command. It is used
+	// for any command being run.
+	User string `yaml:"user"`
+	// Group is the (optional) group to use to execute the command. It is
+	// used for any command being run.
+	Group string `yaml:"group"`
+
+	output *string // all cmd output, read only, do not set!
+	stdout *string // the cmd stdout, read only, do not set!
+	stderr *string // the cmd stderr, read only, do not set!
+
+	interruptChan chan struct{}
+	wg            *sync.WaitGroup
 }
 
 // Default returns some sensible defaults for this resource.
@@ -71,10 +108,27 @@ func (obj *ExecRes) Default() engine.Res {
 	return &ExecRes{}
 }
 
+// getCmd returns the actual command to run. When Cmd is not specified, we use
+// the Name.
+func (obj *ExecRes) getCmd() string {
+	if obj.Cmd != "" {
+		return obj.Cmd
+	}
+	return obj.Name()
+}
+
 // Validate if the params passed in are valid data.
 func (obj *ExecRes) Validate() error {
-	if obj.Cmd == "" { // this is the only thing that is really required
-		return fmt.Errorf("command can't be empty")
+	if obj.getCmd() == "" { // this is the only thing that is really required
+		return fmt.Errorf("the Cmd can't be empty")
+	}
+
+	split := strings.Fields(obj.getCmd())
+	if len(obj.Args) > 0 && obj.Shell != "" {
+		return fmt.Errorf("the Args param can't be used with a Shell")
+	}
+	if len(obj.Args) > 0 && len(split) > 1 {
+		return fmt.Errorf("the Args param can't be used when Cmd has args")
 	}
 
 	// check that, if an user or a group is set, we're running as root
@@ -84,10 +138,16 @@ func (obj *ExecRes) Validate() error {
 			return errwrap.Wrapf(err, "error looking up current user")
 		}
 		if currentUser.Uid != "0" {
-			return errwrap.Errorf("running as root is required if you want to use exec with a different user/group")
+			return fmt.Errorf("running as root is required if you want to use exec with a different user/group")
 		}
 	}
 
+	// check that environment variables' format is valid
+	for key := range obj.Env {
+		if err := isNameValid(key); err != nil {
+			return errwrap.Wrapf(err, "invalid variable name")
+		}
+	}
 	return nil
 }
 
@@ -95,6 +155,7 @@ func (obj *ExecRes) Validate() error {
 func (obj *ExecRes) Init(init *engine.Init) error {
 	obj.init = init // save for later
 
+	obj.interruptChan = make(chan struct{})
 	obj.wg = &sync.WaitGroup{}
 
 	return nil
@@ -107,7 +168,7 @@ func (obj *ExecRes) Close() error {
 
 // Watch is the primary listener for this resource and it outputs events.
 func (obj *ExecRes) Watch() error {
-	ioChan := make(chan *bufioOutput)
+	ioChan := make(chan *cmdOutput)
 	defer obj.wg.Wait()
 
 	if obj.WatchCmd != "" {
@@ -125,7 +186,10 @@ func (obj *ExecRes) Watch() error {
 			cmdName = obj.WatchShell // usually bash, or sh
 			cmdArgs = []string{"-c", obj.WatchCmd}
 		}
-		cmd := exec.Command(cmdName, cmdArgs...)
+
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		cmd := exec.CommandContext(ctx, cmdName, cmdArgs...)
 		cmd.Dir = obj.WatchCwd // run program in pwd if ""
 		// ignore signals sent to parent process (we're in our own group)
 		cmd.SysProcAttr = &syscall.SysProcAttr{
@@ -139,25 +203,9 @@ func (obj *ExecRes) Watch() error {
 			return errwrap.Wrapf(err, "error while setting credential")
 		}
 
-		cmdReader, err := cmd.StdoutPipe()
-		if err != nil {
-			return errwrap.Wrapf(err, "error creating StdoutPipe for Cmd")
+		if ioChan, err = obj.cmdOutputRunner(ctx, cmd); err != nil {
+			return errwrap.Wrapf(err, "error starting WatchCmd")
 		}
-		scanner := bufio.NewScanner(cmdReader)
-
-		defer cmd.Wait() // wait for the command to exit before return!
-		defer func() {
-			// FIXME: without wrapping this in this func it panic's
-			// when running certain graphs... why?
-			cmd.Process.Kill() // shutdown the Watch command on exit
-		}()
-		if err := cmd.Start(); err != nil {
-			return errwrap.Wrapf(err, "error starting Cmd")
-		}
-
-		ctx, cancel := context.WithCancel(context.Background())
-		defer cancel() // unblock and cleanup
-		ioChan = obj.bufioChanScanner(ctx, scanner)
 	}
 
 	obj.init.Running() // when started, notify engine that we're running
@@ -172,12 +220,34 @@ func (obj *ExecRes) Watch() error {
 				return fmt.Errorf("reached EOF")
 			}
 			if err := data.err; err != nil {
-				// error reading input?
-				return errwrap.Wrapf(err, "unknown error")
+				// error reading input or cmd failure
+				exitErr, ok := err.(*exec.ExitError) // embeds an os.ProcessState
+				if !ok {
+					// command failed in some bad way
+					return errwrap.Wrapf(err, "watchcmd failed in some bad way")
+				}
+				pStateSys := exitErr.Sys() // (*os.ProcessState) Sys
+				wStatus, ok := pStateSys.(syscall.WaitStatus)
+				if !ok {
+					return errwrap.Wrapf(err, "could not get exit status of watchcmd")
+				}
+				exitStatus := wStatus.ExitStatus()
+				if exitStatus == 0 {
+					// i'm not sure if this could happen
+					return errwrap.Wrapf(err, "unexpected watchcmd exit status of zero")
+				}
+
+				obj.init.Logf("watchcmd exited with: %d", exitStatus)
+				return errwrap.Wrapf(err, "watchcmd errored")
 			}
 
 			// each time we get a line of output, we loop!
-			obj.init.Logf("watch output: %s", data.text)
+			if s := data.text; s == "" {
+				obj.init.Logf("watch output is empty!")
+			} else {
+				obj.init.Logf("watch output is:")
+				obj.init.Logf(s)
+			}
 			if data.text != "" {
 				send = true
 			}
@@ -231,11 +301,43 @@ func (obj *ExecRes) CheckApply(apply bool) (bool, error) {
 			return false, errwrap.Wrapf(err, "error while setting credential")
 		}
 
+		var out splitWriter
+		out.Init()
+		cmd.Stdout = out.Stdout
+		cmd.Stderr = out.Stderr
+
 		if err := cmd.Run(); err != nil {
-			// TODO: check exit value
+			exitErr, ok := err.(*exec.ExitError) // embeds an os.ProcessState
+			if !ok {
+				// command failed in some bad way
+				return false, errwrap.Wrapf(err, "ifcmd failed in some bad way")
+			}
+			pStateSys := exitErr.Sys() // (*os.ProcessState) Sys
+			wStatus, ok := pStateSys.(syscall.WaitStatus)
+			if !ok {
+				return false, errwrap.Wrapf(err, "could not get exit status of ifcmd")
+			}
+			exitStatus := wStatus.ExitStatus()
+			if exitStatus == 0 {
+				// i'm not sure if this could happen
+				return false, errwrap.Wrapf(err, "unexpected ifcmd exit status of zero")
+			}
+
+			obj.init.Logf("ifcmd exited with: %d", exitStatus)
+			if s := out.String(); s == "" {
+				obj.init.Logf("ifcmd output is empty!")
+			} else {
+				obj.init.Logf("ifcmd output is:")
+				obj.init.Logf(s)
+			}
 			return true, nil // don't run
 		}
-
+		if s := out.String(); s == "" {
+			obj.init.Logf("ifcmd output is empty!")
+		} else {
+			obj.init.Logf("ifcmd output is:")
+			obj.init.Logf(s)
+		}
 	}
 
 	// state is not okay, no work done, exit, but without error
@@ -251,17 +353,46 @@ func (obj *ExecRes) CheckApply(apply bool) (bool, error) {
 		// call without a shell
 		// FIXME: are there still whitespace splitting issues?
 		// TODO: we could make the split character user selectable...!
-		split := strings.Fields(obj.Cmd)
+		split := strings.Fields(obj.getCmd())
 		cmdName = split[0]
 		//d, _ := os.Getwd() // TODO: how does this ever error ?
 		//cmdName = path.Join(d, cmdName)
 		cmdArgs = split[1:]
+		if len(obj.Args) > 0 {
+			if len(split) != 1 { // should not happen
+				return false, fmt.Errorf("validation error")
+			}
+			cmdArgs = obj.Args
+		}
 	} else {
 		cmdName = obj.Shell // usually bash, or sh
-		cmdArgs = []string{"-c", obj.Cmd}
+		cmdArgs = []string{"-c", obj.getCmd()}
 	}
-	cmd := exec.Command(cmdName, cmdArgs...)
+
+	wg := &sync.WaitGroup{}
+	defer wg.Wait() // this must be above the defer cancel() call
+	var ctx context.Context
+	var cancel context.CancelFunc
+	if obj.Timeout > 0 { // cmd.Process.Kill() is called on timeout
+		ctx, cancel = context.WithTimeout(context.Background(), time.Duration(obj.Timeout)*time.Second)
+	} else { // zero timeout means no timer
+		ctx, cancel = context.WithCancel(context.Background())
+	}
+	defer cancel()
+	cmd := exec.CommandContext(ctx, cmdName, cmdArgs...)
 	cmd.Dir = obj.Cwd // run program in pwd if ""
+
+	envKeys := []string{}
+	for key := range obj.Env {
+		envKeys = append(envKeys, key)
+	}
+	sort.Strings(envKeys)
+	cmdEnv := []string{}
+	for _, k := range envKeys {
+		cmdEnv = append(cmdEnv, k+"="+obj.Env[k])
+	}
+	cmd.Env = cmdEnv
+
 	// ignore signals sent to parent process (we're in our own group)
 	cmd.SysProcAttr = &syscall.SysProcAttr{
 		Setpgid: true,
@@ -285,35 +416,32 @@ func (obj *ExecRes) CheckApply(apply bool) (bool, error) {
 		return false, errwrap.Wrapf(err, "error starting cmd")
 	}
 
-	timeout := obj.Timeout
-	if timeout == 0 { // zero timeout means no timer, so disable it
-		timeout = -1
-	}
-	done := make(chan error)
-	go func() { done <- cmd.Wait() }()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		select {
+		case <-obj.interruptChan:
+			cancel()
+		case <-ctx.Done():
+			// let this exit
+		}
+	}()
 
-	select {
-	case e := <-done:
-		err = e // store
-
-	case <-util.TimeAfterOrBlock(timeout):
-		cmd.Process.Kill() // TODO: check error?
-		return false, fmt.Errorf("timeout for cmd")
-	}
+	err = cmd.Wait() // we can unblock this with the timeout
 
 	// save in memory for send/recv
 	// we use pointers to strings to indicate if used or not
 	if out.Stdout.Activity || out.Stderr.Activity {
 		str := out.String()
-		obj.Output = &str
+		obj.output = &str
 	}
 	if out.Stdout.Activity {
 		str := out.Stdout.String()
-		obj.Stdout = &str
+		obj.stdout = &str
 	}
 	if out.Stderr.Activity {
 		str := out.Stderr.String()
-		obj.Stderr = &str
+		obj.stderr = &str
 	}
 
 	// process the err result from cmd, we process non-zero exits here too!
@@ -324,7 +452,18 @@ func (obj *ExecRes) CheckApply(apply bool) (bool, error) {
 		if !ok {
 			return false, errwrap.Wrapf(err, "error running cmd")
 		}
-		return false, fmt.Errorf("cmd error, exit status: %d", wStatus.ExitStatus())
+		exitStatus := wStatus.ExitStatus()
+		if !wStatus.Signaled() { // not a timeout or cancel (no signal)
+			return false, errwrap.Wrapf(err, "cmd error, exit status: %d", exitStatus)
+		}
+		sig := wStatus.Signal()
+
+		// we get this on timeout, because ctx calls cmd.Process.Kill()
+		if sig == syscall.SIGKILL {
+			return false, errwrap.Wrapf(err, "cmd timeout, exit status: %d", exitStatus)
+		}
+
+		return false, errwrap.Wrapf(err, "unknown cmd error, signal: %s, exit status: %d", sig, exitStatus)
 
 	} else if err != nil {
 		return false, errwrap.Wrapf(err, "general cmd error")
@@ -334,11 +473,18 @@ func (obj *ExecRes) CheckApply(apply bool) (bool, error) {
 	// would be nice, but it would require terminal log output that doesn't
 	// interleave all the parallel parts which would mix it all up...
 	if s := out.String(); s == "" {
-		obj.init.Logf("Command output is empty!")
-
+		obj.init.Logf("command output is empty!")
 	} else {
-		obj.init.Logf("Command output is:")
-		obj.init.Logf(out.String())
+		obj.init.Logf("command output is:")
+		obj.init.Logf(s)
+	}
+
+	if err := obj.init.Send(&ExecSends{
+		Output: obj.output,
+		Stdout: obj.stdout,
+		Stderr: obj.stderr,
+	}); err != nil {
+		return false, err
 	}
 
 	// The state tracking is for exec resources that can't "detect" their
@@ -351,58 +497,67 @@ func (obj *ExecRes) CheckApply(apply bool) (bool, error) {
 
 // Cmp compares two resources and returns an error if they are not equivalent.
 func (obj *ExecRes) Cmp(r engine.Res) error {
-	if !obj.Compare(r) {
-		return fmt.Errorf("did not compare")
-	}
-	return nil
-}
-
-// Compare two resources and return if they are equivalent.
-func (obj *ExecRes) Compare(r engine.Res) bool {
 	// we can only compare ExecRes to others of the same resource kind
 	res, ok := r.(*ExecRes)
 	if !ok {
-		return false
+		return fmt.Errorf("not a %s", obj.Kind())
 	}
 
 	if obj.Cmd != res.Cmd {
-		return false
+		return fmt.Errorf("the Cmd differs")
+	}
+	if len(obj.Args) != len(res.Args) {
+		return fmt.Errorf("the Args differ")
+	}
+	for i, a := range obj.Args {
+		if a != res.Args[i] {
+			return fmt.Errorf("the Args differ at index: %d", i)
+		}
 	}
 	if obj.Cwd != res.Cwd {
-		return false
+		return fmt.Errorf("the Cwd differs")
 	}
 	if obj.Shell != res.Shell {
-		return false
+		return fmt.Errorf("the Shell differs")
 	}
 	if obj.Timeout != res.Timeout {
-		return false
-	}
-	if obj.WatchCmd != res.WatchCmd {
-		return false
-	}
-	if obj.WatchCwd != res.WatchCwd {
-		return false
-	}
-	if obj.WatchShell != res.WatchShell {
-		return false
-	}
-	if obj.IfCmd != res.IfCmd {
-		return false
-	}
-	if obj.IfCwd != res.IfCwd {
-		return false
-	}
-	if obj.IfShell != res.IfShell {
-		return false
-	}
-	if obj.User != res.User {
-		return false
-	}
-	if obj.Group != res.Group {
-		return false
+		return fmt.Errorf("the Timeout differs")
 	}
 
-	return true
+	if obj.WatchCmd != res.WatchCmd {
+		return fmt.Errorf("the WatchCmd differs")
+	}
+	if obj.WatchCwd != res.WatchCwd {
+		return fmt.Errorf("the WatchCwd differs")
+	}
+	if obj.WatchShell != res.WatchShell {
+		return fmt.Errorf("the WatchShell differs")
+	}
+
+	if obj.IfCmd != res.IfCmd {
+		return fmt.Errorf("the IfCmd differs")
+	}
+	if obj.IfCwd != res.IfCwd {
+		return fmt.Errorf("the IfCwd differs")
+	}
+	if obj.IfShell != res.IfShell {
+		return fmt.Errorf("the IfShell differs")
+	}
+
+	if obj.User != res.User {
+		return fmt.Errorf("the User differs")
+	}
+	if obj.Group != res.Group {
+		return fmt.Errorf("the Group differs")
+	}
+
+	return nil
+}
+
+// Interrupt is called to ask the execution of this resource to end early.
+func (obj *ExecRes) Interrupt() error {
+	close(obj.interruptChan)
+	return nil
 }
 
 // ExecUID is the UID struct for ExecRes.
@@ -415,25 +570,38 @@ type ExecUID struct {
 
 // ExecResAutoEdges holds the state of the auto edge generator.
 type ExecResAutoEdges struct {
-	edges []engine.ResUID
+	edges   []engine.ResUID
+	pointer int
 }
 
 // Next returns the next automatic edge.
 func (obj *ExecResAutoEdges) Next() []engine.ResUID {
-	return obj.edges
+	if len(obj.edges) == 0 {
+		return nil
+	}
+	value := obj.edges[obj.pointer]
+	obj.pointer++
+	return []engine.ResUID{value}
 }
 
-// Test gets results of the earlier Next() call, & returns if we should continue!
+// Test gets results of the earlier Next() call, & returns if we should
+// continue!
 func (obj *ExecResAutoEdges) Test(input []bool) bool {
-	return false // never keep going
-	// TODO: we could return false if we find as many edges as the number of different path's in cmdFiles()
+	if len(obj.edges) <= obj.pointer {
+		return false
+	}
+	if len(input) != 1 { // in case we get given bad data
+		panic(fmt.Sprintf("Expecting a single value!"))
+	}
+	return true // keep going
 }
 
 // AutoEdges returns the AutoEdge interface. In this case the systemd units.
 func (obj *ExecRes) AutoEdges() (engine.AutoEdge, error) {
 	var data []engine.ResUID
+	var reversed = true
+
 	for _, x := range obj.cmdFiles() {
-		var reversed = true
 		data = append(data, &PkgFileUID{
 			BaseUID: engine.BaseUID{
 				Name:     obj.Name(),
@@ -442,26 +610,75 @@ func (obj *ExecRes) AutoEdges() (engine.AutoEdge, error) {
 			},
 			path: x, // what matters
 		})
+		data = append(data, &FileUID{
+			BaseUID: engine.BaseUID{
+				Name:     obj.Name(),
+				Kind:     obj.Kind(),
+				Reversed: &reversed,
+			},
+			path: x,
+		})
 	}
+	if obj.User != "" {
+		data = append(data, &UserUID{
+			BaseUID: engine.BaseUID{
+				Name:     obj.Name(),
+				Kind:     obj.Kind(),
+				Reversed: &reversed,
+			},
+			name: obj.User,
+		})
+	}
+	if obj.Group != "" {
+		data = append(data, &GroupUID{
+			BaseUID: engine.BaseUID{
+				Name:     obj.Name(),
+				Kind:     obj.Kind(),
+				Reversed: &reversed,
+			},
+			name: obj.Group,
+		})
+	}
+
 	return &ExecResAutoEdges{
-		edges: data,
+		edges:   data,
+		pointer: 0,
 	}, nil
 }
 
-// UIDs includes all params to make a unique identification of this object.
-// Most resources only return one, although some resources can return multiple.
+// UIDs includes all params to make a unique identification of this object. Most
+// resources only return one, although some resources can return multiple.
 func (obj *ExecRes) UIDs() []engine.ResUID {
 	x := &ExecUID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
-		Cmd:     obj.Cmd,
+		Cmd:     obj.getCmd(),
 		IfCmd:   obj.IfCmd,
 		// TODO: add more params here
 	}
 	return []engine.ResUID{x}
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
-// It is primarily useful for setting the defaults.
+// ExecSends is the struct of data which is sent after a successful Apply.
+type ExecSends struct {
+	// Output is the combined stdout and stderr of the command.
+	Output *string `lang:"output"`
+	// Stdout is the stdout of the command.
+	Stdout *string `lang:"stdout"`
+	// Stderr is the stderr of the command.
+	Stderr *string `lang:"stderr"`
+}
+
+// Sends represents the default struct of values we can send using Send/Recv.
+func (obj *ExecRes) Sends() interface{} {
+	return &ExecSends{
+		Output: nil,
+		Stdout: nil,
+		Stderr: nil,
+	}
+}
+
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
+// primarily useful for setting the defaults.
 func (obj *ExecRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes ExecRes // indirection to avoid infinite recursion
 
@@ -516,7 +733,7 @@ func (obj *ExecRes) cmdFiles() []string {
 	var paths []string
 	if obj.Shell != "" {
 		paths = append(paths, obj.Shell)
-	} else if cmdSplit := strings.Fields(obj.Cmd); len(cmdSplit) > 0 {
+	} else if cmdSplit := strings.Fields(obj.getCmd()); len(cmdSplit) > 0 {
 		paths = append(paths, cmdSplit[0])
 	}
 	if obj.WatchShell != "" {
@@ -532,36 +749,56 @@ func (obj *ExecRes) cmdFiles() []string {
 	return paths
 }
 
-// bufioOutput is the output struct of the bufioChanScanner channel output.
-type bufioOutput struct {
+// cmdOutput is the output struct of the cmdOutputRunner channel output. You
+// should always check the error first. If it's nil, then you can assume the
+// text data is good to use.
+type cmdOutput struct {
 	text string
 	err  error
 }
 
-// bufioChanScanner wraps the scanner output in a channel.
-func (obj *ExecRes) bufioChanScanner(ctx context.Context, scanner *bufio.Scanner) chan *bufioOutput {
-	ch := make(chan *bufioOutput)
+// cmdOutputRunner wraps the Cmd in with a StdoutPipe scanner and reads for
+// errors. It runs Start and Wait, and errors runtime things in the channel. If
+// it can't start up the command, it will fail early. Once it's running, it will
+// return the channel which can be used for the duration of the process.
+// Cancelling the context merely unblocks the sending on the output channel, it
+// does not Kill the cmd process. For that you must do it yourself elsewhere.
+func (obj *ExecRes) cmdOutputRunner(ctx context.Context, cmd *exec.Cmd) (chan *cmdOutput, error) {
+	cmdReader, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "error creating StdoutPipe for Cmd")
+	}
+	scanner := bufio.NewScanner(cmdReader)
+	if err := cmd.Start(); err != nil {
+		return nil, errwrap.Wrapf(err, "error starting Cmd")
+	}
+
+	ch := make(chan *cmdOutput)
 	obj.wg.Add(1)
 	go func() {
 		defer obj.wg.Done()
 		defer close(ch)
 		for scanner.Scan() {
 			select {
-			case ch <- &bufioOutput{text: scanner.Text()}: // blocks here ?
+			case ch <- &cmdOutput{text: scanner.Text()}: // blocks here ?
 			case <-ctx.Done():
 				return
 			}
 		}
+
 		// on EOF, scanner.Err() will be nil
-		if err := scanner.Err(); err != nil {
+		reterr := scanner.Err()
+		reterr = errwrap.Append(reterr, cmd.Wait()) // always run Wait()
+		// send any misc errors we encounter on the channel
+		if reterr != nil {
 			select {
-			case ch <- &bufioOutput{err: err}: // send any misc errors we encounter
+			case ch <- &cmdOutput{err: reterr}:
 			case <-ctx.Done():
 				return
 			}
 		}
 	}()
-	return ch
+	return ch, nil
 }
 
 // splitWriter mimics what the ssh.CombinedOutput command does, but stores the
@@ -631,3 +868,20 @@ func (obj *wrapWriter) Write(p []byte) (int, error) {
 func (obj *wrapWriter) String() string {
 	return obj.Buffer.String()
 }
+
+// isNameValid checks that environment variable name is valid.
+func isNameValid(varName string) error {
+	if varName == "" {
+		return fmt.Errorf("variable name cannot be an empty string")
+	}
+	for i := range varName {
+		c := varName[i]
+		if i == 0 && '0' <= c && c <= '9' {
+			return fmt.Errorf("variable name cannot begin with number")
+		}
+		if !(c == '_' || '0' <= c && c <= '9' || 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z') {
+			return fmt.Errorf("invalid character in variable name")
+		}
+	}
+	return nil
+}

engine/resources/exec_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -20,20 +20,36 @@
 package resources
 
 import (
+	"context"
+	"fmt"
+	"os/exec"
+	"syscall"
 	"testing"
+	"time"
 
 	"github.com/purpleidea/mgmt/engine"
+	"github.com/purpleidea/mgmt/engine/graph/autoedge"
+	"github.com/purpleidea/mgmt/pgraph"
 )
 
-func fakeInit(t *testing.T) *engine.Init {
+func fakeExecInit(t *testing.T) (*engine.Init, *ExecSends) {
 	debug := testing.Verbose() // set via the -test.v flag to `go test`
 	logf := func(format string, v ...interface{}) {
 		t.Logf("test: "+format, v...)
 	}
+	execSends := &ExecSends{}
 	return &engine.Init{
+		Send: func(st interface{}) error {
+			x, ok := st.(*ExecSends)
+			if !ok {
+				return fmt.Errorf("unable to send")
+			}
+			*execSends = *x // set
+			return nil
+		},
 		Debug: debug,
 		Logf:  logf,
-	}
+	}, execSends
 }
 
 func TestExecSendRecv1(t *testing.T) {
@@ -50,7 +66,8 @@ func TestExecSendRecv1(t *testing.T) {
 			t.Errorf("close failed with: %v", err)
 		}
 	}()
-	if err := r1.Init(fakeInit(t)); err != nil {
+	init, execSends := fakeExecInit(t)
+	if err := r1.Init(init); err != nil {
 		t.Errorf("init failed with: %v", err)
 	}
 	// run artificially without the entire engine
@@ -58,23 +75,23 @@ func TestExecSendRecv1(t *testing.T) {
 		t.Errorf("checkapply failed with: %v", err)
 	}
 
-	t.Logf("output is: %v", r1.Output)
-	if r1.Output != nil {
-		t.Logf("output is: %v", *r1.Output)
+	t.Logf("output is: %v", execSends.Output)
+	if execSends.Output != nil {
+		t.Logf("output is: %v", *execSends.Output)
 	}
-	t.Logf("stdout is: %v", r1.Stdout)
-	if r1.Stdout != nil {
-		t.Logf("stdout is: %v", *r1.Stdout)
+	t.Logf("stdout is: %v", execSends.Stdout)
+	if execSends.Stdout != nil {
+		t.Logf("stdout is: %v", *execSends.Stdout)
 	}
-	t.Logf("stderr is: %v", r1.Stderr)
-	if r1.Stderr != nil {
-		t.Logf("stderr is: %v", *r1.Stderr)
+	t.Logf("stderr is: %v", execSends.Stderr)
+	if execSends.Stderr != nil {
+		t.Logf("stderr is: %v", *execSends.Stderr)
 	}
 
-	if r1.Stdout == nil {
+	if execSends.Stdout == nil {
 		t.Errorf("stdout is nil")
 	} else {
-		if out := *r1.Stdout; out != "hello world\n" {
+		if out := *execSends.Stdout; out != "hello world\n" {
 			t.Errorf("got wrong stdout(%d): %s", len(out), out)
 		}
 	}
@@ -94,7 +111,8 @@ func TestExecSendRecv2(t *testing.T) {
 			t.Errorf("close failed with: %v", err)
 		}
 	}()
-	if err := r1.Init(fakeInit(t)); err != nil {
+	init, execSends := fakeExecInit(t)
+	if err := r1.Init(init); err != nil {
 		t.Errorf("init failed with: %v", err)
 	}
 	// run artificially without the entire engine
@@ -102,23 +120,23 @@ func TestExecSendRecv2(t *testing.T) {
 		t.Errorf("checkapply failed with: %v", err)
 	}
 
-	t.Logf("output is: %v", r1.Output)
-	if r1.Output != nil {
-		t.Logf("output is: %v", *r1.Output)
+	t.Logf("output is: %v", execSends.Output)
+	if execSends.Output != nil {
+		t.Logf("output is: %v", *execSends.Output)
 	}
-	t.Logf("stdout is: %v", r1.Stdout)
-	if r1.Stdout != nil {
-		t.Logf("stdout is: %v", *r1.Stdout)
+	t.Logf("stdout is: %v", execSends.Stdout)
+	if execSends.Stdout != nil {
+		t.Logf("stdout is: %v", *execSends.Stdout)
 	}
-	t.Logf("stderr is: %v", r1.Stderr)
-	if r1.Stderr != nil {
-		t.Logf("stderr is: %v", *r1.Stderr)
+	t.Logf("stderr is: %v", execSends.Stderr)
+	if execSends.Stderr != nil {
+		t.Logf("stderr is: %v", *execSends.Stderr)
 	}
 
-	if r1.Stderr == nil {
+	if execSends.Stderr == nil {
 		t.Errorf("stderr is nil")
 	} else {
-		if out := *r1.Stderr; out != "hello world\n" {
+		if out := *execSends.Stderr; out != "hello world\n" {
 			t.Errorf("got wrong stderr(%d): %s", len(out), out)
 		}
 	}
@@ -138,7 +156,8 @@ func TestExecSendRecv3(t *testing.T) {
 			t.Errorf("close failed with: %v", err)
 		}
 	}()
-	if err := r1.Init(fakeInit(t)); err != nil {
+	init, execSends := fakeExecInit(t)
+	if err := r1.Init(init); err != nil {
 		t.Errorf("init failed with: %v", err)
 	}
 	// run artificially without the entire engine
@@ -146,42 +165,171 @@ func TestExecSendRecv3(t *testing.T) {
 		t.Errorf("checkapply failed with: %v", err)
 	}
 
-	t.Logf("output is: %v", r1.Output)
-	if r1.Output != nil {
-		t.Logf("output is: %v", *r1.Output)
+	t.Logf("output is: %v", execSends.Output)
+	if execSends.Output != nil {
+		t.Logf("output is: %v", *execSends.Output)
 	}
-	t.Logf("stdout is: %v", r1.Stdout)
-	if r1.Stdout != nil {
-		t.Logf("stdout is: %v", *r1.Stdout)
+	t.Logf("stdout is: %v", execSends.Stdout)
+	if execSends.Stdout != nil {
+		t.Logf("stdout is: %v", *execSends.Stdout)
 	}
-	t.Logf("stderr is: %v", r1.Stderr)
-	if r1.Stderr != nil {
-		t.Logf("stderr is: %v", *r1.Stderr)
+	t.Logf("stderr is: %v", execSends.Stderr)
+	if execSends.Stderr != nil {
+		t.Logf("stderr is: %v", *execSends.Stderr)
 	}
 
-	if r1.Output == nil {
+	if execSends.Output == nil {
 		t.Errorf("output is nil")
 	} else {
 		// it looks like bash or golang race to the write, so whichever
 		// order they come out in is ok, as long as they come out whole
-		if out := *r1.Output; out != "hello world\ngoodbye world\n" && out != "goodbye world\nhello world\n" {
+		if out := *execSends.Output; out != "hello world\ngoodbye world\n" && out != "goodbye world\nhello world\n" {
 			t.Errorf("got wrong output(%d): %s", len(out), out)
 		}
 	}
 
-	if r1.Stdout == nil {
+	if execSends.Stdout == nil {
 		t.Errorf("stdout is nil")
 	} else {
-		if out := *r1.Stdout; out != "hello world\n" {
+		if out := *execSends.Stdout; out != "hello world\n" {
 			t.Errorf("got wrong stdout(%d): %s", len(out), out)
 		}
 	}
 
-	if r1.Stderr == nil {
+	if execSends.Stderr == nil {
 		t.Errorf("stderr is nil")
 	} else {
-		if out := *r1.Stderr; out != "goodbye world\n" {
+		if out := *execSends.Stderr; out != "goodbye world\n" {
 			t.Errorf("got wrong stderr(%d): %s", len(out), out)
 		}
 	}
 }
+
+func TestExecTimeoutBehaviour(t *testing.T) {
+	// cmd.Process.Kill() is called on timeout
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+	cmdName := "/bin/sleep"    // it's /usr/bin/sleep on modern distros
+	cmdArgs := []string{"300"} // 5 min in seconds
+	cmd := exec.CommandContext(ctx, cmdName, cmdArgs...)
+	// ignore signals sent to parent process (we're in our own group)
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setpgid: true,
+		Pgid:    0,
+	}
+
+	if err := cmd.Start(); err != nil {
+		t.Errorf("error starting cmd: %+v", err)
+		return
+	}
+
+	err := cmd.Wait() // we can unblock this with the timeout
+
+	if err == nil {
+		t.Errorf("expected error, got nil")
+		return
+	}
+
+	exitErr, ok := err.(*exec.ExitError) // embeds an os.ProcessState
+	if err != nil && ok {
+		pStateSys := exitErr.Sys() // (*os.ProcessState) Sys
+		wStatus, ok := pStateSys.(syscall.WaitStatus)
+		if !ok {
+			t.Errorf("error running cmd")
+			return
+		}
+		if !wStatus.Signaled() {
+			t.Errorf("did not get signal, exit status: %d", wStatus.ExitStatus())
+			return
+		}
+
+		// we get this on timeout, because ctx calls cmd.Process.Kill()
+		if sig := wStatus.Signal(); sig != syscall.SIGKILL {
+			t.Errorf("got wrong signal: %+v, exit status: %d", sig, wStatus.ExitStatus())
+			return
+		}
+
+		t.Logf("exit status: %d", wStatus.ExitStatus())
+		return
+
+	} else if err != nil {
+		t.Errorf("general cmd error")
+		return
+	}
+
+	// no error
+}
+
+func TestExecAutoEdge1(t *testing.T) {
+	g, err := pgraph.NewGraph("TestGraph")
+	if err != nil {
+		t.Errorf("error creating graph: %v", err)
+		return
+	}
+
+	resUser, err := engine.NewNamedResource("user", "someuser")
+	if err != nil {
+		t.Errorf("error creating user resource: %v", err)
+		return
+	}
+
+	resGroup, err := engine.NewNamedResource("group", "somegroup")
+	if err != nil {
+		t.Errorf("error creating group resource: %v", err)
+		return
+	}
+
+	resFile, err := engine.NewNamedResource("file", "/somefile")
+	if err != nil {
+		t.Errorf("error creating group resource: %v", err)
+		return
+	}
+
+	resExec, err := engine.NewNamedResource("exec", "somefile")
+	if err != nil {
+		t.Errorf("error creating exec resource: %v", err)
+		return
+	}
+	exc := resExec.(*ExecRes)
+	exc.Cmd = resFile.Name()
+	exc.User = resUser.Name()
+	exc.Group = resGroup.Name()
+
+	g.AddVertex(resUser, resGroup, resFile, resExec)
+
+	if i := g.NumEdges(); i != 0 {
+		t.Errorf("should have 0 edges instead of: %d", i)
+		return
+	}
+
+	debug := testing.Verbose() // set via the -test.v flag to `go test`
+	logf := func(format string, v ...interface{}) {
+		t.Logf("test: "+format, v...)
+	}
+	if err := autoedge.AutoEdge(g, debug, logf); err != nil {
+		t.Errorf("error running autoedges: %v", err)
+		return
+	}
+
+	expected, err := pgraph.NewGraph("Expected")
+	if err != nil {
+		t.Errorf("error creating graph: %v", err)
+		return
+	}
+
+	expectEdge := func(from, to pgraph.Vertex) {
+		edge := &engine.Edge{Name: fmt.Sprintf("%s -> %s (expected)", from, to)}
+		expected.AddEdge(from, to, edge)
+	}
+	expectEdge(resFile, resExec)
+	expectEdge(resUser, resExec)
+	expectEdge(resGroup, resExec)
+
+	vertexCmp := func(v1, v2 pgraph.Vertex) (bool, error) { return v1 == v2, nil } // pointer compare is sufficient
+	edgeCmp := func(e1, e2 pgraph.Edge) (bool, error) { return true, nil }         // we don't care about edges here
+
+	if err := expected.GraphCmp(g, vertexCmp, edgeCmp); err != nil {
+		t.Errorf("graph doesn't match expected: %s", err)
+		return
+	}
+}

engine/resources/file_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -78,7 +78,7 @@ func TestMiscEncodeDecode1(t *testing.T) {
 	e := gob.NewEncoder(&b1)
 	err = e.Encode(&input) // pass with &
 	if err != nil {
-		t.Errorf("Gob failed to Encode: %v", err)
+		t.Errorf("gob failed to Encode: %v", err)
 	}
 	str := base64.StdEncoding.EncodeToString(b1.Bytes())
 
@@ -86,27 +86,27 @@ func TestMiscEncodeDecode1(t *testing.T) {
 	var output interface{}
 	bb, err := base64.StdEncoding.DecodeString(str)
 	if err != nil {
-		t.Errorf("Base64 failed to Decode: %v", err)
+		t.Errorf("base64 failed to Decode: %v", err)
 	}
 	b2 := bytes.NewBuffer(bb)
 	d := gob.NewDecoder(b2)
 	err = d.Decode(&output) // pass with &
 	if err != nil {
-		t.Errorf("Gob failed to Decode: %v", err)
+		t.Errorf("gob failed to Decode: %v", err)
 	}
 
 	res1, ok := input.(engine.Res)
 	if !ok {
-		t.Errorf("Input %v is not a Res", res1)
+		t.Errorf("input %v is not a Res", res1)
 		return
 	}
 	res2, ok := output.(engine.Res)
 	if !ok {
-		t.Errorf("Output %v is not a Res", res2)
+		t.Errorf("output %v is not a Res", res2)
 		return
 	}
 	if err := res1.Cmp(res2); err != nil {
-		t.Errorf("The input and output Res values do not match: %+v", err)
+		t.Errorf("the input and output Res values do not match: %+v", err)
 	}
 }
 
@@ -116,34 +116,135 @@ func TestMiscEncodeDecode2(t *testing.T) {
 	// encode
 	input, err := engine.NewNamedResource("file", "file1")
 	if err != nil {
-		t.Errorf("Can't create: %v", err)
+		t.Errorf("can't create: %v", err)
 		return
 	}
+	// NOTE: Do not add this bit of code, because it would cause the path to
+	// get taken from the actual Path parameter, instead of using the name,
+	// and if we use the name, the Cmp function will detect if the name is
+	// stored properly or not.
+	//fileRes := input.(*FileRes) // must not panic
+	//fileRes.Path = "/tmp/whatever"
 
 	b64, err := engineUtil.ResToB64(input)
 	if err != nil {
-		t.Errorf("Can't encode: %v", err)
+		t.Errorf("can't encode: %v", err)
 		return
 	}
 
 	output, err := engineUtil.B64ToRes(b64)
 	if err != nil {
-		t.Errorf("Can't decode: %v", err)
+		t.Errorf("can't decode: %v", err)
 		return
 	}
 
 	res1, ok := input.(engine.Res)
 	if !ok {
-		t.Errorf("Input %v is not a Res", res1)
+		t.Errorf("input %v is not a Res", res1)
 		return
 	}
 	res2, ok := output.(engine.Res)
 	if !ok {
-		t.Errorf("Output %v is not a Res", res2)
+		t.Errorf("output %v is not a Res", res2)
 		return
 	}
+	// this uses the standalone file cmp function
 	if err := res1.Cmp(res2); err != nil {
-		t.Errorf("The input and output Res values do not match: %+v", err)
+		t.Errorf("the input and output Res values do not match: %+v", err)
+	}
+}
+
+func TestMiscEncodeDecode3(t *testing.T) {
+	var err error
+
+	// encode
+	input, err := engine.NewNamedResource("file", "file1")
+	if err != nil {
+		t.Errorf("can't create: %v", err)
+		return
+	}
+	fileRes := input.(*FileRes) // must not panic
+	fileRes.Path = "/tmp/whatever"
+	// TODO: add other params/traits/etc here!
+
+	b64, err := engineUtil.ResToB64(input)
+	if err != nil {
+		t.Errorf("can't encode: %v", err)
+		return
+	}
+
+	output, err := engineUtil.B64ToRes(b64)
+	if err != nil {
+		t.Errorf("can't decode: %v", err)
+		return
+	}
+
+	res1, ok := input.(engine.Res)
+	if !ok {
+		t.Errorf("input %v is not a Res", res1)
+		return
+	}
+	res2, ok := output.(engine.Res)
+	if !ok {
+		t.Errorf("output %v is not a Res", res2)
+		return
+	}
+	// this uses the more complete, engine cmp function
+	if err := engine.ResCmp(res1, res2); err != nil {
+		t.Errorf("the input and output Res values do not match: %+v", err)
+	}
+}
+
+func TestMiscEncodeDecode4(t *testing.T) {
+	var err error
+	const (
+		Kind = "file"
+		Name = "file1"
+	)
+
+	// encode
+	input, err := engine.NewNamedResource(Kind, Name)
+	if err != nil {
+		t.Errorf("can't create: %v", err)
+		return
+	}
+	fileRes := input.(*FileRes) // must not panic
+	fileRes.Path = "/tmp/whatever"
+	// TODO: add other params/traits/etc here!
+
+	b64, err := engineUtil.ResToB64(input)
+	if err != nil {
+		t.Errorf("can't encode: %v", err)
+		return
+	}
+
+	output, err := engineUtil.B64ToRes(b64)
+	if err != nil {
+		t.Errorf("can't decode: %v", err)
+		return
+	}
+
+	res1, ok := input.(engine.Res)
+	if !ok {
+		t.Errorf("input %v is not a Res", res1)
+		return
+	}
+	res2, ok := output.(engine.Res)
+	if !ok {
+		t.Errorf("output %v is not a Res", res2)
+		return
+	}
+	// this uses the more complete, engine cmp function
+	if err := engine.ResCmp(res1, res2); err != nil {
+		t.Errorf("the input and output Res values do not match: %+v", err)
+	}
+
+	// ensure the kind and name are correctly decoded too!
+	if kind := res2.Kind(); kind != Kind {
+		t.Errorf("the output kind was `%s`, expected `%s`", kind, Kind)
+	}
+	if name := res2.Name(); name != Name {
+		t.Errorf("the output name was `%s`, expected `%s`", name, Name)
 	}
 }
 

engine/resources/group.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -28,8 +28,7 @@ import (
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/engine/traits"
 	"github.com/purpleidea/mgmt/recwatch"
-
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 func init() {
@@ -59,7 +58,7 @@ func (obj *GroupRes) Default() engine.Res {
 // Validate if the params passed in are valid data.
 func (obj *GroupRes) Validate() error {
 	if obj.State != "exists" && obj.State != "absent" {
-		return fmt.Errorf("State must be 'exists' or 'absent'")
+		return fmt.Errorf("state must be 'exists' or 'absent'")
 	}
 	return nil
 }
@@ -119,7 +118,7 @@ func (obj *GroupRes) Watch() error {
 }
 
 // CheckApply method for Group resource.
-func (obj *GroupRes) CheckApply(apply bool) (checkOK bool, err error) {
+func (obj *GroupRes) CheckApply(apply bool) (bool, error) {
 	obj.init.Logf("CheckApply(%t)", apply)
 
 	// check if the group exists
@@ -221,32 +220,24 @@ func (obj *GroupRes) CheckApply(apply bool) (checkOK bool, err error) {
 
 // Cmp compares two resources and returns an error if they are not equivalent.
 func (obj *GroupRes) Cmp(r engine.Res) error {
-	if !obj.Compare(r) {
-		return fmt.Errorf("did not compare")
-	}
-	return nil
-}
-
-// Compare two resources and return if they are equivalent.
-func (obj *GroupRes) Compare(r engine.Res) bool {
 	// we can only compare GroupRes to others of the same resource kind
 	res, ok := r.(*GroupRes)
 	if !ok {
-		return false
+		return fmt.Errorf("not a %s", obj.Kind())
 	}
 
 	if obj.State != res.State {
-		return false
+		return fmt.Errorf("the State differs")
 	}
 	if (obj.GID == nil) != (res.GID == nil) {
-		return false
+		return fmt.Errorf("the GID differs")
 	}
 	if obj.GID != nil && res.GID != nil {
 		if *obj.GID != *res.GID {
-			return false
+			return fmt.Errorf("the GID differs")
 		}
 	}
-	return true
+	return nil
 }
 
 // GroupUID is the UID struct for GroupRes.
@@ -280,8 +271,8 @@ func (obj *GroupUID) IFF(uid engine.ResUID) bool {
 	return true
 }
 
-// UIDs includes all params to make a unique identification of this object.
-// Most resources only return one, although some resources can return multiple.
+// UIDs includes all params to make a unique identification of this object. Most
+// resources only return one, although some resources can return multiple.
 func (obj *GroupRes) UIDs() []engine.ResUID {
 	x := &GroupUID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
@@ -291,8 +282,8 @@ func (obj *GroupRes) UIDs() []engine.ResUID {
 	return []engine.ResUID{x}
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
-// It is primarily useful for setting the defaults.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
+// primarily useful for setting the defaults.
 func (obj *GroupRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes GroupRes // indirection to avoid infinite recursion
 

engine/resources/hostname.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -25,9 +25,9 @@ import (
 	"github.com/purpleidea/mgmt/engine/traits"
 	engineUtil "github.com/purpleidea/mgmt/engine/util"
 	"github.com/purpleidea/mgmt/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
 
 	"github.com/godbus/dbus"
-	errwrap "github.com/pkg/errors"
 )
 
 func init() {
@@ -46,12 +46,12 @@ var ErrResourceInsufficientParameters = errors.New("insufficient parameters for
 
 // HostnameRes is a resource that allows setting and watching the hostname.
 //
-// StaticHostname is the one configured in /etc/hostname or a similar file.
-// It is chosen by the local user. It is not always in sync with the current
-// host name as returned by the gethostname() system call.
+// StaticHostname is the one configured in /etc/hostname or a similar file. It
+// is chosen by the local user. It is not always in sync with the current host
+// name as returned by the gethostname() system call.
 //
-// TransientHostname is the one configured via the kernel's sethostbyname().
-// It can be different from the static hostname in case DHCP or mDNS have been
+// TransientHostname is the one configured via the kernel's sethostbyname(). It
+// can be different from the static hostname in case DHCP or mDNS have been
 // configured to change the name based on network information.
 //
 // PrettyHostname is a free-form UTF8 host name for presentation to the user.
@@ -110,7 +110,7 @@ func (obj *HostnameRes) Watch() error {
 	// if we share the bus with others, we will get each others messages!!
 	bus, err := util.SystemBusPrivateUsable() // don't share the bus connection!
 	if err != nil {
-		return errwrap.Wrap(err, "Failed to connect to bus")
+		return errwrap.Wrapf(err, "failed to connect to bus")
 	}
 	defer bus.Close()
 	// watch the PropertiesChanged signal on the hostname1 dbus path
@@ -120,7 +120,7 @@ func (obj *HostnameRes) Watch() error {
 		dbusPropertiesIface,
 	)
 	if call := bus.BusObject().Call(engineUtil.DBusAddMatch, 0, args); call.Err != nil {
-		return errwrap.Wrap(call.Err, "Failed to subscribe to DBus events for hostname1")
+		return errwrap.Wrapf(call.Err, "failed to subscribe to DBus events for hostname1")
 	}
 	defer bus.BusObject().Call(engineUtil.DBusRemoveMatch, 0, args) // ignore the error
 
@@ -147,18 +147,18 @@ func (obj *HostnameRes) Watch() error {
 	}
 }
 
-func (obj *HostnameRes) updateHostnameProperty(object dbus.BusObject, expectedValue, property, setterName string, apply bool) (checkOK bool, err error) {
+func (obj *HostnameRes) updateHostnameProperty(object dbus.BusObject, expectedValue, property, setterName string, apply bool) (bool, error) {
 	propertyObject, err := object.GetProperty("org.freedesktop.hostname1." + property)
 	if err != nil {
 		return false, errwrap.Wrapf(err, "failed to get org.freedesktop.hostname1.%s", property)
 	}
 	if propertyObject.Value() == nil {
-		return false, errwrap.Errorf("Unexpected nil value received when reading property %s", property)
+		return false, fmt.Errorf("unexpected nil value received when reading property %s", property)
 	}
 
 	propertyValue, ok := propertyObject.Value().(string)
 	if !ok {
-		return false, fmt.Errorf("Received unexpected type as %s value, expected string got '%T'", property, propertyValue)
+		return false, fmt.Errorf("received unexpected type as %s value, expected string got '%T'", property, propertyValue)
 	}
 
 	// expected value and actual value match => checkOk
@@ -182,16 +182,16 @@ func (obj *HostnameRes) updateHostnameProperty(object dbus.BusObject, expectedVa
 }
 
 // CheckApply method for Hostname resource.
-func (obj *HostnameRes) CheckApply(apply bool) (checkOK bool, err error) {
+func (obj *HostnameRes) CheckApply(apply bool) (bool, error) {
 	conn, err := util.SystemBusPrivateUsable()
 	if err != nil {
-		return false, errwrap.Wrap(err, "Failed to connect to the private system bus")
+		return false, errwrap.Wrapf(err, "failed to connect to the private system bus")
 	}
 	defer conn.Close()
 
 	hostnameObject := conn.Object(hostname1Iface, hostname1Path)
 
-	checkOK = true
+	checkOK := true
 	if obj.PrettyHostname != "" {
 		propertyCheckOK, err := obj.updateHostnameProperty(hostnameObject, obj.PrettyHostname, "PrettyHostname", "SetPrettyHostname", apply)
 		if err != nil {
@@ -219,31 +219,23 @@ func (obj *HostnameRes) CheckApply(apply bool) (checkOK bool, err error) {
 
 // Cmp compares two resources and returns an error if they are not equivalent.
 func (obj *HostnameRes) Cmp(r engine.Res) error {
-	if !obj.Compare(r) {
-		return fmt.Errorf("did not compare")
-	}
-	return nil
-}
-
-// Compare two resources and return if they are equivalent.
-func (obj *HostnameRes) Compare(r engine.Res) bool {
 	// we can only compare HostnameRes to others of the same resource kind
 	res, ok := r.(*HostnameRes)
 	if !ok {
-		return false
+		return fmt.Errorf("not a %s", obj.Kind())
 	}
 
 	if obj.PrettyHostname != res.PrettyHostname {
-		return false
+		return fmt.Errorf("the PrettyHostname differs")
 	}
 	if obj.StaticHostname != res.StaticHostname {
-		return false
+		return fmt.Errorf("the StaticHostname differs")
 	}
 	if obj.TransientHostname != res.TransientHostname {
-		return false
+		return fmt.Errorf("the TransientHostname differs")
 	}
 
-	return true
+	return nil
 }
 
 // HostnameUID is the UID struct for HostnameRes.
@@ -256,8 +248,8 @@ type HostnameUID struct {
 	transientHostname string
 }
 
-// UIDs includes all params to make a unique identification of this object.
-// Most resources only return one, although some resources can return multiple.
+// UIDs includes all params to make a unique identification of this object. Most
+// resources only return one, although some resources can return multiple.
 func (obj *HostnameRes) UIDs() []engine.ResUID {
 	x := &HostnameUID{
 		BaseUID:           engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
@@ -269,8 +261,8 @@ func (obj *HostnameRes) UIDs() []engine.ResUID {
 	return []engine.ResUID{x}
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
-// It is primarily useful for setting the defaults.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
+// primarily useful for setting the defaults.
 func (obj *HostnameRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes HostnameRes // indirection to avoid infinite recursion
 

engine/resources/http.go

@@ -0,0 +1,808 @@
+// Mgmt
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/purpleidea/mgmt/engine"
+	"github.com/purpleidea/mgmt/engine/traits"
+	"github.com/purpleidea/mgmt/util/errwrap"
+
+	securefilepath "github.com/cyphar/filepath-securejoin"
+)
+
+func init() {
+	engine.RegisterResource("http:server", func() engine.Res { return &HTTPServerRes{} })
+	engine.RegisterResource("http:file", func() engine.Res { return &HTTPFileRes{} })
+}
+
+const (
+	// HTTPUseSecureJoin specifies that we should add in a "secure join" lib
+	// so that we avoid the ../../etc/passwd and symlink problems.
+	HTTPUseSecureJoin = true
+)
+
+// HTTPServerRes is an http server resource. It serves files, but does not
+// actually apply any state. The name is used as the address to listen on,
+// unless the Address field is specified, and in that case it is used instead.
+// This resource can offer up files for serving that are specified either inline
+// in this resource by specifying an http root, or as http:file resources which
+// will get autogrouped into this resource at runtime. The two methods can be
+// combined as well.
+//
+// This server also supports autogrouping some more magical resources into it.
+// For example, the http:flag and http:ui resources add in magic endpoints.
+//
+// This server is not meant as a featureful replacement for the venerable and
+// modern httpd servers out there, but rather as a simple, dynamic, integrated
+// alternative for bootstrapping new machines and clusters in an elegant way.
+//
+// TODO: add support for TLS
+// XXX: Add an http:flag resource that lets an http client set a flag somewhere!
+// XXX: Add a http:ui resource that functions can read data from!
+// XXX: The http:ui resource can also take in values from those functions!
+type HTTPServerRes struct {
+	traits.Base      // add the base methods without re-implementation
+	traits.Edgeable  // XXX: add autoedge support
+	traits.Groupable // can have HTTPFileRes grouped into it
+
+	init *engine.Init
+
+	// Address is the listen address to use for the http server. It is
+	// common to use `:80` (the standard) to listen on TCP port 80 on all
+	// addresses.
+	Address string `lang:"address" yaml:"address"`
+
+	// Timeout is the maximum duration in seconds to use for unspecified
+	// timeouts. In other words, when this value is specified, it is used as
+	// the value for the other *Timeout values when they aren't used. Put
+	// another way, this makes it easy to set all the different timeouts
+	// with a single parameter.
+	Timeout *uint64 `lang:"timeout" yaml:"timeout"`
+
+	// ReadTimeout is the maximum duration in seconds for reading during the
+	// http request. If it is zero, then there is no timeout. If this is
+	// unspecified, then the value of Timeout is used instead if it is set.
+	// For more information, see the golang net/http Server documentation.
+	ReadTimeout *uint64 `lang:"read_timeout" yaml:"read_timeout"`
+
+	// WriteTimeout is the maximum duration in seconds for writing during
+	// the http request. If it is zero, then there is no timeout. If this is
+	// unspecified, then the value of Timeout is used instead if it is set.
+	// For more information, see the golang net/http Server documentation.
+	WriteTimeout *uint64 `lang:"write_timeout" yaml:"write_timeout"`
+
+	// ShutdownTimeout is the maximum duration in seconds to wait for the
+	// server to shutdown gracefully before calling Close. By default it is
+	// nice to let client connections terminate gracefully, however it might
+	// take longer than we are willing to wait, particularly if one is long
+	// polling or running a very long download. As a result, you can set a
+	// timeout here. The default is zero which means it will wait
+	// indefinitely. The shutdown process can also be cancelled by the
+	// interrupt handler which this resource supports. If this is
+	// unspecified, then the value of Timeout is used instead if it is set.
+	ShutdownTimeout *uint64 `lang:"shutdown_timeout" yaml:"shutdown_timeout"`
+
+	// Root is the root directory that we should serve files from. If it is
+	// not specified, then it is not used. Any http file resources will have
+	// precedence over anything in here, in case the same path exists twice.
+	// TODO: should we have a flag to determine the precedence rules here?
+	Root string `lang:"root" yaml:"root"`
+
+	// TODO: should we allow adding a list of one-of files directly here?
+
+	interruptChan chan struct{}
+
+	conn     net.Listener
+	serveMux *http.ServeMux // can't share the global one between resources!
+	server   *http.Server
+}
+
+// Default returns some sensible defaults for this resource.
+func (obj *HTTPServerRes) Default() engine.Res {
+	return &HTTPServerRes{}
+}
+
+// getAddress returns the actual address to use. When Address is not specified,
+// we use the Name.
+func (obj *HTTPServerRes) getAddress() string {
+	if obj.Address != "" {
+		return obj.Address
+	}
+	return obj.Name()
+}
+
+// getReadTimeout determines the value for ReadTimeout, because if unspecified,
+// this will default to the value of Timeout.
+func (obj *HTTPServerRes) getReadTimeout() *uint64 {
+	if obj.ReadTimeout != nil {
+		return obj.ReadTimeout
+	}
+	return obj.Timeout // might be nil
+}
+
+// getWriteTimeout determines the value for WriteTimeout, because if
+// unspecified, this will default to the value of Timeout.
+func (obj *HTTPServerRes) getWriteTimeout() *uint64 {
+	if obj.WriteTimeout != nil {
+		return obj.WriteTimeout
+	}
+	return obj.Timeout // might be nil
+}
+
+// getShutdownTimeout determines the value for ShutdownTimeout, because if
+// unspecified, this will default to the value of Timeout.
+func (obj *HTTPServerRes) getShutdownTimeout() *uint64 {
+	if obj.ShutdownTimeout != nil {
+		return obj.ShutdownTimeout
+	}
+	return obj.Timeout // might be nil
+}
+
+// Validate checks if the resource data structure was populated correctly.
+func (obj *HTTPServerRes) Validate() error {
+	if obj.getAddress() == "" {
+		return fmt.Errorf("empty address")
+	}
+
+	host, _, err := net.SplitHostPort(obj.getAddress())
+	if err != nil {
+		return errwrap.Wrapf(err, "the Address is in an invalid format: %s", obj.getAddress())
+	}
+	if host != "" {
+		// TODO: should we allow fqdn's here?
+		ip := net.ParseIP(host)
+		if ip == nil {
+			return fmt.Errorf("the Address is not a valid IP: %s", host)
+		}
+	}
+
+	if obj.Root != "" && !strings.HasPrefix(obj.Root, "/") {
+		return fmt.Errorf("the Root must be absolute")
+	}
+	if obj.Root != "" && !strings.HasSuffix(obj.Root, "/") {
+		return fmt.Errorf("the Root must be a dir")
+	}
+
+	// XXX: validate that the autogrouped resources don't have paths that
+	// conflict with each other. We can only have a single unique entry for
+	// what handles a /whatever URL.
+
+	return nil
+}
+
+// Init runs some startup code for this resource.
+func (obj *HTTPServerRes) Init(init *engine.Init) error {
+	obj.init = init // save for later
+
+	// No need to error in Validate if Timeout is ignored, but log it.
+	// These are all specified, so Timeout effectively does nothing.
+	a := obj.ReadTimeout != nil
+	b := obj.WriteTimeout != nil
+	c := obj.ShutdownTimeout != nil
+	if obj.Timeout != nil && (a && b && c) {
+		obj.init.Logf("the Timeout param is being ignored")
+	}
+
+	// NOTE: If we don't Init anything that's autogrouped, then it won't
+	// even get an Init call on it.
+	// TODO: should we do this in the engine? Do we want to decide it here?
+	for _, res := range obj.GetGroup() { // grouped elements
+		if err := res.Init(init); err != nil {
+			return errwrap.Wrapf(err, "autogrouped Init failed")
+		}
+	}
+
+	obj.interruptChan = make(chan struct{})
+
+	return nil
+}
+
+// Close is run by the engine to clean up after the resource is done.
+func (obj *HTTPServerRes) Close() error {
+	return nil
+}
+
+// Watch is the primary listener for this resource and it outputs events.
+func (obj *HTTPServerRes) Watch() error {
+	// TODO: I think we could replace all this with:
+	//obj.conn, err := net.Listen("tcp", obj.getAddress())
+	// ...but what is the advantage?
+	addr, err := net.ResolveTCPAddr("tcp", obj.getAddress())
+	if err != nil {
+		return errwrap.Wrapf(err, "could not resolve address")
+	}
+
+	obj.conn, err = net.ListenTCP("tcp", addr)
+	if err != nil {
+		return errwrap.Wrapf(err, "could not start listener")
+	}
+	defer obj.conn.Close()
+
+	obj.serveMux = http.NewServeMux() // do it here in case Watch restarts!
+	obj.serveMux.HandleFunc("/", obj.handler())
+
+	readTimeout := uint64(0)
+	if i := obj.getReadTimeout(); i != nil {
+		readTimeout = *i
+	}
+	writeTimeout := uint64(0)
+	if i := obj.getWriteTimeout(); i != nil {
+		writeTimeout = *i
+	}
+	obj.server = &http.Server{
+		Addr:         obj.getAddress(),
+		Handler:      obj.serveMux,
+		ReadTimeout:  time.Duration(readTimeout) * time.Second,
+		WriteTimeout: time.Duration(writeTimeout) * time.Second,
+		//MaxHeaderBytes: 1 << 20, XXX: should we add a param for this?
+	}
+
+	obj.init.Running() // when started, notify engine that we're running
+
+	var closeError error
+	closeSignal := make(chan struct{})
+
+	wg := &sync.WaitGroup{}
+	defer wg.Wait()
+
+	shutdownChan := make(chan struct{}) // server shutdown finished signal
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		select {
+		case <-obj.interruptChan:
+			// TODO: should we bubble up the error from Close?
+			// TODO: do we need a mutex around this Close?
+			obj.server.Close() // kill it quickly!
+		case <-shutdownChan:
+			// let this exit
+		}
+	}()
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		defer close(closeSignal)
+
+		err := obj.server.Serve(obj.conn) // blocks until Shutdown() is called!
+		if err == nil || err == http.ErrServerClosed {
+			return
+		}
+		// if this returned on its own, then closeSignal can be used...
+		closeError = errwrap.Wrapf(err, "the server errored")
+	}()
+
+	// When Shutdown is called, Serve, ListenAndServe, and ListenAndServeTLS
+	// immediately return ErrServerClosed. Make sure the program doesn't
+	// exit and waits instead for Shutdown to return.
+	defer func() {
+		defer close(shutdownChan) // signal that shutdown is finished
+		ctx := context.Background()
+		if i := obj.getShutdownTimeout(); i != nil && *i > 0 {
+			var cancel context.CancelFunc
+			ctx, cancel = context.WithTimeout(ctx, time.Duration(*i)*time.Second)
+			defer cancel()
+		}
+		err := obj.server.Shutdown(ctx) // shutdown gracefully
+		if err == context.DeadlineExceeded {
+			// TODO: should we bubble up the error from Close?
+			// TODO: do we need a mutex around this Close?
+			obj.server.Close() // kill it now
+		}
+	}()
+
+	startupChan := make(chan struct{})
+	close(startupChan) // send one initial signal
+
+	var send = false // send event?
+	for {
+		if obj.init.Debug {
+			obj.init.Logf("Looping...")
+		}
+
+		select {
+		case <-startupChan:
+			startupChan = nil
+			send = true
+
+		case <-closeSignal: // something shut us down early
+			return closeError
+
+		case <-obj.init.Done: // closed by the engine to signal shutdown
+			return nil
+		}
+
+		// do all our event sending all together to avoid duplicate msgs
+		if send {
+			send = false
+			obj.init.Event() // notify engine of an event (this can block)
+		}
+	}
+}
+
+// CheckApply never has anything to do for this resource, so it always succeeds.
+// It does however check that certain runtime requirements (such as the Root dir
+// existing if one was specified) are fulfilled.
+func (obj *HTTPServerRes) CheckApply(apply bool) (bool, error) {
+	if obj.init.Debug {
+		obj.init.Logf("CheckApply")
+	}
+
+	// XXX: We don't want the initial CheckApply to return true until the
+	// Watch has started up, so we must block here until that's the case...
+
+	// Cheap runtime validation!
+	if obj.Root != "" {
+		fileInfo, err := os.Stat(obj.Root)
+		if err != nil {
+			return false, errwrap.Wrapf(err, "can't stat Root dir")
+		}
+		if !fileInfo.IsDir() {
+			return false, fmt.Errorf("the Root path is not a dir")
+		}
+	}
+
+	return true, nil // always succeeds, with nothing to do!
+}
+
+// Cmp compares two resources and returns an error if they are not equivalent.
+func (obj *HTTPServerRes) Cmp(r engine.Res) error {
+	// we can only compare HTTPServerRes to others of the same resource kind
+	res, ok := r.(*HTTPServerRes)
+	if !ok {
+		return fmt.Errorf("res is not the same kind")
+	}
+
+	if obj.Address != res.Address {
+		return fmt.Errorf("the Address differs")
+	}
+
+	if (obj.Timeout == nil) != (res.Timeout == nil) { // xor
+		return fmt.Errorf("the Timeout differs")
+	}
+	if obj.Timeout != nil && res.Timeout != nil {
+		if *obj.Timeout != *res.Timeout { // compare the values
+			return fmt.Errorf("the value of Timeout differs")
+		}
+	}
+	if (obj.ReadTimeout == nil) != (res.ReadTimeout == nil) {
+		return fmt.Errorf("the ReadTimeout differs")
+	}
+	if obj.ReadTimeout != nil && res.ReadTimeout != nil {
+		if *obj.ReadTimeout != *res.ReadTimeout {
+			return fmt.Errorf("the value of ReadTimeout differs")
+		}
+	}
+	if (obj.WriteTimeout == nil) != (res.WriteTimeout == nil) {
+		return fmt.Errorf("the WriteTimeout differs")
+	}
+	if obj.WriteTimeout != nil && res.WriteTimeout != nil {
+		if *obj.WriteTimeout != *res.WriteTimeout {
+			return fmt.Errorf("the value of WriteTimeout differs")
+		}
+	}
+	if (obj.ShutdownTimeout == nil) != (res.ShutdownTimeout == nil) {
+		return fmt.Errorf("the ShutdownTimeout differs")
+	}
+	if obj.ShutdownTimeout != nil && res.ShutdownTimeout != nil {
+		if *obj.ShutdownTimeout != *res.ShutdownTimeout {
+			return fmt.Errorf("the value of ShutdownTimeout differs")
+		}
+	}
+
+	// TODO: We could do this sort of thing to skip checking Timeout when it
+	// is not used, but for the moment, this is overkill and not needed yet.
+	//a := obj.ReadTimeout != nil
+	//b := obj.WriteTimeout != nil
+	//c := obj.ShutdownTimeout != nil
+	//if !(obj.Timeout != nil && (a && b && c)) {
+	//	// the Timeout param is not being ignored
+	//}
+
+	if obj.Root != res.Root {
+		return fmt.Errorf("the Root differs")
+	}
+
+	return nil
+}
+
+// Interrupt is called to ask the execution of this resource to end early. It
+// will cause the server Shutdown to end abruptly instead of leading open client
+// connections terminate gracefully. It does this by causing the server Close
+// method to run.
+func (obj *HTTPServerRes) Interrupt() error {
+	close(obj.interruptChan) // this should cause obj.server.Close() to run!
+	return nil
+}
+
+// Copy copies the resource. Don't call it directly, use engine.ResCopy instead.
+// TODO: should this copy internal state?
+func (obj *HTTPServerRes) Copy() engine.CopyableRes {
+	var timeout, readTimeout, writeTimeout, shutdownTimeout *uint64
+	if obj.Timeout != nil {
+		x := *obj.Timeout
+		timeout = &x
+	}
+	if obj.ReadTimeout != nil {
+		x := *obj.ReadTimeout
+		readTimeout = &x
+	}
+	if obj.WriteTimeout != nil {
+		x := *obj.WriteTimeout
+		writeTimeout = &x
+	}
+	if obj.ShutdownTimeout != nil {
+		x := *obj.ShutdownTimeout
+		shutdownTimeout = &x
+	}
+	return &HTTPServerRes{
+		Address:         obj.Address,
+		Timeout:         timeout,
+		ReadTimeout:     readTimeout,
+		WriteTimeout:    writeTimeout,
+		ShutdownTimeout: shutdownTimeout,
+		Root:            obj.Root,
+	}
+}
+
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
+// primarily useful for setting the defaults.
+func (obj *HTTPServerRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type rawRes HTTPServerRes // indirection to avoid infinite recursion
+
+	def := obj.Default()            // get the default
+	res, ok := def.(*HTTPServerRes) // put in the right format
+	if !ok {
+		return fmt.Errorf("could not convert to HTTPServerRes")
+	}
+	raw := rawRes(*res) // convert; the defaults go here
+
+	if err := unmarshal(&raw); err != nil {
+		return err
+	}
+
+	*obj = HTTPServerRes(raw) // restore from indirection with type conversion!
+	return nil
+}
+
+// GroupCmp returns whether two resources can be grouped together or not. Can
+// these two resources be merged, aka, does this resource support doing so? Will
+// resource allow itself to be grouped _into_ this obj?
+func (obj *HTTPServerRes) GroupCmp(r engine.GroupableRes) error {
+	res1, ok1 := r.(*HTTPFileRes) // different from what we usually do!
+	if ok1 {
+		// If the http file resource has the Server field specified,
+		// then it must match against our name field if we want it to
+		// group with us.
+		if res1.Server != "" && res1.Server != obj.Name() {
+			return fmt.Errorf("resource groups with a different server name")
+		}
+
+		return nil
+	}
+
+	return fmt.Errorf("resource is not the right kind")
+}
+
+// readHandler handles all the incoming download requests from clients.
+func (obj *HTTPServerRes) handler() func(http.ResponseWriter, *http.Request) {
+	// TODO: we could statically pre-compute some stuff here...
+
+	return func(w http.ResponseWriter, req *http.Request) {
+
+		if obj.init.Debug {
+			obj.init.Logf("Client: %s", req.RemoteAddr)
+		}
+		// TODO: would this leak anything security sensitive in our log?
+		obj.init.Logf("URL: %s", req.URL)
+		if obj.init.Debug {
+			obj.init.Logf("Path: %s", req.URL.Path)
+		}
+
+		// We only allow GET at the moment.
+		if req.Method != http.MethodGet {
+			w.WriteHeader(http.StatusMethodNotAllowed)
+			return
+		}
+
+		requestPath := req.URL.Path // TODO: is this what we want here?
+
+		//var handle io.Reader // TODO: simplify?
+		var handle io.ReadSeeker
+
+		// Look through the autogrouped resources!
+		// TODO: can we improve performance by only searching here once?
+		for _, x := range obj.GetGroup() { // grouped elements
+			res, ok := x.(*HTTPFileRes) // convert from Res
+			if !ok {
+				continue
+			}
+			if requestPath != res.getPath() {
+				continue // not me
+			}
+
+			if obj.init.Debug {
+				obj.init.Logf("Got grouped file: %s", res.String())
+			}
+			var err error
+			handle, err = res.getContent()
+			if err != nil {
+				obj.init.Logf("could not get content for: %s", requestPath)
+				msg, httpStatus := toHTTPError(err)
+				http.Error(w, msg, httpStatus)
+				return
+			}
+			break
+		}
+
+		// Look in root if we have one, and we haven't got a file yet...
+		if obj.Root != "" && handle == nil {
+
+			p := filepath.Join(obj.Root, requestPath) // normal unsafe!
+			if !strings.HasPrefix(p, obj.Root) {      // root ends with /
+				// user might have tried a ../../etc/passwd hack
+				obj.init.Logf("join inconsistency: %s", p)
+				http.NotFound(w, req) // lie to them...
+				return
+			}
+			if HTTPUseSecureJoin {
+				var err error
+				p, err = securefilepath.SecureJoin(obj.Root, requestPath)
+				if err != nil {
+					obj.init.Logf("secure join fail: %s", p)
+					http.NotFound(w, req) // lie to them...
+					return
+				}
+			}
+			if obj.init.Debug {
+				obj.init.Logf("Got file at root: %s", p)
+			}
+			var err error
+			handle, err = os.Open(p)
+			if err != nil {
+				obj.init.Logf("could not open: %s", p)
+				msg, httpStatus := toHTTPError(err)
+				http.Error(w, msg, httpStatus)
+				return
+			}
+		}
+
+		// We never found a file...
+		if handle == nil {
+			if obj.init.Debug || true { // XXX: maybe we should always do this?
+				obj.init.Logf("File not found: %s", requestPath)
+			}
+			http.NotFound(w, req)
+			return
+		}
+
+		// Determine the last-modified time if we can.
+		modtime := time.Now()
+		if f, ok := handle.(*os.File); ok {
+			fi, err := f.Stat()
+			if err == nil {
+				modtime = fi.ModTime()
+			}
+			// TODO: if Stat errors, should we fail the whole thing?
+		}
+
+		// XXX: is requestPath what we want for the name field?
+		http.ServeContent(w, req, requestPath, modtime, handle)
+		//obj.init.Logf("%d bytes sent", n) // XXX: how do we know (on the server-side) if it worked?
+
+		return
+	}
+}
+
+// HTTPFileRes is a file that exists within an http server. The name is used as
+// the public path of the file, unless the filename field is specified, and in
+// that case it is used instead. The way this works is that it autogroups at
+// runtime with an existing http resource, and in doing so makes the file
+// associated with this resource available for serving from that http server.
+type HTTPFileRes struct {
+	traits.Base      // add the base methods without re-implementation
+	traits.Edgeable  // XXX: add autoedge support
+	traits.Groupable // can be grouped into HTTPServerRes
+
+	init *engine.Init
+
+	// Server is the name of the http server resource to group this into. If
+	// it is omitted, and there is only a single http resource, then it will
+	// be grouped into it automatically. If there is more than one main http
+	// resource being used, then the grouping behaviour is *undefined* when
+	// this is not specified, and it is not recommended to leave this blank!
+	Server string `lang:"server" yaml:"server"`
+
+	// Filename is the name of the file this data should appear as on the
+	// http server.
+	Filename string `lang:"filename" yaml:"filename"`
+
+	// Path is the absolute path to a file that should be used as the source
+	// for this file resource. It must not be combined with the data field.
+	Path string `lang:"path" yaml:"path"`
+
+	// Data is the file content that should be used as the source for this
+	// file resource. It must not be combined with the path field.
+	// TODO: should this be []byte instead?
+	Data string `lang:"data" yaml:"data"`
+}
+
+// Default returns some sensible defaults for this resource.
+func (obj *HTTPFileRes) Default() engine.Res {
+	return &HTTPFileRes{}
+}
+
+// getPath returns the actual path we respond to. When Filename is not
+// specified, we use the Name. Note that this is the filename that will be seen
+// on the http server, it is *not* the source path to the actual file contents
+// being sent by the server.
+func (obj *HTTPFileRes) getPath() string {
+	if obj.Filename != "" {
+		return obj.Filename
+	}
+	return obj.Name()
+}
+
+// getContent returns the content that we expect from this resource. It depends
+// on whether the user specified the Path or Data fields, and whether the Path
+// exists or not.
+func (obj *HTTPFileRes) getContent() (io.ReadSeeker, error) {
+	if obj.Path != "" && obj.Data != "" {
+		// programming error! this should have been caught in Validate!
+		return nil, fmt.Errorf("must not specify Path and Data")
+	}
+
+	if obj.Path != "" {
+		return os.Open(obj.Path)
+	}
+
+	return bytes.NewReader([]byte(obj.Data)), nil
+}
+
+// Validate checks if the resource data structure was populated correctly.
+func (obj *HTTPFileRes) Validate() error {
+	if obj.getPath() == "" {
+		return fmt.Errorf("empty filename")
+	}
+	// FIXME: does getPath need to start with a slash?
+
+	if obj.Path != "" && !strings.HasPrefix(obj.Path, "/") {
+		return fmt.Errorf("the Path must be absolute")
+	}
+
+	if obj.Path != "" && obj.Data != "" {
+		return fmt.Errorf("must not specify Path and Data")
+	}
+
+	// NOTE: if obj.Path == "" && obj.Data == "" then we have an empty file!
+
+	return nil
+}
+
+// Init runs some startup code for this resource.
+func (obj *HTTPFileRes) Init(init *engine.Init) error {
+	obj.init = init // save for later
+
+	return nil
+}
+
+// Close is run by the engine to clean up after the resource is done.
+func (obj *HTTPFileRes) Close() error {
+	return nil
+}
+
+// Watch is the primary listener for this resource and it outputs events. This
+// particular one does absolutely nothing but block until we've received a done
+// signal.
+func (obj *HTTPFileRes) Watch() error {
+	obj.init.Running() // when started, notify engine that we're running
+
+	select {
+	case <-obj.init.Done: // closed by the engine to signal shutdown
+	}
+
+	//obj.init.Event() // notify engine of an event (this can block)
+
+	return nil
+}
+
+// CheckApply never has anything to do for this resource, so it always succeeds.
+func (obj *HTTPFileRes) CheckApply(apply bool) (bool, error) {
+	if obj.init.Debug {
+		obj.init.Logf("CheckApply")
+	}
+
+	return true, nil // always succeeds, with nothing to do!
+}
+
+// Cmp compares two resources and returns an error if they are not equivalent.
+func (obj *HTTPFileRes) Cmp(r engine.Res) error {
+	// we can only compare HTTPFileRes to others of the same resource kind
+	res, ok := r.(*HTTPFileRes)
+	if !ok {
+		return fmt.Errorf("res is not the same kind")
+	}
+
+	if obj.Server != res.Server {
+		return fmt.Errorf("the Server field differs")
+	}
+	if obj.Filename != res.Filename {
+		return fmt.Errorf("the Filename differs")
+	}
+	if obj.Path != res.Path {
+		return fmt.Errorf("the Path differs")
+	}
+	if obj.Data != res.Data {
+		return fmt.Errorf("the Data differs")
+	}
+
+	return nil
+}
+
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
+// primarily useful for setting the defaults.
+func (obj *HTTPFileRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type rawRes HTTPFileRes // indirection to avoid infinite recursion
+
+	def := obj.Default()          // get the default
+	res, ok := def.(*HTTPFileRes) // put in the right format
+	if !ok {
+		return fmt.Errorf("could not convert to HTTPFileRes")
+	}
+	raw := rawRes(*res) // convert; the defaults go here
+
+	if err := unmarshal(&raw); err != nil {
+		return err
+	}
+
+	*obj = HTTPFileRes(raw) // restore from indirection with type conversion!
+	return nil
+}
+
+// toHTTPError returns a non-specific HTTP error message and status code for a
+// given non-nil error value. It's important that toHTTPError does not actually
+// return err.Error(), since msg and httpStatus are returned to users, and
+// historically Go's ServeContent always returned just "404 Not Found" for all
+// errors. We don't want to start leaking information in error messages.
+// NOTE: This was copied and modified slightly from the golang net/http package.
+// See: https://github.com/golang/go/issues/38375
+func toHTTPError(err error) (msg string, httpStatus int) {
+	if os.IsNotExist(err) {
+		//return "404 page not found", http.StatusNotFound
+		return http.StatusText(http.StatusNotFound), http.StatusNotFound
+	}
+	if os.IsPermission(err) {
+		//return "403 Forbidden", http.StatusForbidden
+		return http.StatusText(http.StatusForbidden), http.StatusForbidden
+	}
+	// Default:
+	//return "500 Internal Server Error", http.StatusInternalServerError
+	return http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError
+}

engine/resources/kv.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -18,28 +18,37 @@
 package resources
 
 import (
+	"context"
 	"fmt"
 	"strconv"
+	"sync"
+	"time"
 
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/engine/traits"
-
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 func init() {
 	engine.RegisterResource("kv", func() engine.Res { return &KVRes{} })
 }
 
-// KVResSkipCmpStyle represents the different styles of comparison when using SkipLessThan.
+// KVResSkipCmpStyle represents the different styles of comparison when using
+// SkipLessThan.
 type KVResSkipCmpStyle int
 
-// These are the different allowed comparison styles. Most folks will want SkipCmpStyleInt.
+// These are the different allowed comparison styles. Most folks will want
+// SkipCmpStyleInt.
 const (
 	SkipCmpStyleInt KVResSkipCmpStyle = iota
 	SkipCmpStyleString
 )
 
+const (
+	kvCheckApplyTimeout = 5 * time.Second
+)
+
 // KVRes is a resource which writes a key/value pair into cluster wide storage.
 // It will ensure that the key is set to the requested value. The one exception
 // is that if you use the SkipLessThan parameter, then it will only replace the
@@ -56,14 +65,32 @@ type KVRes struct {
 
 	init *engine.Init
 
-	// XXX: shouldn't the name be the key?
-	Key          string            `yaml:"key"`          // key to set
-	Value        *string           `yaml:"value"`        // value to set (nil to delete)
-	SkipLessThan bool              `yaml:"skiplessthan"` // skip updates as long as stored value is greater
-	SkipCmpStyle KVResSkipCmpStyle `yaml:"skipcmpstyle"` // how to do the less than cmp
+	// Key represents the key to set. If it is not specified, the Name value
+	// is used instead.
+	Key string `lang:"key" yaml:"key"`
+	// Value represents the string value to set. If this value is nil or,
+	// undefined, then this will delete that key.
+	Value *string `lang:"value" yaml:"value"`
+	// SkipLessThan causes the value to be updated as long as it is greater.
+	SkipLessThan bool `lang:"skiplessthan" yaml:"skiplessthan"`
+	// SkipCmpStyle is the type of compare function used when determining if
+	// the value is greater when using the SkipLessThan parameter.
+	SkipCmpStyle KVResSkipCmpStyle `lang:"skipcmpstyle" yaml:"skipcmpstyle"`
+
+	interruptChan chan struct{}
+
 	// TODO: does it make sense to have different backends here? (eg: local)
 }
 
+// getKey returns the key to be used for this resource. If the Key field is
+// specified, it will use that, otherwise it uses the Name.
+func (obj *KVRes) getKey() string {
+	if obj.Key != "" {
+		return obj.Key
+	}
+	return obj.Name()
+}
+
 // Default returns some sensible defaults for this resource.
 func (obj *KVRes) Default() engine.Res {
 	return &KVRes{}
@@ -71,7 +98,7 @@ func (obj *KVRes) Default() engine.Res {
 
 // Validate if the params passed in are valid data.
 func (obj *KVRes) Validate() error {
-	if obj.Key == "" {
+	if obj.getKey() == "" {
 		return fmt.Errorf("key must not be empty")
 	}
 	if obj.SkipLessThan {
@@ -92,6 +119,8 @@ func (obj *KVRes) Validate() error {
 func (obj *KVRes) Init(init *engine.Init) error {
 	obj.init = init // save for later
 
+	obj.interruptChan = make(chan struct{})
+
 	return nil
 }
 
@@ -102,9 +131,17 @@ func (obj *KVRes) Close() error {
 
 // Watch is the primary listener for this resource and it outputs events.
 func (obj *KVRes) Watch() error {
-	obj.init.Running() // when started, notify engine that we're running
+	// FIXME: add timeout to context
+	// The obj.init.Done channel is closed by the engine to signal shutdown.
+	ctx, cancel := util.ContextWithCloser(context.Background(), obj.init.Done)
+	defer cancel()
 
-	ch := obj.init.World.StrMapWatch(obj.Key) // get possible events!
+	ch, err := obj.init.World.StrMapWatch(ctx, obj.getKey()) // get possible events!
+	if err != nil {
+		return err
+	}
+
+	obj.init.Running() // when started, notify engine that we're running
 
 	var send = false // send event?
 	for {
@@ -118,7 +155,7 @@ func (obj *KVRes) Watch() error {
 				return errwrap.Wrapf(err, "unknown %s watcher error", obj)
 			}
 			if obj.init.Debug {
-				obj.init.Logf("Event!")
+				obj.init.Logf("event!")
 			}
 			send = true
 
@@ -135,7 +172,7 @@ func (obj *KVRes) Watch() error {
 }
 
 // lessThanCheck checks for less than validity.
-func (obj *KVRes) lessThanCheck(value string) (checkOK bool, err error) {
+func (obj *KVRes) lessThanCheck(value string) (bool, error) {
 	v := *obj.Value
 	if value == v { // redundant check for safety
 		return true, nil
@@ -173,16 +210,31 @@ func (obj *KVRes) lessThanCheck(value string) (checkOK bool, err error) {
 }
 
 // CheckApply method for Password resource. Does nothing, returns happy!
-func (obj *KVRes) CheckApply(apply bool) (checkOK bool, err error) {
+func (obj *KVRes) CheckApply(apply bool) (bool, error) {
 	obj.init.Logf("CheckApply(%t)", apply)
 
+	wg := &sync.WaitGroup{}
+	defer wg.Wait() // this must be above the defer cancel() call
+	ctx, cancel := context.WithTimeout(context.Background(), kvCheckApplyTimeout)
+	defer cancel()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		select {
+		case <-obj.interruptChan:
+			cancel()
+		case <-ctx.Done():
+			// let this exit
+		}
+	}()
+
 	if val, exists := obj.init.Recv()["Value"]; exists && val.Changed {
 		// if we received on Value, and it changed, wooo, nothing to do.
 		obj.init.Logf("CheckApply: `Value` was updated!")
 	}
 
 	hostname := obj.init.Hostname // me
-	keyMap, err := obj.init.World.StrMapGet(obj.Key)
+	keyMap, err := obj.init.World.StrMapGet(ctx, obj.getKey())
 	if err != nil {
 		return false, errwrap.Wrapf(err, "check error during StrGet")
 	}
@@ -202,7 +254,7 @@ func (obj *KVRes) CheckApply(apply bool) (checkOK bool, err error) {
 		return true, nil // nothing to delete, we're good!
 
 	} else if ok && obj.Value == nil { // delete
-		err := obj.init.World.StrMapDel(obj.Key)
+		err := obj.init.World.StrMapDel(ctx, obj.getKey())
 		return false, errwrap.Wrapf(err, "apply error during StrDel")
 	}
 
@@ -210,7 +262,7 @@ func (obj *KVRes) CheckApply(apply bool) (checkOK bool, err error) {
 		return false, nil
 	}
 
-	if err := obj.init.World.StrMapSet(obj.Key, *obj.Value); err != nil {
+	if err := obj.init.World.StrMapSet(ctx, obj.getKey(), *obj.Value); err != nil {
 		return false, errwrap.Wrapf(err, "apply error during StrSet")
 	}
 
@@ -219,39 +271,37 @@ func (obj *KVRes) CheckApply(apply bool) (checkOK bool, err error) {
 
 // Cmp compares two resources and returns an error if they are not equivalent.
 func (obj *KVRes) Cmp(r engine.Res) error {
-	if !obj.Compare(r) {
-		return fmt.Errorf("did not compare")
-	}
-	return nil
-}
-
-// Compare two resources and return if they are equivalent.
-func (obj *KVRes) Compare(r engine.Res) bool {
 	// we can only compare KVRes to others of the same resource kind
 	res, ok := r.(*KVRes)
 	if !ok {
-		return false
+		return fmt.Errorf("not a %s", obj.Kind())
 	}
 
-	if obj.Key != res.Key {
-		return false
+	if obj.getKey() != res.getKey() {
+		return fmt.Errorf("the Key differs")
 	}
 	if (obj.Value == nil) != (res.Value == nil) { // xor
-		return false
+		return fmt.Errorf("the Value differs")
 	}
 	if obj.Value != nil && res.Value != nil {
 		if *obj.Value != *res.Value { // compare the strings
-			return false
+			return fmt.Errorf("the contents of Value differs")
 		}
 	}
 	if obj.SkipLessThan != res.SkipLessThan {
-		return false
+		return fmt.Errorf("the SkipLessThan param differs")
 	}
 	if obj.SkipCmpStyle != res.SkipCmpStyle {
-		return false
+		return fmt.Errorf("the SkipCmpStyle param differs")
 	}
 
-	return true
+	return nil
+}
+
+// Interrupt is called to ask the execution of this resource to end early.
+func (obj *KVRes) Interrupt() error {
+	close(obj.interruptChan)
+	return nil
 }
 
 // KVUID is the UID struct for KVRes.
@@ -260,8 +310,8 @@ type KVUID struct {
 	name string
 }
 
-// UIDs includes all params to make a unique identification of this object.
-// Most resources only return one, although some resources can return multiple.
+// UIDs includes all params to make a unique identification of this object. Most
+// resources only return one, although some resources can return multiple.
 func (obj *KVRes) UIDs() []engine.ResUID {
 	x := &KVUID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
@@ -270,8 +320,8 @@ func (obj *KVRes) UIDs() []engine.ResUID {
 	return []engine.ResUID{x}
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
-// It is primarily useful for setting the defaults.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
+// primarily useful for setting the defaults.
 func (obj *KVRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes KVRes // indirection to avoid infinite recursion
 

engine/resources/mount.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -33,13 +33,13 @@ import (
 	engineUtil "github.com/purpleidea/mgmt/engine/util"
 	"github.com/purpleidea/mgmt/recwatch"
 	"github.com/purpleidea/mgmt/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
 
 	sdbus "github.com/coreos/go-systemd/dbus"
 	"github.com/coreos/go-systemd/unit"
 	systemdUtil "github.com/coreos/go-systemd/util"
 	fstab "github.com/deniswernert/go-fstab"
 	"github.com/godbus/dbus"
-	errwrap "github.com/pkg/errors"
 	"golang.org/x/sys/unix"
 )
 
@@ -75,6 +75,8 @@ const (
 	// diskByLabel is the location of symlinks for partitions by label.
 	diskByPartLabel = devDisk + "by-partlabel/"
 
+	// dbusSystemdService is the service to connect to systemd itself.
+	dbusSystemd1Service = "org.freedesktop.systemd1"
 	// dbusSystemd1Interface is the base systemd1 path.
 	dbusSystemd1Path = "/org/freedesktop/systemd1"
 	// dbusUnitPath is the dbus path where mount unit files are found.
@@ -88,6 +90,9 @@ const (
 	dbusManagerInterface = dbusSystemd1Interface + ".Manager"
 	// dbusRestartUnit is the dbus method for restarting systemd units.
 	dbusRestartUnit = dbusManagerInterface + ".RestartUnit"
+	// dbusReloadSystemd is the dbus method for reloading systemd settings.
+	// (i.e. systemctl daemon-reload)
+	dbusReloadSystemd = dbusManagerInterface + ".Reload"
 	// restartTimeout is the delay before restartUnit is assumed to have
 	// failed.
 	dbusRestartCtxTimeout = 10
@@ -275,7 +280,7 @@ func (obj *MountRes) Watch() error {
 
 // fstabCheckApply checks /etc/fstab for entries corresponding to the resource
 // definition, and adds or deletes the entry as needed.
-func (obj *MountRes) fstabCheckApply(apply bool) (checkOK bool, err error) {
+func (obj *MountRes) fstabCheckApply(apply bool) (bool, error) {
 	exists, err := fstabEntryExists(fstabPath, obj.mount)
 	if err != nil {
 		return false, errwrap.Wrapf(err, "error checking if fstab entry exists")
@@ -339,8 +344,8 @@ func (obj *MountRes) mountCheckApply(apply bool) (bool, error) {
 // CheckApply is run to check the state and, if apply is true, to apply the
 // necessary changes to reach the desired state. This is run before Watch and
 // again if Watch finds a change occurring to the state.
-func (obj *MountRes) CheckApply(apply bool) (checkOK bool, err error) {
-	checkOK = true
+func (obj *MountRes) CheckApply(apply bool) (bool, error) {
+	checkOK := true
 
 	if c, err := obj.fstabCheckApply(apply); err != nil {
 		return false, err
@@ -398,8 +403,8 @@ func (obj *MountUID) IFF(uid engine.ResUID) bool {
 	return obj.name == res.name
 }
 
-// UIDs includes all params to make a unique identification of this object.
-// Most resources only return one although some resources can return multiple.
+// UIDs includes all params to make a unique identification of this object. Most
+// resources only return one although some resources can return multiple.
 func (obj *MountRes) UIDs() []engine.ResUID {
 	x := &MountUID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
@@ -408,8 +413,8 @@ func (obj *MountRes) UIDs() []engine.ResUID {
 	return []engine.ResUID{x}
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
-// It is primarily useful for setting the defaults.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
+// primarily useful for setting the defaults.
 func (obj *MountRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes MountRes // indirection to avoid infinite recursion
 
@@ -494,8 +499,8 @@ func (obj *MountRes) fstabEntryRemove(file string, mount *fstab.Mount) error {
 	return obj.fstabWrite(file, mounts)
 }
 
-// fstabWrite generates an fstab file with the given mounts, and writes them
-// to the provided fstab file.
+// fstabWrite generates an fstab file with the given mounts, and writes them to
+// the provided fstab file.
 func (obj *MountRes) fstabWrite(file string, mounts fstab.Mounts) error {
 	// build the file contents
 	contents := fmt.Sprintf("# Generated by %s at %d", obj.init.Program, time.Now().UnixNano()) + "\n"
@@ -536,9 +541,9 @@ func mountExists(file string, mount *fstab.Mount) (bool, error) {
 	return false, nil
 }
 
-// mountCompare compares two mounts. It is assumed that the first comes from
-// a resource definition, and the second comes from /proc/mounts. It compares
-// the two after resolving the loopback device's file path (if necessary,) and
+// mountCompare compares two mounts. It is assumed that the first comes from a
+// resource definition, and the second comes from /proc/mounts. It compares the
+// two after resolving the loopback device's file path (if necessary,) and
 // ignores freq and passno, as they may differ between the definition and
 // /proc/mounts.
 func mountCompare(def, proc *fstab.Mount) (bool, error) {
@@ -576,7 +581,10 @@ func mountReload() error {
 	}
 	defer conn.Close()
 	// systemctl daemon-reload
-	conn.BusObject().Call("Reload", 0)
+	call := conn.Object(dbusSystemd1Service, dbusSystemd1Path).Call(dbusReloadSystemd, 0)
+	if call.Err != nil {
+		return errwrap.Wrapf(call.Err, "error reloading systemd")
+	}
 
 	// systemctl restart local-fs.target
 	if err := restartUnit(conn, "local-fs.target"); err != nil {
@@ -584,15 +592,15 @@ func mountReload() error {
 	}
 
 	// systemctl restart remote-fs.target
-	if err := restartUnit(conn, "local-fs.target"); err != nil {
+	if err := restartUnit(conn, "remote-fs.target"); err != nil {
 		return errwrap.Wrapf(err, "error restarting unit")
 	}
 
 	return nil
 }
 
-// restartUnit restarts the given dbus unit and waits for it to finish
-// starting up. If restartTimeout is exceeded, it will return an error.
+// restartUnit restarts the given dbus unit and waits for it to finish starting
+// up. If restartTimeout is exceeded, it will return an error.
 func restartUnit(conn *dbus.Conn, unit string) error {
 	// timeout if we don't get the JobRemoved event
 	ctx, cancel := context.WithTimeout(context.TODO(), dbusRestartCtxTimeout*time.Second)
@@ -619,7 +627,7 @@ func restartUnit(conn *dbus.Conn, unit string) error {
 	defer conn.RemoveSignal(ch)
 
 	// restart the unit
-	sd1 := conn.Object(dbusSystemd1Interface, dbus.ObjectPath(dbusSystemd1Path))
+	sd1 := conn.Object(dbusSystemd1Service, dbus.ObjectPath(dbusSystemd1Path))
 	if call := sd1.Call(dbusRestartUnit, 0, unit, "fail"); call.Err != nil {
 		return errwrap.Wrapf(call.Err, "error restarting unit: %s", unit)
 	}

engine/resources/mount_linux_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/resources/mount_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/resources/msg.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -200,36 +200,28 @@ func (obj *MsgRes) CheckApply(apply bool) (bool, error) {
 
 // Cmp compares two resources and returns an error if they are not equivalent.
 func (obj *MsgRes) Cmp(r engine.Res) error {
-	if !obj.Compare(r) {
-		return fmt.Errorf("did not compare")
-	}
-	return nil
-}
-
-// Compare two resources and return if they are equivalent.
-func (obj *MsgRes) Compare(r engine.Res) bool {
 	// we can only compare MsgRes to others of the same resource kind
 	res, ok := r.(*MsgRes)
 	if !ok {
-		return false
+		return fmt.Errorf("not a %s", obj.Kind())
 	}
 
 	if obj.Body != res.Body {
-		return false
+		return fmt.Errorf("the Body differs")
 	}
 	if obj.Priority != res.Priority {
-		return false
+		return fmt.Errorf("the Priority differs")
 	}
 	if len(obj.Fields) != len(res.Fields) {
-		return false
+		return fmt.Errorf("the length of Fields differs")
 	}
 	for field, value := range obj.Fields {
 		if res.Fields[field] != value {
-			return false
+			return fmt.Errorf("the Fields differ")
 		}
 	}
 
-	return true
+	return nil
 }
 
 // MsgUID is a unique representation for a MsgRes object.
@@ -239,8 +231,8 @@ type MsgUID struct {
 	body string
 }
 
-// UIDs includes all params to make a unique identification of this object.
-// Most resources only return one, although some resources can return multiple.
+// UIDs includes all params to make a unique identification of this object. Most
+// resources only return one, although some resources can return multiple.
 func (obj *MsgRes) UIDs() []engine.ResUID {
 	x := &MsgUID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
@@ -249,8 +241,8 @@ func (obj *MsgRes) UIDs() []engine.ResUID {
 	return []engine.ResUID{x}
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
-// It is primarily useful for setting the defaults.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
+// primarily useful for setting the defaults.
 func (obj *MsgRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes MsgRes // indirection to avoid infinite recursion
 

engine/resources/msg_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/resources/net.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -34,10 +34,9 @@ import (
 	"github.com/purpleidea/mgmt/engine/traits"
 	"github.com/purpleidea/mgmt/recwatch"
 	"github.com/purpleidea/mgmt/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
 	"github.com/purpleidea/mgmt/util/socketset"
 
-	multierr "github.com/hashicorp/go-multierror"
-	errwrap "github.com/pkg/errors"
 	// XXX: Do NOT use subscribe methods from this lib, as they are racey and
 	// do not clean up spawned goroutines. Should be replaced when a suitable
 	// alternative is available.
@@ -82,17 +81,32 @@ const (
 	socketFile = "pipe.sock" // path in vardir to store our socket file
 )
 
-// NetRes is a network interface resource based on netlink. It manages the
-// state of a network link. Configuration is also stored in a networkd
-// configuration file, so the network is available upon reboot.
+// NetRes is a network interface resource based on netlink. It manages the state
+// of a network link. Configuration is also stored in a networkd configuration
+// file, so the network is available upon reboot. The name of the resource is
+// the string representing the network interface name. This could be "eth0" for
+// example.
 type NetRes struct {
 	traits.Base // add the base methods without re-implementation
 
 	init *engine.Init
 
-	State   string   `yaml:"state"`   // up, down, or empty
-	Addrs   []string `yaml:"addrs"`   // list of addresses in cidr format
-	Gateway string   `yaml:"gateway"` // gateway address
+	// State is the desired state of the interface. It can be "up", "down",
+	// or the empty string to leave that unspecified.
+	State string `lang:"state" yaml:"state"`
+
+	// Addrs is the list of addresses to set on the interface. They must
+	// each be in CIDR notation such as: 192.0.2.42/24 for example.
+	Addrs []string `lang:"addrs" yaml:"addrs"`
+
+	// Gateway represents the default route to set for the interface.
+	Gateway string `lang:"gateway" yaml:"gateway"`
+
+	// IPForward is a boolean that sets whether we should forward incoming
+	// packets onward when this is set. It default to unspecified, which
+	// downstream (in the systemd-networkd configuration) defaults to false.
+	// XXX: this could also be "ipv4" or "ipv6", add those as a second option?
+	IPForward *bool `lang:"ip_forward" yaml:"ip_forward"`
 
 	iface        *iface // a struct containing the net.Interface and netlink.Link
 	unitFilePath string // the interface unit file path
@@ -100,8 +114,8 @@ type NetRes struct {
 	socketFile string // path for storing the pipe socket file
 }
 
-// nlChanStruct defines the channel used to send netlink messages and errors
-// to the event processing loop in Watch.
+// nlChanStruct defines the channel used to send netlink messages and errors to
+// the event processing loop in Watch.
 type nlChanStruct struct {
 	msg []syscall.NetlinkMessage
 	err error
@@ -179,9 +193,7 @@ func (obj *NetRes) Close() error {
 		return fmt.Errorf("socket file should not be the root path")
 	}
 	if obj.socketFile != "" { // safety
-		if err := os.Remove(obj.socketFile); err != nil {
-			errList = multierr.Append(errList, err)
-		}
+		errList = errwrap.Append(errList, os.Remove(obj.socketFile))
 	}
 
 	return errList
@@ -374,8 +386,8 @@ func (obj *NetRes) addrCheckApply(apply bool) (bool, error) {
 	return false, nil
 }
 
-// gatewayCheckApply checks if the interface has the correct default gateway
-// and adds/deletes routes as necessary.
+// gatewayCheckApply checks if the interface has the correct default gateway and
+// adds/deletes routes as necessary.
 func (obj *NetRes) gatewayCheckApply(apply bool) (bool, error) {
 	// get all routes from the interface
 	routes, err := netlink.RouteList(obj.iface.link, netlink.FAMILY_V4)
@@ -463,8 +475,8 @@ func (obj *NetRes) fileCheckApply(apply bool) (bool, error) {
 // CheckApply is run to check the state and, if apply is true, to apply the
 // necessary changes to reach the desired state. This is run before Watch and
 // again if Watch finds a change occurring to the state.
-func (obj *NetRes) CheckApply(apply bool) (checkOK bool, err error) {
-	checkOK = true
+func (obj *NetRes) CheckApply(apply bool) (bool, error) {
+	checkOK := true
 
 	// check the network device
 	if c, err := obj.ifaceCheckApply(apply); err != nil {
@@ -509,34 +521,26 @@ func (obj *NetRes) CheckApply(apply bool) (checkOK bool, err error) {
 
 // Cmp compares two resources and returns an error if they are not equivalent.
 func (obj *NetRes) Cmp(r engine.Res) error {
-	if !obj.Compare(r) {
-		return fmt.Errorf("did not compare")
-	}
-	return nil
-}
-
-// Compare two resources and return if they are equivalent.
-func (obj *NetRes) Compare(r engine.Res) bool {
 	// we can only compare NetRes to others of the same resource kind
 	res, ok := r.(*NetRes)
 	if !ok {
-		return false
+		return fmt.Errorf("not a %s", obj.Kind())
 	}
 
 	if obj.State != res.State {
-		return false
+		return fmt.Errorf("the State differs")
 	}
 	if (obj.Addrs == nil) != (res.Addrs == nil) {
-		return false
+		return fmt.Errorf("the Addrs differ")
 	}
 	if err := util.SortedStrSliceCompare(obj.Addrs, res.Addrs); err != nil {
-		return false
+		return fmt.Errorf("the Addrs differ")
 	}
 	if obj.Gateway != res.Gateway {
-		return false
+		return fmt.Errorf("the Gateway differs")
 	}
 
-	return true
+	return nil
 }
 
 // NetUID is a unique resource identifier.
@@ -559,8 +563,8 @@ func (obj *NetUID) IFF(uid engine.ResUID) bool {
 	return obj.name == res.name
 }
 
-// UIDs includes all params to make a unique identification of this object.
-// Most resources only return one although some resources can return multiple.
+// UIDs includes all params to make a unique identification of this object. Most
+// resources only return one although some resources can return multiple.
 func (obj *NetRes) UIDs() []engine.ResUID {
 	x := &NetUID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
@@ -569,8 +573,8 @@ func (obj *NetRes) UIDs() []engine.ResUID {
 	return []engine.ResUID{x}
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
-// It is primarily useful for setting the defaults.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
+// primarily useful for setting the defaults.
 func (obj *NetRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes NetRes // indirection to avoid infinite recursion
 
@@ -601,6 +605,13 @@ func (obj *NetRes) unitFileContents() []byte {
 	if obj.Gateway != "" {
 		u = append(u, fmt.Sprintf("Gateway=%s", obj.Gateway))
 	}
+	if obj.IPForward != nil {
+		b := "false"
+		if *obj.IPForward {
+			b = "true"
+		}
+		u = append(u, fmt.Sprintf("IPForward=%s", b))
+	}
 	c := strings.Join(u, "\n")
 	return []byte(c)
 }
@@ -636,8 +647,8 @@ func (obj *iface) linkUpDown(state string) error {
 	return netlink.LinkSetDown(obj.link)
 }
 
-// getAddrs returns a list of strings containing all of the interface's
-// IP addresses in CIDR format.
+// getAddrs returns a list of strings containing all of the interface's IP
+// addresses in CIDR format.
 func (obj *iface) getAddrs() ([]string, error) {
 	var ifaceAddrs []string
 	a, err := obj.iface.Addrs()
@@ -705,8 +716,8 @@ func (obj *iface) kernelApply(addrs []string) error {
 	return nil
 }
 
-// addrApplyDelete, checks the interface's addresses and deletes any that are not
-// in the list/definition.
+// addrApplyDelete, checks the interface's addresses and deletes any that are
+// not in the list/definition.
 func (obj *iface) addrApplyDelete(objAddrs []string) error {
 	ifaceAddrs, err := obj.getAddrs()
 	if err != nil {

engine/resources/net_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/resources/noop.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -75,7 +75,7 @@ func (obj *NoopRes) Watch() error {
 }
 
 // CheckApply method for Noop resource. Does nothing, returns happy!
-func (obj *NoopRes) CheckApply(apply bool) (checkOK bool, err error) {
+func (obj *NoopRes) CheckApply(apply bool) (bool, error) {
 	if obj.init.Refresh() {
 		obj.init.Logf("received a notification!")
 	}
@@ -103,8 +103,8 @@ type NoopUID struct {
 	name string
 }
 
-// UIDs includes all params to make a unique identification of this object.
-// Most resources only return one, although some resources can return multiple.
+// UIDs includes all params to make a unique identification of this object. Most
+// resources only return one, although some resources can return multiple.
 func (obj *NoopRes) UIDs() []engine.ResUID {
 	x := &NoopUID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
@@ -126,8 +126,8 @@ func (obj *NoopRes) GroupCmp(r engine.GroupableRes) error {
 	return nil // noop resources can always be grouped together!
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
-// It is primarily useful for setting the defaults.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
+// primarily useful for setting the defaults.
 func (obj *NoopRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes NoopRes // indirection to avoid infinite recursion
 

engine/resources/noop_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/resources/nspawn.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -21,18 +21,19 @@ import (
 	"errors"
 	"fmt"
 	"strconv"
+	"strings"
 	"unicode"
 
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/engine/traits"
 	engineUtil "github.com/purpleidea/mgmt/engine/util"
 	"github.com/purpleidea/mgmt/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
 
 	systemdDbus "github.com/coreos/go-systemd/dbus"
 	machined "github.com/coreos/go-systemd/machine1"
 	systemdUtil "github.com/coreos/go-systemd/util"
 	"github.com/godbus/dbus"
-	errwrap "github.com/pkg/errors"
 )
 
 const (
@@ -72,8 +73,8 @@ func (obj *NspawnRes) Default() engine.Res {
 	}
 }
 
-// makeComposite creates a pointer to a SvcRes. The pointer is used to
-// validate and initialize the nested svc.
+// makeComposite creates a pointer to a SvcRes. The pointer is used to validate
+// and initialize the nested svc.
 func (obj *NspawnRes) makeComposite() (*SvcRes, error) {
 	res, err := engine.NewNamedResource("svc", fmt.Sprintf(nspawnServiceTmpl, obj.Name()))
 	if err != nil {
@@ -112,7 +113,7 @@ func (obj *NspawnRes) Validate() error {
 		return errwrap.Wrapf(err, "makeComposite failed in validate")
 	}
 	if err := svc.Validate(); err != nil { // composite resource
-		return errwrap.Wrapf(err, "validate failed for embedded svc: %s", obj.svc)
+		return errwrap.Wrapf(err, "validate failed for embedded svc: %s", svc)
 	}
 	return nil
 }
@@ -127,10 +128,7 @@ func (obj *NspawnRes) Init(init *engine.Init) error {
 	}
 	obj.svc = svc
 	// TODO: we could build a new init that adds a prefix to the logger...
-	if err := obj.svc.Init(init); err != nil {
-		return err
-	}
-	return nil
+	return obj.svc.Init(init)
 }
 
 // Close is run by the engine to clean up after the resource is done.
@@ -201,7 +199,7 @@ func (obj *NspawnRes) Watch() error {
 // CheckApply is run to check the state and, if apply is true, to apply the
 // necessary changes to reach the desired state. This is run before Watch and
 // again if Watch finds a change occurring to the state.
-func (obj *NspawnRes) CheckApply(apply bool) (checkOK bool, err error) {
+func (obj *NspawnRes) CheckApply(apply bool) (bool, error) {
 	// this resource depends on systemd to ensure that it's running
 	if !systemdUtil.IsRunningSystemd() {
 		return false, errors.New("systemd is not running")
@@ -260,35 +258,27 @@ func (obj *NspawnRes) CheckApply(apply bool) (checkOK bool, err error) {
 
 // Cmp compares two resources and returns an error if they are not equivalent.
 func (obj *NspawnRes) Cmp(r engine.Res) error {
-	if !obj.Compare(r) {
-		return fmt.Errorf("did not compare")
-	}
-	return nil
-}
-
-// Compare two resources and return if they are equivalent.
-func (obj *NspawnRes) Compare(r engine.Res) bool {
 	// we can only compare NspawnRes to others of the same resource kind
 	res, ok := r.(*NspawnRes)
 	if !ok {
-		return false
+		return fmt.Errorf("not a %s", obj.Kind())
 	}
 
 	if obj.State != res.State {
-		return false
+		return fmt.Errorf("the State differs")
 	}
 
 	// TODO: why is res.svc ever nil?
 	if (obj.svc == nil) != (res.svc == nil) { // xor
-		return false
+		return fmt.Errorf("the svc differs")
 	}
 	if obj.svc != nil && res.svc != nil {
-		if !obj.svc.Compare(res.svc) {
-			return false
+		if err := obj.svc.Cmp(res.svc); err != nil {
+			return errwrap.Wrapf(err, "the svc differs")
 		}
 	}
 
-	return true
+	return nil
 }
 
 // NspawnUID is a unique resource identifier.
@@ -311,8 +301,8 @@ func (obj *NspawnUID) IFF(uid engine.ResUID) bool {
 	return obj.name == res.name
 }
 
-// UIDs includes all params to make a unique identification of this object.
-// Most resources only return one although some resources can return multiple.
+// UIDs includes all params to make a unique identification of this object. Most
+// resources only return one although some resources can return multiple.
 func (obj *NspawnRes) UIDs() []engine.ResUID {
 	x := &NspawnUID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
@@ -321,8 +311,8 @@ func (obj *NspawnRes) UIDs() []engine.ResUID {
 	return append([]engine.ResUID{x}, obj.svc.UIDs()...)
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
-// It is primarily useful for setting the defaults.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
+// primarily useful for setting the defaults.
 func (obj *NspawnRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes NspawnRes // indirection to avoid infinite recursion
 
@@ -358,10 +348,12 @@ func systemdVersion() (uint16, error) {
 		return 0, errwrap.Wrapf(err, "could not get version property")
 	}
 	// lose the surrounding quotes
-	verNum, err := strconv.Unquote(verString)
+	verNumString, err := strconv.Unquote(verString)
 	if err != nil {
 		return 0, errwrap.Wrapf(err, "error unquoting version number")
 	}
+	// trim possible version suffix like in "242.19-1"
+	verNum := strings.Split(verNumString, ".")[0]
 	// cast to uint16
 	ver, err := strconv.ParseUint(verNum, 10, 16)
 	if err != nil {

engine/resources/packagekit/packagekit.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -27,10 +27,9 @@ import (
 
 	engineUtil "github.com/purpleidea/mgmt/engine/util"
 	"github.com/purpleidea/mgmt/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
 
 	"github.com/godbus/dbus"
-	multierr "github.com/hashicorp/go-multierror"
-	errwrap "github.com/pkg/errors"
 )
 
 // global tweaks of verbosity and code path
@@ -38,7 +37,8 @@ const (
 	Paranoid = false // enable if you see any ghosts
 )
 
-// constants which might need to be tweaked or which contain special dbus strings.
+// constants which might need to be tweaked or which contain special dbus
+// strings.
 const (
 	// FIXME: if PkBufferSize is too low, install seems to drop signals
 	PkBufferSize = 1000
@@ -72,7 +72,7 @@ var (
 	}
 )
 
-//type enum_filter uint64
+// type enum_filter uint64
 // https://github.com/hughsie/PackageKit/blob/master/lib/packagekit-glib2/pk-enum.c
 const ( //static const PkEnumMatch enum_filter[]
 	PkFilterEnumUnknown        uint64 = 1 << iota // "unknown"
@@ -155,7 +155,8 @@ type Conn struct {
 	Logf  func(format string, v ...interface{})
 }
 
-// PkPackageIDActionData is a struct that is returned by PackagesToPackageIDs in the map values.
+// PkPackageIDActionData is a struct that is returned by PackagesToPackageIDs in
+// the map values.
 type PkPackageIDActionData struct {
 	Found     bool
 	Installed bool
@@ -186,7 +187,8 @@ func (obj *Conn) Close() error {
 	return obj.conn.Close()
 }
 
-// internal helper to add signal matches to the bus, should only be called once
+// matchSignal is an internal helper to add signal matches to the bus. It should
+// only be called once.
 func (obj *Conn) matchSignal(ch chan *dbus.Signal, path dbus.ObjectPath, iface string, signals []string) (func() error, error) {
 	if obj.Debug {
 		obj.Logf("matchSignal(%v, %v, %s, %v)", ch, path, iface, signals)
@@ -198,9 +200,8 @@ func (obj *Conn) matchSignal(ch chan *dbus.Signal, path dbus.ObjectPath, iface s
 	removeSignals := func() error {
 		var errList error
 		for i := len(argsList) - 1; i >= 0; i-- { // last in first out
-			if call := bus.Call(engineUtil.DBusRemoveMatch, 0, argsList[i]); call.Err != nil {
-				errList = multierr.Append(errList, call.Err)
-			}
+			call := bus.Call(engineUtil.DBusRemoveMatch, 0, argsList[i])
+			errList = errwrap.Append(errList, call.Err)
 		}
 		return errList
 	}
@@ -354,7 +355,7 @@ loop:
 				// should already be broken
 				break loop
 			} else {
-				return []string{}, fmt.Errorf("PackageKit: Error: %v", signal.Body)
+				return []string{}, fmt.Errorf("error in body: %v", signal.Body)
 			}
 		}
 	}
@@ -365,9 +366,9 @@ loop:
 func (obj *Conn) IsInstalledList(packages []string) ([]bool, error) {
 	var filter uint64          // initializes at the "zero" value of 0
 	filter += PkFilterEnumArch // always search in our arch
-	packageIDs, e := obj.ResolvePackages(packages, filter)
-	if e != nil {
-		return nil, fmt.Errorf("ResolvePackages error: %v", e)
+	packageIDs, err := obj.ResolvePackages(packages, filter)
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "error resolving packages")
 	}
 
 	var m = make(map[string]int)
@@ -445,7 +446,7 @@ loop:
 			}
 
 			if signal.Name == FmtTransactionMethod("ErrorCode") {
-				return fmt.Errorf("PackageKit: Error: %v", signal.Body)
+				return fmt.Errorf("error in body: %v", signal.Body)
 			} else if signal.Name == FmtTransactionMethod("Package") {
 				// a package was installed...
 				// only start the timer once we're here...
@@ -456,14 +457,14 @@ loop:
 			} else if signal.Name == FmtTransactionMethod("Destroy") {
 				return nil // success
 			} else {
-				return fmt.Errorf("PackageKit: Error: %v", signal.Body)
+				return fmt.Errorf("error in body: %v", signal.Body)
 			}
 		case <-util.TimeAfterOrBlock(timeout):
 			if finished {
 				obj.Logf("Timeout: InstallPackages: Waiting for 'Destroy'")
 				return nil // got tired of waiting for Destroy
 			}
-			return fmt.Errorf("PackageKit: Timeout: InstallPackages: %s", strings.Join(packageIDs, ", "))
+			return fmt.Errorf("timeout installing packages: %s", strings.Join(packageIDs, ", "))
 		}
 	}
 }
@@ -502,7 +503,7 @@ loop:
 			}
 
 			if signal.Name == FmtTransactionMethod("ErrorCode") {
-				return fmt.Errorf("PackageKit: Error: %v", signal.Body)
+				return fmt.Errorf("error in body: %v", signal.Body)
 			} else if signal.Name == FmtTransactionMethod("Package") {
 				// a package was installed...
 				continue loop
@@ -513,7 +514,7 @@ loop:
 				// should already be broken
 				break loop
 			} else {
-				return fmt.Errorf("PackageKit: Error: %v", signal.Body)
+				return fmt.Errorf("error in body: %v", signal.Body)
 			}
 		}
 	}
@@ -551,7 +552,7 @@ loop:
 			}
 
 			if signal.Name == FmtTransactionMethod("ErrorCode") {
-				return fmt.Errorf("PackageKit: Error: %v", signal.Body)
+				return fmt.Errorf("error in body: %v", signal.Body)
 			} else if signal.Name == FmtTransactionMethod("Package") {
 			} else if signal.Name == FmtTransactionMethod("Finished") {
 				// TODO: should we wait for the Destroy signal?
@@ -560,14 +561,15 @@ loop:
 				// should already be broken
 				break loop
 			} else {
-				return fmt.Errorf("PackageKit: Error: %v", signal.Body)
+				return fmt.Errorf("error in body: %v", signal.Body)
 			}
 		}
 	}
 	return nil
 }
 
-// GetFilesByPackageID gets the list of files that are contained inside a list of packageIDs.
+// GetFilesByPackageID gets the list of files that are contained inside a list
+// of packageIDs.
 func (obj *Conn) GetFilesByPackageID(packageIDs []string) (files map[string][]string, err error) {
 	// NOTE: the maximum number of files in an RPM is 52116 in Fedora 23
 	// https://gist.github.com/purpleidea/b98e60dcd449e1ac3b8a
@@ -603,7 +605,7 @@ loop:
 			}
 
 			if signal.Name == FmtTransactionMethod("ErrorCode") {
-				err = fmt.Errorf("PackageKit: Error: %v", signal.Body)
+				err = fmt.Errorf("error in body: %v", signal.Body)
 				return
 
 				// one signal returned per packageID found...
@@ -628,7 +630,7 @@ loop:
 				// should already be broken
 				break loop
 			} else {
-				err = fmt.Errorf("PackageKit: Error: %v", signal.Body)
+				err = fmt.Errorf("error in body: %v", signal.Body)
 				return
 			}
 		}
@@ -636,7 +638,8 @@ loop:
 	return
 }
 
-// GetUpdates gets a list of packages that are installed and which can be updated, mod filter.
+// GetUpdates gets a list of packages that are installed and which can be
+// updated, mod filter.
 func (obj *Conn) GetUpdates(filter uint64) ([]string, error) {
 	if obj.Debug {
 		obj.Logf("GetUpdates()")
@@ -671,7 +674,7 @@ loop:
 			}
 
 			if signal.Name == FmtTransactionMethod("ErrorCode") {
-				return nil, fmt.Errorf("PackageKit: Error: %v", signal.Body)
+				return nil, fmt.Errorf("error in body: %v", signal.Body)
 			} else if signal.Name == FmtTransactionMethod("Package") {
 
 				//pkg_int, ok := signal.Body[0].(int)
@@ -694,7 +697,7 @@ loop:
 				// should already be broken
 				break loop
 			} else {
-				return nil, fmt.Errorf("PackageKit: Error: %v", signal.Body)
+				return nil, fmt.Errorf("error in body: %v", signal.Body)
 			}
 		}
 	}
@@ -720,9 +723,9 @@ func (obj *Conn) PackagesToPackageIDs(packageMap map[string]string, filter uint6
 	if obj.Debug {
 		obj.Logf("PackagesToPackageIDs(): %s", strings.Join(packages, ", "))
 	}
-	resolved, e := obj.ResolvePackages(packages, filter)
-	if e != nil {
-		return nil, fmt.Errorf("Resolve error: %v", e)
+	resolved, err := obj.ResolvePackages(packages, filter)
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "error resolving")
 	}
 
 	found := make([]bool, count) // default false
@@ -760,7 +763,7 @@ func (obj *Conn) PackagesToPackageIDs(packageMap map[string]string, filter uint6
 		}
 		state := packageMap[pkg] // lookup the requested state/version
 		if state == "" {
-			return nil, fmt.Errorf("Empty package state for %v", pkg)
+			return nil, fmt.Errorf("empty package state for: `%s`", pkg)
 		}
 		found[index] = true
 		stateIsVersion := (state != "installed" && state != "uninstalled" && state != "newest") // must be a ver. string
@@ -796,9 +799,9 @@ func (obj *Conn) PackagesToPackageIDs(packageMap map[string]string, filter uint6
 	// to be done, and if so, anything that needs updating isn't newest!
 	// if something isn't installed, we can't verify it with this method
 	// FIXME: https://github.com/hughsie/PackageKit/issues/116
-	updates, e := obj.GetUpdates(filter)
-	if e != nil {
-		return nil, fmt.Errorf("Updates error: %v", e)
+	updates, err := obj.GetUpdates(filter)
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "updates error")
 	}
 	for _, packageID := range updates {
 		//obj.Logf("* %v", packageID)
@@ -846,9 +849,9 @@ func (obj *Conn) PackagesToPackageIDs(packageMap map[string]string, filter uint6
 			if obj.Debug {
 				obj.Logf("PackagesToPackageIDs(): Recurse: %s", strings.Join(checkPackages, ", "))
 			}
-			recursion, e = obj.PackagesToPackageIDs(filteredPackageMap, filter+PkFilterEnumNewest)
-			if e != nil {
-				return nil, fmt.Errorf("Recursion error: %v", e)
+			recursion, err = obj.PackagesToPackageIDs(filteredPackageMap, filter+PkFilterEnumNewest)
+			if err != nil {
+				return nil, errwrap.Wrapf(err, "recursion error")
 			}
 		}
 	}
@@ -878,7 +881,8 @@ func (obj *Conn) PackagesToPackageIDs(packageMap map[string]string, filter uint6
 	return result, nil
 }
 
-// FilterPackageIDs returns a list of packageIDs which match the set of package names in packages.
+// FilterPackageIDs returns a list of packageIDs which match the set of package
+// names in packages.
 func FilterPackageIDs(m map[string]*PkPackageIDActionData, packages []string) ([]string, error) {
 	result := []string{}
 	for _, k := range packages {
@@ -892,7 +896,8 @@ func FilterPackageIDs(m map[string]*PkPackageIDActionData, packages []string) ([
 	return result, nil
 }
 
-// FilterState returns a map of whether each package queried matches the particular state.
+// FilterState returns a map of whether each package queried matches the
+// particular state.
 func FilterState(m map[string]*PkPackageIDActionData, packages []string, state string) (result map[string]bool, err error) {
 	result = make(map[string]bool)
 	pkgs := []string{} // bad pkgs that don't have a bool state
@@ -922,7 +927,8 @@ func FilterState(m map[string]*PkPackageIDActionData, packages []string, state s
 	return result, err
 }
 
-// FilterPackageState returns all packages that are in package and match the specific state.
+// FilterPackageState returns all packages that are in package and match the
+// specific state.
 func FilterPackageState(m map[string]*PkPackageIDActionData, packages []string, state string) (result []string, err error) {
 	result = []string{}
 	for _, k := range packages {
@@ -948,7 +954,8 @@ func FilterPackageState(m map[string]*PkPackageIDActionData, packages []string,
 	return result, err
 }
 
-// FlagInData asks whether a flag exists inside the data portion of a packageID field?
+// FlagInData asks whether a flag exists inside the data portion of a packageID
+// field?
 func FlagInData(flag, data string) bool {
 	flags := strings.Split(data, ":")
 	for _, f := range flags {

engine/resources/password.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -29,8 +29,7 @@ import (
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/engine/traits"
 	"github.com/purpleidea/mgmt/recwatch"
-
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 func init() {
@@ -210,7 +209,7 @@ func (obj *PasswordRes) Watch() error {
 }
 
 // CheckApply method for Password resource. Does nothing, returns happy!
-func (obj *PasswordRes) CheckApply(apply bool) (checkOK bool, err error) {
+func (obj *PasswordRes) CheckApply(apply bool) (bool, error) {
 	var refresh = obj.init.Refresh() // do we have a pending reload to apply?
 	var exists = true                // does the file (aka the token) exist?
 	var generate bool                // do we need to generate a new password?
@@ -296,33 +295,25 @@ func (obj *PasswordRes) CheckApply(apply bool) (checkOK bool, err error) {
 
 // Cmp compares two resources and returns an error if they are not equivalent.
 func (obj *PasswordRes) Cmp(r engine.Res) error {
-	if !obj.Compare(r) {
-		return fmt.Errorf("did not compare")
-	}
-	return nil
-}
-
-// Compare two resources and return if they are equivalent.
-func (obj *PasswordRes) Compare(r engine.Res) bool {
 	// we can only compare PasswordRes to others of the same resource kind
 	res, ok := r.(*PasswordRes)
 	if !ok {
-		return false
+		return fmt.Errorf("not a %s", obj.Kind())
 	}
 
 	if obj.Length != res.Length {
-		return false
+		return fmt.Errorf("the Length differs")
 	}
 	// TODO: we *could* optimize by allowing CheckApply to move from
 	// saved->!saved, by removing the file, but not likely worth it!
 	if obj.Saved != res.Saved {
-		return false
+		return fmt.Errorf("the Saved differs")
 	}
 	if obj.CheckRecovery != res.CheckRecovery {
-		return false
+		return fmt.Errorf("the CheckRecovery differs")
 	}
 
-	return true
+	return nil
 }
 
 // PasswordUID is the UID struct for PasswordRes.
@@ -331,8 +322,8 @@ type PasswordUID struct {
 	name string
 }
 
-// UIDs includes all params to make a unique identification of this object.
-// Most resources only return one, although some resources can return multiple.
+// UIDs includes all params to make a unique identification of this object. Most
+// resources only return one, although some resources can return multiple.
 func (obj *PasswordRes) UIDs() []engine.ResUID {
 	x := &PasswordUID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
@@ -344,7 +335,7 @@ func (obj *PasswordRes) UIDs() []engine.ResUID {
 // PasswordSends is the struct of data which is sent after a successful Apply.
 type PasswordSends struct {
 	// Password is the generated password being sent.
-	Password *string
+	Password *string `lang:"password"`
 	// Hashing is the algorithm used for this password. Empty is plain text.
 	Hashing string // TODO: implement me
 }
@@ -356,8 +347,8 @@ func (obj *PasswordRes) Sends() interface{} {
 	}
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
-// It is primarily useful for setting the defaults.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
+// primarily useful for setting the defaults.
 func (obj *PasswordRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes PasswordRes // indirection to avoid infinite recursion
 

engine/resources/pippet.go

@@ -0,0 +1,329 @@
+// Mgmt
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os/exec"
+	"sync"
+
+	"github.com/purpleidea/mgmt/engine"
+	"github.com/purpleidea/mgmt/engine/traits"
+	"github.com/purpleidea/mgmt/util/errwrap"
+)
+
+var pippetReceiverInstance *pippetReceiver
+var pippetReceiverOnce sync.Once
+
+func init() {
+	engine.RegisterResource("pippet", func() engine.Res { return &PippetRes{} })
+}
+
+// PippetRes is a wrapper resource for puppet. It implements the functional
+// equivalent of an exec resource that calls "puppet resource <type> <title>
+// <params>", but offers superior performance through a long-running Puppet
+// process that receives resources through a pipe (hence the name).
+type PippetRes struct {
+	traits.Base // add the base methods without re-implementation
+	traits.Refreshable
+
+	init *engine.Init
+
+	// Type is the exact name of the wrapped Puppet resource type, e.g.
+	// "file", "mount". This needs not be a core type. It can be a type
+	// from a module. The Puppet installation local to the mgmt agent
+	// machine must be able recognize it. It has to be a native type though,
+	// as opposed to defined types from your Puppet manifest code.
+	Type string `yaml:"type" json:"type"`
+	// Title is used by Puppet as the resource title. Puppet will often
+	// assign special meaning to the title, e.g. use it as the path for a
+	// file resource, or the name of a package.
+	Title string `yaml:"title" json:"title"`
+	// Params is expected to be a hash in YAML format, pairing resource
+	// parameter names with their respective values, e.g. { ensure: present
+	// }
+	Params string `yaml:"params" json:"params"`
+
+	runner *pippetReceiver
+}
+
+// Default returns an example Pippet resource.
+func (obj *PippetRes) Default() engine.Res {
+	return &PippetRes{
+		Params: "{}", // use an empty params hash per default
+	}
+}
+
+// Validate never errors out. We don't know the set of potential types that
+// Puppet supports. Resource names are arbitrary. We cannot really validate the
+// parameter YAML, because we cannot assume that it can be unmarshalled into a
+// map[string]string; Puppet supports complex parameter values.
+func (obj *PippetRes) Validate() error {
+	return nil
+}
+
+// Init makes sure that the PippetReceiver object is initialized.
+func (obj *PippetRes) Init(init *engine.Init) error {
+	obj.init = init // save for later
+	obj.runner = getPippetReceiverInstance()
+	return obj.runner.Register()
+}
+
+// Close is run by the engine to clean up after the resource is done.
+func (obj *PippetRes) Close() error {
+	return obj.runner.Unregister()
+}
+
+// Watch is the primary listener for this resource and it outputs events.
+func (obj *PippetRes) Watch() error {
+	obj.init.Running() // when started, notify engine that we're running
+
+	select {
+	case <-obj.init.Done: // closed by the engine to signal shutdown
+	}
+
+	//obj.init.Event() // notify engine of an event (this can block)
+
+	return nil
+}
+
+// CheckApply synchronizes the resource if required.
+func (obj *PippetRes) CheckApply(apply bool) (bool, error) {
+	changed, err := applyPippetRes(obj.runner, obj)
+	if err != nil {
+		return false, fmt.Errorf("pippet: %s[%s]: ERROR - %v", obj.Type, obj.Title, err)
+	}
+	return !changed, nil
+}
+
+// Cmp compares two resources and returns an error if they are not equivalent.
+func (obj *PippetRes) Cmp(r engine.Res) error {
+	res, ok := r.(*PippetRes)
+	if !ok {
+		return fmt.Errorf("not a %s", obj.Kind())
+	}
+
+	if obj.Type != res.Type {
+		return fmt.Errorf("the Type param differs")
+	}
+
+	if obj.Title != res.Title {
+		return fmt.Errorf("the Title param differs")
+	}
+
+	// FIXME: This is a lie. Parameter lists can be equivalent but not
+	// lexically identical (e.g. whitespace differences, parameter order).
+	// This is difficult to handle because we cannot casually unmarshall the
+	// YAML content.
+	if obj.Params != res.Params {
+		return fmt.Errorf("the Param param differs")
+	}
+
+	return nil
+}
+
+// PippetUID is the UID struct for PippetRes.
+type PippetUID struct {
+	engine.BaseUID
+	resourceType  string
+	resourceTitle string
+}
+
+// UIDs includes all params to make a unique identification of this object. Most
+// resources only return one, although some resources can return multiple.
+func (obj *PippetRes) UIDs() []engine.ResUID {
+	x := &PippetUID{
+		BaseUID:       engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
+		resourceType:  obj.Type,
+		resourceTitle: obj.Title,
+	}
+	return []engine.ResUID{x}
+}
+
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
+// primarily useful for setting the defaults.
+func (obj *PippetRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type rawRes PippetRes // indirection to avoid infinite recursion
+
+	def := obj.Default()        // get the default
+	res, ok := def.(*PippetRes) // put in the right format
+	if !ok {
+		return fmt.Errorf("could not convert to PippetRes")
+	}
+	raw := rawRes(*res) // convert; the defaults go here
+
+	if err := unmarshal(&raw); err != nil {
+		return err
+	}
+
+	*obj = PippetRes(raw) // restore from indirection with type conversion!
+	return nil
+}
+
+// PippetRunner is the interface used to communicate with the PippetReceiver
+// object. Its main purpose is dependency injection.
+type PippetRunner interface {
+	LockApply()
+	UnlockApply()
+	InputStream() io.WriteCloser
+	OutputStream() io.ReadCloser
+}
+
+// PippetResult is the structured return value type for the PippetReceiver's
+// Apply function.
+type PippetResult struct {
+	Error     bool
+	Failed    bool
+	Changed   bool
+	Exception string
+}
+
+// GetPippetReceiverInstance returns a pointer to the PippetReceiver object. The
+// PippetReceiver is supposed to be a singleton object. The pippet resource code
+// should always use the PippetReceiverInstance function to gain access to the
+// pippetReceiver object. Other objects of type pippetReceiver should not be
+// created.
+func getPippetReceiverInstance() *pippetReceiver {
+	for pippetReceiverInstance == nil {
+		pippetReceiverOnce.Do(func() { pippetReceiverInstance = &pippetReceiver{} })
+	}
+	return pippetReceiverInstance
+}
+
+type pippetReceiver struct {
+	stdin         io.WriteCloser
+	stdout        io.ReadCloser
+	registerMutex sync.Mutex
+	applyMutex    sync.Mutex
+	registered    int
+}
+
+// Init runs the Puppet process that will perform the work of synchronizing
+// resources that are sent to its stdin. The process will keep running until
+// Close is called. Init should not be called directly. It is implicitly called
+// by the Register function.
+func (obj *pippetReceiver) Init() error {
+	cmd := exec.Command("puppet", "yamlresource", "receive", "--color=no")
+	var err error
+	obj.stdin, err = cmd.StdinPipe()
+	if err != nil {
+		return err
+	}
+	obj.stdout, err = cmd.StdoutPipe()
+	if err != nil {
+		return errwrap.Append(err, obj.stdin.Close())
+	}
+	if err = cmd.Start(); err != nil {
+		return errwrap.Append(err, obj.stdin.Close())
+	}
+	buf := make([]byte, 80)
+	if _, err = obj.stdout.Read(buf); err != nil {
+		return errwrap.Append(err, obj.stdin.Close())
+	}
+	return nil
+}
+
+// Register should be called by any user (i.e., any pippet resource) before
+// using the PippetRunner functions on this receiver object. Register implicitly
+// takes care of calling Init if required.
+func (obj *pippetReceiver) Register() error {
+	obj.registerMutex.Lock()
+	defer obj.registerMutex.Unlock()
+	obj.registered = obj.registered + 1
+	if obj.registered > 1 {
+		return nil
+	}
+	// count was increased from 0 to 1, we need to (re-)init
+	var err error
+	if err = obj.Init(); err != nil {
+		obj.registered = 0
+	}
+	return err
+}
+
+// Unregister should be called by any object that registered itself using the
+// Register function, and which no longer needs the receiver. This should
+// typically happen at closing time of the pippet resource that registered
+// itself. Unregister implicitly calls Close in case all registered resources
+// have unregistered.
+func (obj *pippetReceiver) Unregister() error {
+	obj.registerMutex.Lock()
+	defer obj.registerMutex.Unlock()
+	obj.registered = obj.registered - 1
+	if obj.registered == 0 {
+		return obj.Close()
+	}
+	if obj.registered < 0 {
+		return fmt.Errorf("pippet runner: ERROR: unregistered more resources than were registered")
+	}
+	return nil
+}
+
+// LockApply locks the pippetReceiver's mutex for an "Apply" transaction.
+func (obj *pippetReceiver) LockApply() {
+	obj.applyMutex.Lock()
+}
+
+// UnlockApply unlocks the pippetReceiver's mutex for an "Apply" transaction.
+func (obj *pippetReceiver) UnlockApply() {
+	obj.applyMutex.Unlock()
+}
+
+// InputStream returns the pippetReceiver's pipe writer.
+func (obj *pippetReceiver) InputStream() io.WriteCloser {
+	return obj.stdin
+}
+
+// OutputStream returns the pippetReceiver's pipe reader.
+func (obj *pippetReceiver) OutputStream() io.ReadCloser {
+	return obj.stdout
+}
+
+// Close stops the backend puppet process by closing its stdin handle. It should
+// not be called directly. It is implicitly called by the Unregister function if
+// appropriate.
+func (obj *pippetReceiver) Close() error {
+	return obj.stdin.Close()
+}
+
+// applyPippetRes does the actual work of making Puppet synchronize a resource.
+func applyPippetRes(runner PippetRunner, resource *PippetRes) (bool, error) {
+	runner.LockApply()
+	defer runner.UnlockApply()
+	if err := json.NewEncoder(runner.InputStream()).Encode(resource); err != nil {
+		return false, errwrap.Wrapf(err, "failed to send resource to puppet")
+	}
+
+	result := PippetResult{
+		Error:     true,
+		Exception: "missing output fields",
+	}
+	if err := json.NewDecoder(runner.OutputStream()).Decode(&result); err != nil {
+		return false, errwrap.Wrapf(err, "failed to read response from puppet")
+	}
+
+	if result.Error {
+		return false, fmt.Errorf("puppet did not sync: %s", result.Exception)
+	}
+	if result.Failed {
+		return false, fmt.Errorf("puppet failed to sync")
+	}
+	return result.Changed, nil
+}

engine/resources/pippet_test.go

@@ -0,0 +1,136 @@
+// Mgmt
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// +build !root
+
+package resources
+
+import (
+	"io"
+	"testing"
+)
+
+type nullWriteCloser struct {
+}
+
+type fakePippetReceiver struct {
+	stdin  nullWriteCloser
+	stdout *io.PipeReader
+	Locked bool
+}
+
+func (obj nullWriteCloser) Write(data []byte) (int, error) {
+	return len(data), nil
+}
+
+func (obj nullWriteCloser) Close() error {
+	return nil
+}
+
+func (obj *fakePippetReceiver) LockApply() {
+	obj.Locked = true
+}
+
+func (obj *fakePippetReceiver) UnlockApply() {
+	obj.Locked = false
+}
+
+func (obj *fakePippetReceiver) InputStream() io.WriteCloser {
+	return obj.stdin
+}
+
+func (obj *fakePippetReceiver) OutputStream() io.ReadCloser {
+	return obj.stdout
+}
+
+func newFakePippetReceiver(jsonTestOutput string) *fakePippetReceiver {
+	output, input := io.Pipe()
+
+	result := &fakePippetReceiver{
+		stdout: output,
+	}
+
+	go func() {
+		// this will appear on the fake stdout
+		input.Write([]byte(jsonTestOutput))
+	}()
+
+	return result
+}
+
+var pippetTestRes = &PippetRes{
+	Type:   "notify",
+	Title:  "testmessage",
+	Params: `{msg: "This is a test"}`,
+}
+
+func TestNormalPuppetOutput(t *testing.T) {
+	r := newFakePippetReceiver(`{"resource":"Notify[test]","failed":false,"changed":true,"noop":false,"error":false,"exception":null}`)
+	changed, err := applyPippetRes(r, pippetTestRes)
+
+	if err != nil {
+		t.Errorf("normal Puppet output led to an apply error: %v", err)
+	}
+
+	if !changed {
+		t.Errorf("return values of applyPippetRes did not reflect the changed state")
+	}
+}
+
+func TestUnchangedPuppetOutput(t *testing.T) {
+	r := newFakePippetReceiver(`{"resource":"Notify[test]","failed":false,"changed":false,"noop":false,"error":false,"exception":null}`)
+	changed, err := applyPippetRes(r, pippetTestRes)
+
+	if err != nil {
+		t.Errorf("normal Puppet output led to an apply error: %v", err)
+	}
+
+	if changed {
+		t.Errorf("return values of applyPippetRes did not reflect the changed state")
+	}
+}
+
+func TestFailingPuppetOutput(t *testing.T) {
+	r := newFakePippetReceiver(`{"resource":"Notify[test]","failed":false,"changed":false,"noop":false,"error":true,"exception":"I failed!"}`)
+	_, err := applyPippetRes(r, pippetTestRes)
+
+	if err == nil {
+		t.Errorf("failing Puppet output led to an apply error: %v", err)
+	}
+}
+
+func TestEmptyPuppetOutput(t *testing.T) {
+	t.Skip("empty output will currently make the application (and the test) hang")
+}
+
+func TestPartialPuppetOutput(t *testing.T) {
+	r := newFakePippetReceiver(`{"resource":"Notify[test]","failed":false,"changed":true}`)
+	_, err := applyPippetRes(r, pippetTestRes)
+
+	if err == nil {
+		t.Errorf("partial Puppet output did not lead to an apply error")
+	}
+}
+
+func TestMalformedPuppetOutput(t *testing.T) {
+	r := newFakePippetReceiver(`oops something went wrong!!1!eleven`)
+	_, err := applyPippetRes(r, pippetTestRes)
+
+	if err == nil {
+		t.Errorf("malformed Puppet output did not lead to an apply error")
+	}
+}

engine/resources/pkg.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -26,8 +26,7 @@ import (
 	"github.com/purpleidea/mgmt/engine/resources/packagekit"
 	"github.com/purpleidea/mgmt/engine/traits"
 	"github.com/purpleidea/mgmt/util"
-
-	errwrap "github.com/pkg/errors"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 func init() {
@@ -101,8 +100,8 @@ func (obj *PkgRes) Close() error {
 	return nil
 }
 
-// Watch is the primary listener for this resource and it outputs events.
-// It uses the PackageKit UpdatesChanged signal to watch for changes.
+// Watch is the primary listener for this resource and it outputs events. It
+// uses the PackageKit UpdatesChanged signal to watch for changes.
 // TODO: https://github.com/hughsie/PackageKit/issues/109
 // TODO: https://github.com/hughsie/PackageKit/issues/110
 func (obj *PkgRes) Watch() error {
@@ -263,7 +262,7 @@ func (obj *PkgRes) populateFileList() error {
 
 // CheckApply checks the resource state and applies the resource if the bool
 // input is true. It returns error info and if the state check passed or not.
-func (obj *PkgRes) CheckApply(apply bool) (checkOK bool, err error) {
+func (obj *PkgRes) CheckApply(apply bool) (bool, error) {
 	obj.init.Logf("Check: %s", obj.fmtNames(obj.getNames()))
 
 	bus := packagekit.NewBus()
@@ -505,7 +504,8 @@ func (obj *PkgResAutoEdges) Next() []engine.ResUID {
 	return result
 }
 
-// Test gets results of the earlier Next() call, & returns if we should continue!
+// Test gets results of the earlier Next() call, & returns if we should
+// continue!
 func (obj *PkgResAutoEdges) Test(input []bool) bool {
 	if !obj.testIsNext {
 		panic("expecting a call to Next()")
@@ -592,8 +592,8 @@ func (obj *PkgRes) AutoEdges() (engine.AutoEdge, error) {
 	}, nil
 }
 
-// UIDs includes all params to make a unique identification of this object.
-// Most resources only return one, although some resources can return multiple.
+// UIDs includes all params to make a unique identification of this object. Most
+// resources only return one, although some resources can return multiple.
 func (obj *PkgRes) UIDs() []engine.ResUID {
 	x := &PkgUID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
@@ -612,9 +612,9 @@ func (obj *PkgRes) UIDs() []engine.ResUID {
 	return result
 }
 
-// GroupCmp returns whether two resources can be grouped together or not.
-// Can these two resources be merged, aka, does this resource support doing so?
-// Will resource allow itself to be grouped _into_ this obj?
+// GroupCmp returns whether two resources can be grouped together or not. Can
+// these two resources be merged, aka, does this resource support doing so? Will
+// resource allow itself to be grouped _into_ this obj?
 func (obj *PkgRes) GroupCmp(r engine.GroupableRes) error {
 	res, ok := r.(*PkgRes)
 	if !ok {
@@ -632,8 +632,8 @@ func (obj *PkgRes) GroupCmp(r engine.GroupableRes) error {
 	return nil
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
-// It is primarily useful for setting the defaults.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
+// primarily useful for setting the defaults.
 func (obj *PkgRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes PkgRes // indirection to avoid infinite recursion
 
@@ -652,7 +652,8 @@ func (obj *PkgRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	return nil
 }
 
-// ReturnSvcInFileList returns a list of svc names for matches like: `/usr/lib/systemd/system/*.service`.
+// ReturnSvcInFileList returns a list of svc names for matches like:
+// `/usr/lib/systemd/system/*.service`.
 func ReturnSvcInFileList(fileList []string) []string {
 	result := []string{}
 	for _, x := range fileList {

engine/resources/pkg_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/resources/print.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -40,6 +40,10 @@ type PrintRes struct {
 	init *engine.Init
 
 	Msg string `lang:"msg" yaml:"msg"` // the message to display
+	// RefreshOnly is an option that causes the message to be printed only
+	// when notified by another resource. When set to true, this resource
+	// cannot be autogrouped.
+	RefreshOnly bool `lang:"refresh_only" yaml:"refresh_only"`
 }
 
 // Default returns some sensible defaults for this resource.
@@ -78,24 +82,32 @@ func (obj *PrintRes) Watch() error {
 }
 
 // CheckApply method for Print resource. Does nothing, returns happy!
-func (obj *PrintRes) CheckApply(apply bool) (checkOK bool, err error) {
+func (obj *PrintRes) CheckApply(apply bool) (bool, error) {
 	obj.init.Logf("CheckApply: %t", apply)
 	if val, exists := obj.init.Recv()["Msg"]; exists && val.Changed {
 		// if we received on Msg, and it changed, log message
 		obj.init.Logf("CheckApply: Received `Msg` of: %s", obj.Msg)
 	}
 
-	if obj.init.Refresh() {
+	var refresh = obj.init.Refresh()
+	// we output if not RefreshOnly, or we are in refresh mode and RefreshOnly
+	var display = refresh || !obj.RefreshOnly
+
+	if refresh {
 		obj.init.Logf("Received a notification!")
 	}
-	obj.init.Logf("Msg: %s", obj.Msg)
+	if display {
+		obj.init.Logf("Msg: %s", obj.Msg)
+	}
 	if g := obj.GetGroup(); len(g) > 0 { // add any grouped elements
 		for _, x := range g {
 			print, ok := x.(*PrintRes) // convert from Res
 			if !ok {
 				panic(fmt.Sprintf("grouped member %v is not a %s", x, obj.Kind()))
 			}
-			obj.init.Logf("%s: Msg: %s", print, print.Msg)
+			if display {
+				obj.init.Logf("%s: Msg: %s", print, print.Msg)
+			}
 		}
 	}
 	return true, nil // state is always okay
@@ -103,24 +115,16 @@ func (obj *PrintRes) CheckApply(apply bool) (checkOK bool, err error) {
 
 // Cmp compares two resources and returns an error if they are not equivalent.
 func (obj *PrintRes) Cmp(r engine.Res) error {
-	if !obj.Compare(r) {
-		return fmt.Errorf("did not compare")
-	}
-	return nil
-}
-
-// Compare two resources and return if they are equivalent.
-func (obj *PrintRes) Compare(r engine.Res) bool {
 	// we can only compare PrintRes to others of the same resource kind
 	res, ok := r.(*PrintRes)
 	if !ok {
-		return false
+		return fmt.Errorf("not a %s", obj.Kind())
 	}
 
 	if obj.Msg != res.Msg {
-		return false
+		return fmt.Errorf("the Msg differs")
 	}
-	return true
+	return nil
 }
 
 // PrintUID is the UID struct for PrintRes.
@@ -129,8 +133,8 @@ type PrintUID struct {
 	name string
 }
 
-// UIDs includes all params to make a unique identification of this object.
-// Most resources only return one, although some resources can return multiple.
+// UIDs includes all params to make a unique identification of this object. Most
+// resources only return one, although some resources can return multiple.
 func (obj *PrintRes) UIDs() []engine.ResUID {
 	x := &PrintUID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
@@ -141,15 +145,19 @@ func (obj *PrintRes) UIDs() []engine.ResUID {
 
 // GroupCmp returns whether two resources can be grouped together or not.
 func (obj *PrintRes) GroupCmp(r engine.GroupableRes) error {
-	_, ok := r.(*PrintRes)
+	res, ok := r.(*PrintRes)
 	if !ok {
 		return fmt.Errorf("resource is not the same kind")
 	}
+	// we don't group if it's RefreshOnly: only the notifier may trigger
+	if obj.RefreshOnly || res.RefreshOnly {
+		return fmt.Errorf("resource uses RefreshOnly, it cannot be merged")
+	}
 	return nil // grouped together if we were asked to
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
-// It is primarily useful for setting the defaults.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
+// primarily useful for setting the defaults.
 func (obj *PrintRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes PrintRes // indirection to avoid infinite recursion