lib: Update the help text

Give some longer descriptions so they show up nicely for the user.

.github/FUNDING.yml

@@ -1,2 +1,5 @@
-# You can add one username per supported platform and one custom link
+# You can add one username per supported platform and one custom link.
+custom: "https://paypal.me/purpleidea"
+github: purpleidea
+liberapay: purpleidea
 patreon: purpleidea

.github/settings.yml

@@ -68,6 +68,8 @@ labels:
     color: e11d21
   - name: question
     color: cc317c
+  - name: needinfo
+    color: fbca04
   - name: wontfix
     color: ffffff
 #  - name: first-timers-only

.github/workflows/test.yaml

@@ -0,0 +1,70 @@
+# Docs: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+
+# If the name is omitted, it uses the filename instead.
+#name: Test
+on:
+  # Run on all pull requests.
+  pull_request:
+    #branches:
+    #- master
+  # Run on all pushes.
+  push:
+  # Run daily at 4am.
+  schedule:
+  - cron: 0 4 * * *
+
+jobs:
+  maketest:
+    name: Test (${{ matrix.test_block }}) on ${{ matrix.os }} with golang ${{ matrix.golang_version }}
+    runs-on: ${{ matrix.os }}
+    env:
+      GOPATH: /home/runner/work/mgmt/mgmt/go
+    strategy:
+      matrix:
+        # TODO: Add tip when it's supported: https://github.com/actions/setup-go/issues/21
+        os:
+          - ubuntu-latest
+          # macos tests are currently failing in CI
+          #- macos-latest
+        golang_version:
+          # TODO: add 1.15.x and tip
+          # minimum required and latest published go_version
+          #- 1.13
+          - 1.15
+        test_block:
+          - basic
+          - shell
+          - race
+      #fail-fast: false
+
+    steps:
+      # Do not shallow fetch, will fail when building bindata/
+      # The path can't be absolute, so we need to move it to the
+      # expected location later.
+      - name: Clone mgmt
+        uses: actions/checkout@v2
+        with:
+          submodules: recursive
+          fetch-depth: 0
+          path: ./go/src/github.com/purpleidea/mgmt
+
+      - name: Install Go ${{ matrix.golang_version }}
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.golang_version }}
+
+      # Install & configure ruby, fixes gem permissions error
+      - name: Install Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: head
+
+      - name: Install dependencies
+        working-directory: ./go/src/github.com/purpleidea/mgmt
+        run: |
+          make deps
+
+      - name: Run test
+        working-directory: ./go/src/github.com/purpleidea/mgmt
+        run: |
+          TEST_BLOCK="${{ matrix.test_block }}" make test

.gitignore

@@ -17,3 +17,5 @@ rpmbuild/
 releases/
 # vim swap files
 .*.sw[op]
+# prevent `echo foo 2>1` typo errors by making this file read-only
+1

.gitmodules

@@ -1,5 +1,5 @@
 [submodule "vendor/github.com/coreos/etcd"]
-	path = vendor/github.com/coreos/etcd
+	path = vendor/go.etcd.io/etcd
 	url = https://github.com/coreos/etcd/
 [submodule "vendor/google.golang.org/grpc"]
 	path = vendor/google.golang.org/grpc
@@ -28,6 +28,12 @@
 [submodule "vendor/github.com/purpleidea/distribution"]
 	path = vendor/github.com/docker/distribution
 	url = https://github.com/purpleidea/distribution
-[submodule "vendor/github.com/purpleidea/go-connections"]
+[submodule "vendor/github.com/hashicorp/go-multierror"]
-	path = vendor/github.com/docker/go-connections
+	path = vendor/github.com/hashicorp/go-multierror
-	url = https://github.com/docker/go-connections
+	url = https://github.com/hashicorp/go-multierror
+[submodule "vendor/github.com/containerd/containerd"]
+	path = vendor/github.com/containerd/containerd
+	url = https://github.com/purpleidea/containerd
+[submodule "vendor/github.com/hashicorp/consul"]
+	path = vendor/github.com/hashicorp/consul
+	url = https://github.com/hashicorp/consul/

.travis.yml

@@ -24,21 +24,21 @@ install: 'make deps'
 matrix:
   fast_finish: false
   allow_failures:
-  - go: 1.12.x
+  - go: 1.14.x
   - go: tip
   - os: osx
   # include only one build for osx for a quicker build as the nr. of these runners are sparse
   include:
   - name: "basic tests"
-    go: 1.11.x
+    go: 1.13.x
     env: TEST_BLOCK=basic
   - name: "shell tests"
-    go: 1.11.x
+    go: 1.13.x
     env: TEST_BLOCK=shell
   - name: "race tests"
-    go: 1.11.x
+    go: 1.13.x
     env: TEST_BLOCK=race
-  - go: 1.12.x
+  - go: 1.14.x
   - go: tip
   - os: osx
 script: 'TEST_BLOCK="$TEST_BLOCK" make test'
@@ -47,8 +47,8 @@ script: 'TEST_BLOCK="$TEST_BLOCK" make test'
 # with a value of: irc.freenode.net#mgmtconfig to eliminate noise from forks...
 notifications:
   irc:
-    channels:
+    #channels:
-      - secure: htcuWAczm3C1zKC9vUfdRzhIXM1vtF+q0cLlQFXK1IQQlk693/pM30Mmf2L/9V2DVDeps+GyLdip0ARXD1DZEJV0lK+Ca1qbHdFP1r4Xv6l5+jaDb5Y88YU5LI8K758QShiZJojuQ1aO2j8xmmt9V0/5y5QwlpPeHbKYBOFPBX3HvlT9DhvwZNKGhBb4qJOEaPVOwq9IkN3DyQ456MHcJ3q3vF9Lb440uTuLsJNof2AbYZH8ZIHCSG2N8tBj2qhJOpWQboYtQJzE2pRaGkGBL4kYcHZSZMXX8sl4cBM1vx/IRUkvBxJUpLJz2gn/eRI+/gr59juZE2K0+FOLlx9dLnX626Y9xSViopBI6JsIoHJDqNC7aGaF2qaYulGYN65VNKVqmghjgt6JLmmiKeH10hYrJMMvt2rms8l4+5iwmCwXvhH/WU9edzk2p5wqERMnostJFEJib0zI3yzLoF0sdJs+veKtagzfayY2d2l7hlmt951IpqqVWldVgWUcQKVvi8gmRarbwFlK+5D7BEnkUDcLNly/cqf7BgEeX6YfF+FiR4pgfOhYvGCD+2q91NgWQXHBCxbyN0be1TVdkXD94f0Lkn94VyEJJ+PkPlG+rPgFwGcjqN4oEGkJeJmES2If05q2Ms1dJLwYQDL3+Py4lNMSdSWj24TzlFVhtwHepuw=
+    #  - secure: htcuWAczm3C1zKC9vUfdRzhIXM1vtF+q0cLlQFXK1IQQlk693/pM30Mmf2L/9V2DVDeps+GyLdip0ARXD1DZEJV0lK+Ca1qbHdFP1r4Xv6l5+jaDb5Y88YU5LI8K758QShiZJojuQ1aO2j8xmmt9V0/5y5QwlpPeHbKYBOFPBX3HvlT9DhvwZNKGhBb4qJOEaPVOwq9IkN3DyQ456MHcJ3q3vF9Lb440uTuLsJNof2AbYZH8ZIHCSG2N8tBj2qhJOpWQboYtQJzE2pRaGkGBL4kYcHZSZMXX8sl4cBM1vx/IRUkvBxJUpLJz2gn/eRI+/gr59juZE2K0+FOLlx9dLnX626Y9xSViopBI6JsIoHJDqNC7aGaF2qaYulGYN65VNKVqmghjgt6JLmmiKeH10hYrJMMvt2rms8l4+5iwmCwXvhH/WU9edzk2p5wqERMnostJFEJib0zI3yzLoF0sdJs+veKtagzfayY2d2l7hlmt951IpqqVWldVgWUcQKVvi8gmRarbwFlK+5D7BEnkUDcLNly/cqf7BgEeX6YfF+FiR4pgfOhYvGCD+2q91NgWQXHBCxbyN0be1TVdkXD94f0Lkn94VyEJJ+PkPlG+rPgFwGcjqN4oEGkJeJmES2If05q2Ms1dJLwYQDL3+Py4lNMSdSWj24TzlFVhtwHepuw=
     template:
       - "%{repository} (%{commit}: %{author}): %{message}"
       - "More info : %{build_url}"

AUTHORS

@@ -6,6 +6,7 @@ This list is sorted alphabetically by first name.
 
 Felix Frank
 James Shubin
+Joe Groocock
 Johan Bloemberg
 Jonathan Gold
 Julien Pivotto

COPYRIGHT

@@ -1,5 +1,5 @@
 Mgmt
-Copyright (C) 2013-2019+ James Shubin and the project contributors
+Copyright (C) 2013-2021+ James Shubin and the project contributors
 Written by James Shubin <james@shubin.ca> and the project contributors
 
 This program is free software: you can redistribute it and/or modify

Makefile

@@ -1,5 +1,5 @@
 # Mgmt
-# Copyright (C) 2013-2019+ James Shubin and the project contributors
+# Copyright (C) 2013-2021+ James Shubin and the project contributors
 # Written by James Shubin <james@shubin.ca> and the project contributors
 #
 # This program is free software: you can redistribute it and/or modify
@@ -18,8 +18,8 @@
 SHELL = /usr/bin/env bash
 .PHONY: all art cleanart version program lang path deps run race bindata generate build build-debug crossbuild clean test gofmt yamlfmt format docs
 .PHONY: rpmbuild mkdirs rpm srpm spec tar upload upload-sources upload-srpms upload-rpms upload-releases copr tag
-.PHONY: mkosi mkosi_fedora-30 mkosi_fedora-29 mkosi_debian-10 mkosi_ubuntu-bionic mkosi_archlinux
+.PHONY: mkosi mkosi_fedora-30 mkosi_fedora-29 mkosi_centos-7 mkosi_debian-10 mkosi_ubuntu-bionic mkosi_archlinux
-.PHONY: release releases_path release_fedora-30 release_fedora-29 release_debian-10 release_ubuntu-bionic release_archlinux
+.PHONY: release releases_path release_fedora-30 release_fedora-29 release_centos-7 release_debian-10 release_ubuntu-bionic release_archlinux
 .PHONY: funcgen
 .SILENT: clean bindata
 
@@ -55,18 +55,21 @@ GOHOSTARCH = $(shell go env GOHOSTARCH)
 
 TOKEN_FEDORA-30 = fedora-30
 TOKEN_FEDORA-29 = fedora-29
+TOKEN_CENTOS-7 = centos-7
 TOKEN_DEBIAN-10 = debian-10
 TOKEN_UBUNTU-BIONIC = ubuntu-bionic
 TOKEN_ARCHLINUX = archlinux
 
 FILE_FEDORA-30 = mgmt-$(TOKEN_FEDORA-30)-$(VERSION)-1.x86_64.rpm
 FILE_FEDORA-29 = mgmt-$(TOKEN_FEDORA-29)-$(VERSION)-1.x86_64.rpm
+FILE_CENTOS-7 = mgmt-$(TOKEN_CENTOS-7)-$(VERSION)-1.x86_64.rpm
 FILE_DEBIAN-10 = mgmt_$(TOKEN_DEBIAN-10)_$(VERSION)_amd64.deb
 FILE_UBUNTU-BIONIC = mgmt_$(TOKEN_UBUNTU-BIONIC)_$(VERSION)_amd64.deb
 FILE_ARCHLINUX = mgmt-$(TOKEN_ARCHLINUX)-$(VERSION)-1-x86_64.pkg.tar.xz
 
 PKG_FEDORA-30 = releases/$(VERSION)/$(TOKEN_FEDORA-30)/$(FILE_FEDORA-30)
 PKG_FEDORA-29 = releases/$(VERSION)/$(TOKEN_FEDORA-29)/$(FILE_FEDORA-29)
+PKG_CENTOS-7 = releases/$(VERSION)/$(TOKEN_CENTOS-7)/$(FILE_CENTOS-7)
 PKG_DEBIAN-10 = releases/$(VERSION)/$(TOKEN_DEBIAN-10)/$(FILE_DEBIAN-10)
 PKG_UBUNTU-BIONIC = releases/$(VERSION)/$(TOKEN_UBUNTU-BIONIC)/$(FILE_UBUNTU-BIONIC)
 PKG_ARCHLINUX = releases/$(VERSION)/$(TOKEN_ARCHLINUX)/$(FILE_ARCHLINUX)
@@ -167,19 +170,14 @@ GOOS=$(firstword $(subst -, ,$*))
 GOARCH=$(lastword $(subst -, ,$*))
 build/mgmt-%: $(GO_FILES) $(MCL_FILES) | bindata lang funcgen
 	@echo "Building: $(PROGRAM), os/arch: $*, version: $(SVERSION)..."
-	@# reassigning GOOS and GOARCH to make build command copy/pastable
+	@time env GOOS=${GOOS} GOARCH=${GOARCH} go build -i -ldflags=$(PKGNAME)="-X main.program=$(PROGRAM) -X main.version=$(SVERSION) ${LDFLAGS}" -o $@ $(BUILD_FLAGS)
-	@# go 1.10+ requires specifying the package for ldflags
-	@if go version | grep -qE 'go1.9'; then \
-		time env GOOS=${GOOS} GOARCH=${GOARCH} go build -i -ldflags "-X main.program=$(PROGRAM) -X main.version=$(SVERSION) ${LDFLAGS}" -o $@ $(BUILD_FLAGS); \
-	else \
-		time env GOOS=${GOOS} GOARCH=${GOARCH} go build -i -ldflags=$(PKGNAME)="-X main.program=$(PROGRAM) -X main.version=$(SVERSION) ${LDFLAGS}" -o $@ $(BUILD_FLAGS); \
-	fi
 
 # create a list of binary file names to use as make targets
 crossbuild_targets = $(addprefix build/mgmt-,$(subst /,-,${GOOSARCHES}))
 crossbuild: ${crossbuild_targets}
 
 clean: ## clean things up
+	$(MAKE) --quiet -C test clean
 	$(MAKE) --quiet -C bindata clean
 	$(MAKE) --quiet -C lang/funcs clean
 	$(MAKE) --quiet -C lang clean
@@ -193,6 +191,8 @@ clean: ## clean things up
 	rm -f build/mgmt-*
 
 test: build ## run tests
+	@# recursively run make in child dir named test
+	@$(MAKE) --quiet -C test
 	./test.sh
 
 # create all test targets for make tab completion (eg: make test-gofmt)
@@ -365,7 +365,7 @@ tag: ## tags a new release
 #
 #	mkosi
 #
-mkosi: mkosi_fedora-30 mkosi_fedora-29 mkosi_debian-10 mkosi_ubuntu-bionic mkosi_archlinux ## builds distro packages via mkosi
+mkosi: mkosi_fedora-30 mkosi_fedora-29 mkosi_centos-7 mkosi_debian-10 mkosi_ubuntu-bionic mkosi_archlinux ## builds distro packages via mkosi
 
 mkosi_fedora-30: releases/$(VERSION)/.mkdir
 	@title='$@' ; echo "Generating: $${title#'mkosi_'} via mkosi..."
@@ -375,6 +375,10 @@ mkosi_fedora-29: releases/$(VERSION)/.mkdir
 	@title='$@' ; echo "Generating: $${title#'mkosi_'} via mkosi..."
 	@title='$@' ; distro=$${title#'mkosi_'} ; ./misc/mkosi/make.sh $${distro} `realpath "releases/$(VERSION)/"`
 
+mkosi_centos-7: releases/$(VERSION)/.mkdir
+	@title='$@' ; echo "Generating: $${title#'mkosi_'} via mkosi..."
+	@title='$@' ; distro=$${title#'mkosi_'} ; ./misc/mkosi/make.sh $${distro} `realpath "releases/$(VERSION)/"`
+
 mkosi_debian-10: releases/$(VERSION)/.mkdir
 	@title='$@' ; echo "Generating: $${title#'mkosi_'} via mkosi..."
 	@title='$@' ; distro=$${title#'mkosi_'} ; ./misc/mkosi/make.sh $${distro} `realpath "releases/$(VERSION)/"`
@@ -398,11 +402,12 @@ releases_path:
 
 release_fedora-30: $(PKG_FEDORA-30)
 release_fedora-29: $(PKG_FEDORA-29)
+release_centos-7: $(PKG_CENTOS-7)
 release_debian-10: $(PKG_DEBIAN-10)
 release_ubuntu-bionic: $(PKG_UBUNTU-BIONIC)
 release_archlinux: $(PKG_ARCHLINUX)
 
-releases/$(VERSION)/mgmt-release.url: $(PKG_FEDORA-30) $(PKG_FEDORA-29) $(PKG_DEBIAN-10) $(PKG_UBUNTU-BIONIC) $(PKG_ARCHLINUX) $(SHA256SUMS_ASC)
+releases/$(VERSION)/mgmt-release.url: $(PKG_FEDORA-30) $(PKG_FEDORA-29) $(PKG_CENTOS-7) $(PKG_DEBIAN-10) $(PKG_UBUNTU-BIONIC) $(PKG_ARCHLINUX) $(SHA256SUMS_ASC)
 	@echo "Pushing git tag $(VERSION) to origin..."
 	git push origin $(VERSION)
 	@echo "Creating github release..."
@@ -410,6 +415,7 @@ releases/$(VERSION)/mgmt-release.url: $(PKG_FEDORA-30) $(PKG_FEDORA-29) $(PKG_DE
 		-F <( echo -e "$(VERSION)\n";echo "Verify the signatures of all packages before you use them. The signing key can be downloaded from https://purpleidea.com/contact/#pgp-key to verify the release." ) \
 		-a $(PKG_FEDORA-30) \
 		-a $(PKG_FEDORA-29) \
+		-a $(PKG_CENTOS-7) \
 		-a $(PKG_DEBIAN-10) \
 		-a $(PKG_UBUNTU-BIONIC) \
 		-a $(PKG_ARCHLINUX) \
@@ -420,7 +426,7 @@ releases/$(VERSION)/mgmt-release.url: $(PKG_FEDORA-30) $(PKG_FEDORA-29) $(PKG_DE
 		|| rm -f releases/$(VERSION)/mgmt-release.url
 
 releases/$(VERSION)/.mkdir:
-	mkdir -p releases/$(VERSION)/{$(TOKEN_FEDORA-30),$(TOKEN_FEDORA-29),$(TOKEN_DEBIAN-10),$(TOKEN_UBUNTU-BIONIC),$(TOKEN_ARCHLINUX)}/ && touch releases/$(VERSION)/.mkdir
+	mkdir -p releases/$(VERSION)/{$(TOKEN_FEDORA-30),$(TOKEN_FEDORA-29),$(TOKEN_CENTOS-7),$(TOKEN_DEBIAN-10),$(TOKEN_UBUNTU-BIONIC),$(TOKEN_ARCHLINUX)}/ && touch releases/$(VERSION)/.mkdir
 
 releases/$(VERSION)/$(TOKEN_FEDORA-30)/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir
 	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Generating: $${distro} changelog..."
@@ -438,6 +444,14 @@ $(PKG_FEDORA-29): releases/$(VERSION)/$(TOKEN_FEDORA-29)/changelog
 	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Building: $${distro} package..."
 	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/fpm-pack.sh $${distro} $(VERSION) "$(FILE_FEDORA-29)" libvirt-devel augeas-devel
 
+releases/$(VERSION)/$(TOKEN_CENTOS-7)/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Generating: $${distro} changelog..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/make-rpm-changelog.sh "$${distro}" $(VERSION)
+
+$(PKG_CENTOS-7): releases/$(VERSION)/$(TOKEN_CENTOS-7)/changelog
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Building: $${distro} package..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/fpm-pack.sh $${distro} $(VERSION) "$(FILE_CENTOS-7)" libvirt-devel augeas-devel
+
 releases/$(VERSION)/$(TOKEN_DEBIAN-10)/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir
 	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Generating: $${distro} changelog..."
 	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/make-deb-changelog.sh "$${distro}" $(VERSION)
@@ -458,10 +472,10 @@ $(PKG_ARCHLINUX): $(PROGRAM) releases/$(VERSION)/.mkdir
 	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Building: $${distro} package..."
 	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/fpm-pack.sh $${distro} $(VERSION) "$(FILE_ARCHLINUX)" libvirt augeas
 
-$(SHA256SUMS): $(PKG_FEDORA-30) $(PKG_FEDORA-29) $(PKG_DEBIAN-10) $(PKG_UBUNTU-BIONIC) $(PKG_ARCHLINUX)
+$(SHA256SUMS): $(PKG_FEDORA-30) $(PKG_FEDORA-29) $(PKG_CENTOS-7) $(PKG_DEBIAN-10) $(PKG_UBUNTU-BIONIC) $(PKG_ARCHLINUX)
 	@# remove the directory separator in the SHA256SUMS file
 	@echo "Generating: sha256 sum..."
-	sha256sum $(PKG_FEDORA-30) $(PKG_FEDORA-29) $(PKG_DEBIAN-10) $(PKG_UBUNTU-BIONIC) $(PKG_ARCHLINUX) | awk -F '/| ' '{print $$1"  "$$6}' > $(SHA256SUMS)
+	sha256sum $(PKG_FEDORA-30) $(PKG_FEDORA-29) $(PKG_CENTOS-7) $(PKG_DEBIAN-10) $(PKG_UBUNTU-BIONIC) $(PKG_ARCHLINUX) | awk -F '/| ' '{print $$1"  "$$6}' > $(SHA256SUMS)
 
 $(SHA256SUMS_ASC): $(SHA256SUMS)
 	@echo "Signing sha256 sum..."
@@ -487,14 +501,10 @@ help: ## show this help screen
 		awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
 	@echo ''
 
-funcgen: lang/funcs/core/generated_funcs_test.go lang/funcs/core/generated_funcs.go
+funcgen: lang/funcs/core/generated_funcs.go
-
-lang/funcs/core/generated_funcs_test.go: lang/funcs/funcgen/*.go lang/funcs/core/funcgen.yaml lang/funcs/funcgen/templates/generated_funcs_test.go.tpl
-	@echo "Generating: funcs test..."
-	@go run lang/funcs/funcgen/*.go -templates lang/funcs/funcgen/templates/generated_funcs_test.go.tpl 2>/dev/null
 
 lang/funcs/core/generated_funcs.go: lang/funcs/funcgen/*.go lang/funcs/core/funcgen.yaml lang/funcs/funcgen/templates/generated_funcs.go.tpl
 	@echo "Generating: funcs..."
-	@go run lang/funcs/funcgen/*.go -templates lang/funcs/funcgen/templates/generated_funcs.go.tpl 2>/dev/null
+	@go run `find lang/funcs/funcgen/ -maxdepth 1 -type f -name '*.go' -not -name '*_test.go'` -templates=lang/funcs/funcgen/templates/generated_funcs.go.tpl >/dev/null
 
 # vim: ts=8

README.md

@@ -4,8 +4,9 @@
 
 [![Go Report Card](https://goreportcard.com/badge/github.com/purpleidea/mgmt?style=flat-square)](https://goreportcard.com/report/github.com/purpleidea/mgmt)
 [![Build Status](https://img.shields.io/travis/purpleidea/mgmt/master.svg?style=flat-square)](http://travis-ci.org/purpleidea/mgmt)
+[![Build Status](https://github.com/purpleidea/mgmt/workflows/.github/workflows/test.yaml/badge.svg)](https://github.com/purpleidea/mgmt/actions/)
 [![GoDoc](https://img.shields.io/badge/godoc-reference-5272B4.svg?style=flat-square)](https://godoc.org/github.com/purpleidea/mgmt)
-[![IRC](https://img.shields.io/badge/irc-%23mgmtconfig-orange.svg?style=flat-square)](https://webchat.freenode.net/?channels=#mgmtconfig)
+[![IRC](https://img.shields.io/badge/irc-%23mgmtconfig-orange.svg?style=flat-square)](https://web.libera.chat/?channels=#mgmtconfig)
 [![Patreon](https://img.shields.io/badge/patreon-donate-yellow.svg?style=flat-square)](https://www.patreon.com/purpleidea)
 [![Liberapay](https://img.shields.io/badge/liberapay-donate-yellow.svg?style=flat-square)](https://liberapay.com/purpleidea/donate)
 
@@ -21,7 +22,7 @@ ensure that your file server is set to read-only when it's friday.
 import "datetime"
 $is_friday = datetime.weekday(datetime.now()) == "friday"
 file "/srv/files/" {
-	state => "exists",
+	state => $const.res.file.state.exists,
 	mode => if $is_friday { # this updates the mode, the instant it changes!
 		"0550"
 	} else {
@@ -65,7 +66,7 @@ Come join us in the `mgmt` community!
 
 | Medium | Link |
 |---|---|
-| IRC | [#mgmtconfig](https://webchat.freenode.net/?channels=#mgmtconfig) on Freenode |
+| IRC | [#mgmtconfig](https://web.libera.chat/?channels=#mgmtconfig) on Libera.Chat |
 | Twitter | [@mgmtconfig](https://twitter.com/mgmtconfig) & [#mgmtconfig](https://twitter.com/hashtag/mgmtconfig) |
 | Mailing list | [mgmtconfig-list@redhat.com](https://www.redhat.com/mailman/listinfo/mgmtconfig-list) |
 | Patreon | [purpleidea](https://www.patreon.com/purpleidea) on Patreon |

Vagrantfile

@@ -6,7 +6,7 @@ Vagrant.configure(2) do |config|
 	config.vm.synced_folder ".", "/vagrant", disabled: true
 
 	config.vm.define "mgmt-dev" do |instance|
-		instance.vm.box = "fedora/28-cloud-base"
+		instance.vm.box = "bento/fedora-31"
 	end
 
 	config.vm.provider "virtualbox" do |v|
@@ -23,8 +23,7 @@ Vagrant.configure(2) do |config|
 	config.vm.provision "file", source: "vagrant/mgmt.bashrc", destination: ".mgmt.bashrc"
 	config.vm.provision "file", source: "~/.gitconfig", destination: ".gitconfig"
 
-	# copied from make-deps.sh (with added git)
+	config.vm.provision "shell", inline: "dnf install -y golang git make"
-	config.vm.provision "shell", inline: "dnf install -y libvirt-devel golang golang-googlecode-tools-stringer hg git make gem"
 
 	# set up packagekit
 	config.vm.provision "shell" do |shell|
@@ -39,8 +38,10 @@ Vagrant.configure(2) do |config|
 	script = <<-SCRIPT
 		grep -q 'mgmt\.bashrc' ~/.bashrc || echo '. ~/.mgmt.bashrc' >>~/.bashrc
 		. ~/.mgmt.bashrc
-		go get -u github.com/purpleidea/mgmt
+		mkdir -p ~/gopath/src/github.com/purpleidea
-		cd ~/gopath/src/github.com/purpleidea/mgmt
+		cd ~/gopath/src/github.com/purpleidea
+		git clone https://github.com/purpleidea/mgmt --recursive
+		cd mgmt
 		make deps
 	SCRIPT
 	config.vm.provision "shell" do |shell|

bindata/Makefile

@@ -1,5 +1,5 @@
 # Mgmt
-# Copyright (C) 2013-2019+ James Shubin and the project contributors
+# Copyright (C) 2013-2021+ James Shubin and the project contributors
 # Written by James Shubin <james@shubin.ca> and the project contributors
 #
 # This program is free software: you can redistribute it and/or modify

converger/converger.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

converger/converger_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

debian/copyright

@@ -3,7 +3,7 @@ Upstream-Name: mgmt
 Source: <https://github.com/purpleidea/mgmt>
 
 Files: *
-Copyright: Copyright (C) 2013-2019+ James Shubin and the project contributors
+Copyright: Copyright (C) 2013-2021+ James Shubin and the project contributors
 License: GPL-3.0
 
 License: GPL-3.0

doc.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

docker/Dockerfile

@@ -1,10 +1,10 @@
-FROM golang:1.9
+FROM golang:1.13
 
 MAINTAINER Michał Czeraszkiewicz <contact@czerasz.com>
 
 # Set the reset cache variable
 # Read more here: http://czerasz.com/2014/11/13/docker-tip-and-tricks/#use-refreshedat-variable-for-better-cache-control
-ENV REFRESHED_AT 2017-11-16
+ENV REFRESHED_AT 2020-09-23
 
 # Update the package list to be able to use required packages
 RUN apt-get update

docker/Dockerfile.development

@@ -1,4 +1,4 @@
-FROM golang:1.11
+FROM golang:1.13
 
 MAINTAINER Michał Czeraszkiewicz <contact@czerasz.com>
 

docs/conf.py

@@ -51,7 +51,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'mgmt'
-copyright = u'2013-2019+ James Shubin and the project contributors'
+copyright = u'2013-2021+ James Shubin and the project contributors'
 author = u'James Shubin'
 
 # The version info for the project you're documenting, acts as replacement for

docs/development.md

@@ -28,7 +28,7 @@ required for running the _test_ suite.
 
 ### Build
 
-* `golang` 1.11 or higher (required, available in some distros and distributed
+* `golang` 1.13 or higher (required, available in some distros and distributed
 as a binary officially by [golang.org](https://golang.org/dl/))
 
 ### Runtime

docs/documentation.md

@@ -122,6 +122,10 @@ entire set of running mgmt agents will need to all simultaneously converge for
 the group to exit. This is particularly useful for bootstrapping new clusters
 which need to exchange information that is only available at run time.
 
+This existed in earlier versions of mgmt as a `--remote` option, but it has been
+removed and is being ported to a more powerful variant where you can remote
+execute via a `remote` resource.
+
 #### Blog post
 
 You can read the introductory blog post about this topic here:
@@ -360,12 +364,6 @@ collision with this globally defined semaphore. The size value must be greater
 than zero at this time. The traditional non-parallel execution found in config
 management tools such as `Puppet` can be obtained with `--sema 1`.
 
-#### `--remote <graph.yaml>`
-
-Point to a graph file to run on the remote host specified within. This parameter
-can be used multiple times if you'd like to remotely run on multiple hosts in
-parallel.
-
 #### `--allow-interactive`
 
 Allow interactive prompting for SSH passwords if there is no authentication
@@ -404,8 +402,8 @@ default prefix. This can't be combined with the `--prefix` option.
 If this option is specified, we will attempt to fall back to a temporary prefix
 if the primary prefix couldn't be created. This is useful for avoiding failures
 in environments where the primary prefix may or may not be available, but you'd
-like to try. The canonical example is when running `mgmt` with `--remote` there
+like to try. The canonical example is when running `mgmt` with remote execution
-might be a cached copy of the binary in the primary prefix, but in case there's
+there might be a cached copy of the binary in the primary prefix, but if there's
 no binary available continue working in a temporary directory to avoid failure.
 
 ### Compilation options
@@ -488,7 +486,7 @@ To report any bugs, please file a ticket at: [https://github.com/purpleidea/mgmt
 
 ## Authors
 
-Copyright (C) 2013-2019+ James Shubin and the project contributors
+Copyright (C) 2013-2021+ James Shubin and the project contributors
 
 Please see the
 [AUTHORS](https://github.com/purpleidea/mgmt/tree/master/AUTHORS) file

docs/faq.md

@@ -53,10 +53,11 @@ find a number of tutorials online.
 3. Spend between four to six hours with the [golang tour](https://tour.golang.org/).
 Skip over the longer problems, but try and get a solid overview of everything.
 If you forget something, you can always go back and repeat those parts.
-4. Connect to our [#mgmtconfig](https://webchat.freenode.net/?channels=#mgmtconfig)
+4. Connect to our [#mgmtconfig](https://web.libera.chat/?channels=#mgmtconfig)
-IRC channel on the [Freenode](https://freenode.net/) network. You can use any
+IRC channel on the [Libera.Chat](https://libera.chat/) network. You can use any
-IRC client that you'd like, but the [hosted web portal](https://webchat.freenode.net/?channels=#mgmtconfig)
+IRC client that you'd like, but the [hosted web portal](https://web.libera.chat/?channels=#mgmtconfig)
-will suffice if you don't know what else to use.
+will suffice if you don't know what else to use. [Here are a few suggestions for
+alternative clients.](https://libera.chat/guides/clients)
 5. Now it's time to try and starting writing a patch! We have tagged a bunch of
 [open issues as #mgmtlove](https://github.com/purpleidea/mgmt/issues?q=is%3Aissue+is%3Aopen+label%3Amgmtlove)
 for new users to have somewhere to get involved. Look through them to see if
@@ -242,7 +243,7 @@ gets created in case it is not present, then you must also specify the state:
 
 ```
 file "/tmp/foo" {
-	state => "exists",
+	state => $const.res.file.state.exists,
 	content => "hello world\n",
 }
 ```
@@ -254,6 +255,13 @@ prevent masking an error for a situation when you expected a file to already be
 at that location. It also turns out to simplify the internals significantly, and
 remove an ambiguous scenario with the reversable file resource.
 
+### Why do function names inside of templates include underscores?
+
+The golang template library which we use to implement the template() function
+doesn't support the dot notation, so we import all our normal functions, and
+just replace dots with underscores. As an example, the standard `datetime.print`
+function is shown within mcl scripts as datetime_print after being imported.
+
 ### On startup `mgmt` hangs after: `etcd: server: starting...`.
 
 If you get an error message similar to:
@@ -287,6 +295,13 @@ an instance of mgmt running, or if a related file locking issue occurred. To
 solve this, shutdown and running mgmt process, run `rm mgmt` to remove the file,
 and then get a new one by running `make` again.
 
+### The docs speaks of `--remote` but the CLI errors out?
+
+The `--remote` flag existed in an earlier version of mgmt. It was removed and
+will be replaced with a more powerful version, which is a "remote" resource. The
+code is mostly ready but it's not finished. If you'd like to help finish it or
+sponsor the work, please let me know.
+
 ### Does this support Windows? OSX? GNU Hurd?
 
 Mgmt probably works best on Linux, because that's what most developers use for
@@ -362,7 +377,7 @@ which definitely existed before the band did.
 
 ### You didn't answer my question, or I have a question!
 
-It's best to ask on [IRC](https://webchat.freenode.net/?channels=#mgmtconfig)
+It's best to ask on [IRC](https://web.libera.chat/?channels=#mgmtconfig)
 to see if someone can help you. If you don't get a response from IRC, you can
 contact me through my [technical blog](https://purpleidea.com/contact/) and I'll
 do my best to help. If you have a good question, please add it as a patch to

docs/function-guide.md

@@ -37,8 +37,10 @@ available types and values in the mgmt language. It is very easy to use, and
 should be fairly intuitive. Most of what you'll need to know can be inferred
 from looking at example code.
 
-To implement a function, you'll need to create a file in
+To implement a function, you'll need to create a file that imports the
-[`lang/funcs/simple/`](https://github.com/purpleidea/mgmt/tree/master/lang/funcs/simple/).
+[`lang/funcs/simple/`](https://github.com/purpleidea/mgmt/tree/master/lang/funcs/simple/)
+module. It should probably get created in the correct directory inside of:
+[`lang/funcs/core/`](https://github.com/purpleidea/mgmt/tree/master/lang/funcs/core/).
 The function should be implemented as a `FuncValue` in our type system. It is
 then registered with the engine during `init()`. An example explains it best:
 
@@ -50,14 +52,15 @@ package simple
 import (
 	"fmt"
 
+	"github.com/purpleidea/mgmt/lang/funcs/simple"
 	"github.com/purpleidea/mgmt/lang/types"
 )
 
 // you must register your functions in init when the program starts up
 func init() {
 	// Example function that squares an int and prints out answer as an str.
-	Register("talkingsquare", &types.FuncValue{
+	simple.ModuleRegister(ModuleName, "talkingsquare", &types.FuncValue{
-		T: types.NewType("func(a int) str"), // declare the signature
+		T: types.NewType("func(int) str"), // declare the signature
 		V: func(input []types.Value) (types.Value, error) {
 			i := input[0].Int() // get first arg as an int64
 			// must return the above specified value
@@ -109,15 +112,20 @@ As with the simple, non-polymorphic API, you can only implement [pure](https://e
 functions, without writing too much boilerplate code. They will be automatically
 re-evaluated as needed when their input values change.
 
-To implement a function, you'll need to create a file in
+To implement a function, you'll need to create a file that imports the
-[`lang/funcs/simplepoly/`](https://github.com/purpleidea/mgmt/tree/master/lang/funcs/simplepoly/).
+[`lang/funcs/simplepoly/`](https://github.com/purpleidea/mgmt/tree/master/lang/funcs/simplepoly/)
+module. It should probably get created in the correct directory inside of:
+[`lang/funcs/core/`](https://github.com/purpleidea/mgmt/tree/master/lang/funcs/core/).
 The function should be implemented as a list of `FuncValue`'s in our type
 system. It is then registered with the engine during `init()`. You may also use
 the `variant` type in your type definitions. This special type will never be
 seen inside a running program, and will get converted to a concrete type if a
 suitable match to this signature can be found. Be warned that signatures which
 contain too many variants, or which are very general, might be hard for the
-compiler to match, and ambiguous type graphs make for user compiler errors.
+compiler to match, and ambiguous type graphs make for user compiler errors. The
+top-level type must still be a function type, it may only contain variants as
+part of its signature. It is probably more difficult to unify a function if its
+return type is a variant, as opposed to if one of its args was.
 
 An example explains it best:
 
@@ -127,11 +135,13 @@ An example explains it best:
 import (
 	"fmt"
 
-	"github.com/purpleidea/mgmt/lang/types"
 	"github.com/purpleidea/mgmt/lang/funcs/simplepoly"
+	"github.com/purpleidea/mgmt/lang/types"
 )
 
 func init() {
+	// You may use the simplepoly.ModuleRegister method to register your
+	// function if it's in a module, as seen in the simple function example.
 	simplepoly.Register("len", []*types.FuncValue{
 		{
 			T: types.NewType("func([]variant) int"),
@@ -190,7 +200,7 @@ if it meets your needs. Most functions will be able to use that API. If you
 really need something more powerful, then you can use the regular function API.
 What follows are each of the method signatures and a description of each.
 
-### Default
+### Info
 
 ```golang
 Info() *interfaces.Info
@@ -435,6 +445,11 @@ generator to build your `FuncValue` implementations, and pass in the unique
 signature to each one as you are building them. Using a generator is a common
 technique which was mentioned previously.
 
+One obvious situation where this might occur is if your function doesn't take
+any inputs! An example `math.fortytwo()` function was implemented that
+demonstrates the use of function generators to pass the type signatures into the
+implementations.
+
 ### Where can I find more information about mgmt?
 
 Additional blog posts, videos and other material [is available!](https://github.com/purpleidea/mgmt/blob/master/docs/on-the-web.md).

docs/language-guide.md

@@ -54,7 +54,7 @@ can be impossible to infer the item's type.
 
 An unordered set of unique keys of the same type and corresponding value pairs
 of another type, eg:
-`{"boiling" => 100, "freezing" => 0, "room" => "25", "house" => 22, "canada" => -30,}`.
+`{"boiling" => 100, "freezing" => 0, "room" => 25, "house" => 22, "canada" => -30,}`.
 That is to say, all of the keys must have the same type, and all of the values
 must have the same type. You can use any type for either, although it is
 probably advisable to avoid using very complex types as map keys.
@@ -206,7 +206,7 @@ value to use if that boolean is true. You can do this with the resource-specific
 $b = true # change me to false and then try editing the file manually
 file "/tmp/mgmt-elvis" {
 	content => $b ?: "hello world\n",
-	state => "exists",
+	state => $const.res.file.state.exists,
 }
 ```
 
@@ -293,7 +293,7 @@ send notifications. You may have multiples of these per resource, including
 multiple `Depend` lines if necessary. Each of these properties also supports the
 conditional inclusion `elvis` operator as well.
 
-For example, you may write is:
+For example, you may write:
 
 ```mcl
 $b = true # for example purposes

docs/on-the-web.md

@@ -52,3 +52,7 @@ if we missed something that you think is relevant!
 | James Shubin | video | [Recording from FOSDEM Containers Devroom 2019](https://video.fosdem.org/2019/UA2.114/containers_mgmt.webm) |
 | James Shubin | video | [Recording from FOSDEM Monitoring Devroom 2019](https://video.fosdem.org/2019/UB2.252A/real_time_merging_of_config_management_and_monitoring.webm) |
 | James Shubin | blog | [Mgmt Configuration Language: Class and Include](https://purpleidea.com/blog/2019/07/26/class-and-include-in-mgmt/) |
+| James Shubin | video | [Recording from FOSDEM 2020, Main Track (History)](https://video.fosdem.org/2020/Janson/automation.webm) |
+| James Shubin | video | [Recording from FOSDEM 2020, Infra Management Devroom](https://video.fosdem.org/2020/UA2.120/mgmt.webm) |
+| James Shubin | video | [Recording from FOSDEM 2020, Minimalistic Languages Devroom](https://video.fosdem.org/2020/AW1.125/mgmtconfigmore.webm) |
+| James Shubin | video | [Recording from CfgMgmtCamp.eu 2020](https://www.youtube.com/watch?v=Kd7FAORFtsc) |

docs/quick-start-guide.md

@@ -37,7 +37,7 @@ You'll need some dependencies, including `golang`, and some associated tools.
 
 #### Installing golang
 
-* You need golang version 1.11 or greater installed.
+* You need golang version 1.13 or greater installed.
 	* To install on rpm style systems: `sudo dnf install golang`
 	* To install on apt style systems: `sudo apt install golang`
 	* To install on macOS systems install [Homebrew](https://brew.sh)

docs/resource-guide.md

@@ -105,7 +105,7 @@ when parameters take a zero value, whenever this is possible.)
 
 ```golang
 // Default returns some sensible defaults for this resource.
-func (obj *FooRes) Default() Res {
+func (obj *FooRes) Default() engine.Res {
 	return &FooRes{
 		Answer: 42, // sometimes, defaults shouldn't be the zero value
 	}
@@ -642,8 +642,8 @@ The signature intentionally matches what is required to satisfy the `go-yaml`
 #### Example
 
 ```golang
-// UnmarshalYAML is the custom unmarshal handler for this struct.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
-// It is primarily useful for setting the defaults.
+// primarily useful for setting the defaults.
 func (obj *FooRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes FooRes // indirection to avoid infinite recursion
 

docs/resources.md

@@ -17,6 +17,7 @@ You might want to look at the [generated documentation](https://godoc.org/github
 for more up-to-date information about these resources.
 
 * [Augeas](#Augeas): Manipulate files using augeas.
+* [Consul:KV](#ConsulKV): Set keys in a Consul datastore.
 * [Docker](#Docker):[Container](#Container) Manage docker containers.
 * [Exec](#Exec): Execute shell commands on the system.
 * [File](#File): Manage files and directories.
@@ -32,6 +33,8 @@ for more up-to-date information about these resources.
 * [Print](#Print): Print messages to the console.
 * [Svc](#Svc): Manage system systemd services.
 * [Test](#Test): A mostly harmless resource that is used for internal testing.
+* [Tftp:File](#TftpFile): Add files to the small embedded embedded tftp server.
+* [Tftp:Server](#TftpServer): Run a small embedded tftp server.
 * [Timer](#Timer): Manage system systemd services.
 * [User](#User): Manage system users.
 * [Virt](#Virt): Manage virtual machines with libvirt.
@@ -71,7 +74,7 @@ It has the following properties:
 * `path`: absolute file path (directories have a trailing slash here)
 * `state`: either `exists`, `absent`, or undefined
 * `content`: raw file content
-* `mode`: octal unix file permissions
+* `mode`: octal unix file permissions or symbolic string
 * `owner`: username or uid for the file owner
 * `group`: group name or gid for the file group
 
@@ -98,6 +101,13 @@ The content property is a string that specifies the desired file contents.
 The source property points to a source file or directory path that we wish to
 copy over and use as the desired contents for our resource.
 
+### Fragments
+
+The fragments property lets you specify a list of files to concatenate together
+to make up the contents of this file. They will be combined in the order that
+they are listed in. If one of the files specified is a directory, then the
+files in that top-level directory will be themselves combined together and used.
+
 ### Recurse
 
 The recurse property limits whether file resource operations should recurse into
@@ -109,6 +119,12 @@ The force property is required if we want the file resource to be able to change
 a file into a directory or vice-versa. If such a change is needed, but the force
 property is not set to `true`, then this file resource will error.
 
+### Purge
+
+The purge property is used when this file represents a directory, and we'd like
+to remove any unmanaged files from within it. Please note that any unmanaged
+files in a directory with this flag set will be irreversibly deleted.
+
 ## Group
 
 The group resource manages the system groups from `/etc/group`.
@@ -211,6 +227,16 @@ The service resource is still very WIP. Please help us by improving it!
 
 The test resource is mostly harmless and is used for internal tests.
 
+## Tftp:File
+
+This adds files to the running tftp server. It's useful because it allows you to
+add individual files without needing to create them on disk.
+
+## Tftp:Server
+
+Run a small embedded tftp server. This doesn't apply any state, but instead runs
+a pure golang tftp server in the Watch loop.
+
 ## Timer
 
 This resource needs better documentation. Please help us by improving it!

docs/style-guide.md

@@ -61,6 +61,12 @@ Occasionally inline, two line source code comments are used within a function.
 These should usually be balanced so that you don't have one line with 78
 characters and the second with only four. Split the comment between the two.
 
+### Default values
+
+Whenever a constant or function parameter is defined, try and have the safer or
+default value be the `zero` value. For example, instead of `const NoDanger`, use
+`const AllowDanger` so that the `false` value is the safe scenario.
+
 ### Method receiver naming
 
 [Contrary](https://github.com/golang/go/wiki/CodeReviewComments#receiver-names)
@@ -84,6 +90,57 @@ func (obj *Foo) Bar(baz string) int {
 }
 ```
 
+### Variable naming
+
+We prefer shorter, scoped variables rather than `unnecessarilyLongIdentifiers`.
+Remember the scoping rules and feel free to use new variables where appropriate.
+For example, in a short string snippet you can use `s` instead of `myString`, as
+well as other common choices. `i` is a common `int` counter, `f` for files, `fn`
+for functions, `x` for something else and so on.
+
+### Variable re-use
+
+Feel free to create and use new variables instead of attempting to re-use the
+same string. For example, if a function input arg is named `s`, you can use a
+new variable to receive the first computation result on `s` instead of storing
+it back into the original `s`. This avoids confusion if a different part of the
+code wants to read the original input, and it avoids any chance of edit by
+reference of the original callers copy of the variable.
+
+#### Example
+
+```golang
+MyNotIdealFunc(s string, b bool) string {
+	if !b {
+		return s + "hey"
+	}
+	s = strings.Replace(s, "blah", "", -1) // not ideal (re-use of `s` var)
+	return s
+}
+
+MyOkayFunc(s string, b bool) string {
+	if !b {
+		return s + "hey"
+	}
+	s2 := strings.Replace(s, "blah", "", -1) // doesn't re-use `s` variable
+	return s2
+}
+
+MyGreatFunc(s string, b bool) string {
+	if !b {
+		return s + "hey"
+	}
+	return strings.Replace(s, "blah", "", -1) // even cleaner
+}
+```
+
+### Constants in code
+
+If a function takes a specifier (often a bool) it's sometimes better to name
+that variable (often with a `const`) rather than leaving a naked `bool` in the
+code. For example, `x := MyFoo("blah", false)` is less clear than
+`const useMagic = false; x := MyFoo("blah", useMagic)`.
+
 ### Consistent ordering
 
 In general we try to preserve a logical ordering in source files which usually
@@ -96,6 +153,23 @@ declared in the interface.
 When implementing code for the various types in the language, please follow this
 order: `bool`, `str`, `int`, `float`, `list`, `map`, `struct`, `func`.
 
+For other aspects where you have a set of items, try to be internally consistent
+as well. For example, if you have two switch statements with `A`, `B`, and `C`,
+please use the same ordering for these elements elsewhere that they appear in
+the code and in the commentary if it is not illogical to do so.
+
+### Product identifiers
+
+Try to avoid references in the code to `mgmt` or a specific program name string
+if possible. This makes it easier to rename code if we ever pick a better name
+or support `libmgmt` better if we embed it. You can use the `Program` variable
+which is available in numerous places if you want a string to put in the logs.
+
+It is also recommended to avoid the `go` (programming language name) string if
+possible. Try to use `golang` if required, since the word `go` is already
+overloaded, and in particular it was even already used by the
+[`go!`](https://en.wikipedia.org/wiki/Go!_(programming_language)).
+
 ## Overview for mcl code
 
 The `mcl` language is quite new, so this guide will probably change over time as

engine/autoedge.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -68,7 +68,8 @@ type AutoEdge interface {
 	Test([]bool) bool // call until false
 }
 
-// ResUID is a unique identifier for a resource, namely it's name, and the kind ("type").
+// ResUID is a unique identifier for a resource, namely it's name, and the kind
+// ("type").
 type ResUID interface {
 	fmt.Stringer // String() string
 
@@ -104,9 +105,9 @@ func (obj *BaseUID) String() string {
 }
 
 // IFF looks at two UID's and if and only if they are equivalent, returns true.
-// If they are not equivalent, it returns false.
+// If they are not equivalent, it returns false. Most resources will want to
-// Most resources will want to override this method, since it does the important
+// override this method, since it does the important work of actually discerning
-// work of actually discerning if two resources are identical in function.
+// if two resources are identical in function.
 func (obj *BaseUID) IFF(uid ResUID) bool {
 	res, ok := uid.(*BaseUID)
 	if !ok {

engine/autoedge_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/autogroup.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -39,7 +39,7 @@ type GroupableRes interface {
 	SetAutoGroupMeta(*AutoGroupMeta)
 
 	// GroupCmp compares two resources and decides if they're suitable for
-	//grouping. This usually needs to be unique to your resource.
+	// grouping. This usually needs to be unique to your resource.
 	GroupCmp(res GroupableRes) error
 
 	// GroupRes groups resource argument (res) into self.

engine/cmp.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/copy.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/edge.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/error.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/fs.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/actions.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/autoedge.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/autoedge/autoedge.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -90,8 +90,8 @@ func AutoEdge(graph *pgraph.Graph, debug bool, logf func(format string, v ...int
 		}
 	}
 
-	// It would be great to ensure we didn't add any loops here, but instead
+	// It would be great to ensure we didn't add any graph cycles here, but
-	// of checking now, we'll move the check into the main loop.
+	// instead of checking now, we'll move the check into the main loop.
 	return nil
 }
 

engine/graph/autogroup.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -74,10 +74,10 @@ func (obj *wrappedGrouper) VertexCmp(v1, v2 pgraph.Vertex) error {
 		return fmt.Errorf("v2 is not a GroupableRes")
 	}
 
-	if r1.Kind() != r2.Kind() { // we must group similar kinds
+	// Some resources of different kinds can now group together!
-		// TODO: maybe future resources won't need this limitation?
+	//if r1.Kind() != r2.Kind() { // we must group similar kinds
-		return fmt.Errorf("the two resources aren't the same kind")
+	//	return fmt.Errorf("the two resources aren't the same kind")
-	}
+	//}
 	// someone doesn't want to group!
 	if r1.AutoGroupMeta().Disabled || r2.AutoGroupMeta().Disabled {
 		return fmt.Errorf("one of the autogroup flags is false")

engine/graph/autogroup/autogroup.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -54,7 +54,7 @@ func AutoGroup(ag engine.AutoGrouper, g *pgraph.Graph, debug bool, logf func(for
 			logf("!VertexMerge for: %s into: %s", wStr, vStr)
 
 		} else { // success!
-			logf("success for: %s into: %s", wStr, vStr)
+			logf("%s into %s", wStr, vStr)
 			merged = true // woo
 		}
 
@@ -66,8 +66,8 @@ func AutoGroup(ag engine.AutoGrouper, g *pgraph.Graph, debug bool, logf func(for
 		}
 	}
 
-	// It would be great to ensure we didn't add any loops here, but instead
+	// It would be great to ensure we didn't add any graph cycles here, but
-	// of checking now, we'll move the check into the main loop.
+	// instead of checking now, we'll move the check into the main loop.
 
 	return nil
 }

engine/graph/autogroup/autogroup_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -595,10 +595,11 @@ func TestPgraphGrouping11(t *testing.T) {
 	runGraphCmp(t, g1, g2)
 }
 
-// simple merge 1
+/* simple merge 1
 // a1   a2         a1,a2
 //   \ /     >>>     |     (arrows point downwards)
 //    b              b
+*/
 func TestPgraphGrouping12(t *testing.T) {
 	g1, _ := pgraph.NewGraph("g1") // original graph
 	{
@@ -620,10 +621,11 @@ func TestPgraphGrouping12(t *testing.T) {
 	runGraphCmp(t, g1, g2)
 }
 
-// simple merge 2
+/* simple merge 2
 //    b              b
 //   / \     >>>     |     (arrows point downwards)
 // a1   a2         a1,a2
+*/
 func TestPgraphGrouping13(t *testing.T) {
 	g1, _ := pgraph.NewGraph("g1") // original graph
 	{
@@ -645,10 +647,11 @@ func TestPgraphGrouping13(t *testing.T) {
 	runGraphCmp(t, g1, g2)
 }
 
-// triple merge
+/* triple merge
 // a1 a2  a3         a1,a2,a3
 //   \ | /     >>>       |      (arrows point downwards)
 //     b                 b
+*/
 func TestPgraphGrouping14(t *testing.T) {
 	g1, _ := pgraph.NewGraph("g1") // original graph
 	{
@@ -673,12 +676,13 @@ func TestPgraphGrouping14(t *testing.T) {
 	runGraphCmp(t, g1, g2)
 }
 
-// chain merge
+/* chain merge
 //    a1             a1
 //   /  \             |
 // b1    b2   >>>   b1,b2   (arrows point downwards)
 //   \  /             |
 //    c1             c1
+*/
 func TestPgraphGrouping15(t *testing.T) {
 	g1, _ := pgraph.NewGraph("g1") // original graph
 	{
@@ -708,7 +712,7 @@ func TestPgraphGrouping15(t *testing.T) {
 	runGraphCmp(t, g1, g2)
 }
 
-// re-attach 1 (outer)
+/* re-attach 1 (outer)
 // technically the second possibility is valid too, depending on which order we
 // merge edges in, and if we don't filter out any unnecessary edges afterwards!
 // a1    a2         a1,a2        a1,a2
@@ -716,6 +720,7 @@ func TestPgraphGrouping15(t *testing.T) {
 // b1  /      >>>    b1     OR    b1  /   (arrows point downwards)
 //  | /               |            | /
 // c1                c1           c1
+*/
 func TestPgraphGrouping16(t *testing.T) {
 	g1, _ := pgraph.NewGraph("g1") // original graph
 	{
@@ -743,12 +748,13 @@ func TestPgraphGrouping16(t *testing.T) {
 	runGraphCmp(t, g1, g2)
 }
 
-// re-attach 2 (inner)
+/* re-attach 2 (inner)
 // a1    b2          a1
 //  |   /             |
 // b1  /      >>>   b1,b2   (arrows point downwards)
 //  | /               |
 // c1                c1
+*/
 func TestPgraphGrouping17(t *testing.T) {
 	g1, _ := pgraph.NewGraph("g1") // original graph
 	{
@@ -776,13 +782,14 @@ func TestPgraphGrouping17(t *testing.T) {
 	runGraphCmp(t, g1, g2)
 }
 
-// re-attach 3 (double)
+/* re-attach 3 (double)
 // similar to "re-attach 1", technically there is a second possibility for this
 // a2   a1    b2         a1,a2
 //   \   |   /             |
 //    \ b1  /      >>>   b1,b2   (arrows point downwards)
 //     \ | /               |
 //      c1                c1
+*/
 func TestPgraphGrouping18(t *testing.T) {
 	g1, _ := pgraph.NewGraph("g1") // original graph
 	{
@@ -813,10 +820,11 @@ func TestPgraphGrouping18(t *testing.T) {
 	runGraphCmp(t, g1, g2)
 }
 
-// connected merge 0, (no change!)
+/* connected merge 0, (no change!)
 // a1            a1
 //   \     >>>     \     (arrows point downwards)
 //    a2            a2
+*/
 func TestPgraphGroupingConnected0(t *testing.T) {
 	g1, _ := pgraph.NewGraph("g1") // original graph
 	{
@@ -835,12 +843,13 @@ func TestPgraphGroupingConnected0(t *testing.T) {
 	runGraphCmp(t, g1, g2)
 }
 
-// connected merge 1, (no change!)
+/* connected merge 1, (no change!)
 // a1              a1
 //   \               \
 //    b      >>>      b      (arrows point downwards)
 //     \               \
 //      a2              a2
+*/
 func TestPgraphGroupingConnected1(t *testing.T) {
 	g1, _ := pgraph.NewGraph("g1") // original graph
 	{

engine/graph/autogroup/base.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -89,6 +89,19 @@ func (ag *baseGrouper) VertexNext() (v1, v2 pgraph.Vertex, err error) {
 			ag.done = true
 		}
 	}
+	// TODO: is this index swap better or even valid?
+	//if ag.i < l {
+	//	ag.i++
+	//}
+	//if ag.i == l {
+	//	ag.i = 0
+	//	if ag.j < l {
+	//		ag.j++
+	//	}
+	//	if ag.j == l {
+	//		ag.done = true
+	//	}
+	//}
 
 	return
 }
@@ -110,7 +123,7 @@ func (ag *baseGrouper) VertexMerge(v1, v2 pgraph.Vertex) (v pgraph.Vertex, err e
 	return nil, fmt.Errorf("vertexMerge needs to be overridden")
 }
 
-// EdgeMerge can be overridden, since it just simple returns the first edge.
+// EdgeMerge can be overridden, since it just simply returns the first edge.
 func (ag *baseGrouper) EdgeMerge(e1, e2 pgraph.Edge) pgraph.Edge {
 	return e1 // noop
 }

engine/graph/autogroup/nonreachability.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/autogroup/util.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -25,9 +25,9 @@ import (
 	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
-// VertexMerge merges v2 into v1 by reattaching the edges where appropriate,
+// VertexMerge merges v2 into v1 by reattaching the edges where appropriate, and
-// and then by deleting v2 from the graph. Since more than one edge between two
+// then by deleting v2 from the graph. Since more than one edge between two
-// vertices is not allowed, duplicate edges are merged as well. an edge merge
+// vertices is not allowed, duplicate edges are merged as well. An edge merge
 // function can be provided if you'd like to control how you merge the edges!
 func VertexMerge(g *pgraph.Graph, v1, v2 pgraph.Vertex, vertexMergeFn func(pgraph.Vertex, pgraph.Vertex) (pgraph.Vertex, error), edgeMergeFn func(pgraph.Edge, pgraph.Edge) pgraph.Edge) error {
 	// methodology

engine/graph/engine.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -191,7 +191,7 @@ func (obj *Engine) Commit() error {
 
 		obj.waits[vertex] = &sync.WaitGroup{}
 		obj.state[vertex] = &State{
-			//Graph: obj.graph, // TODO: what happens if we swap the graph?
+			Graph:  obj.graph, // Update if we swap the graph!
 			Vertex: vertex,
 
 			Program:  obj.Program,
@@ -329,14 +329,14 @@ func (obj *Engine) Commit() error {
 	// the changes that we'd made to the previously primary graph. This is
 	// because this function is meant to atomically swap the graphs safely.
 
-	// TODO: update all the `State` structs with the new Graph pointer
+	// Update all the `State` structs with the new Graph pointer.
-	//for _, vertex := range obj.graph.Vertices() {
+	for _, vertex := range obj.graph.Vertices() {
-	//	state, exists := obj.state[vertex]
+		state, exists := obj.state[vertex]
-	//	if !exists {
+		if !exists {
-	//		continue
+			continue
-	//	}
+		}
-	//	state.Graph = obj.graph // update pointer to graph
+		state.Graph = obj.graph // update pointer to graph
-	//}
+	}
 
 	return nil
 }

engine/graph/graph_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/refresh.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/reverse.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -150,8 +150,8 @@ func (obj *Engine) Reversals() error {
 	}
 	// TODO: Do we want a way for stored reversals to add edges too?
 
-	// It would be great to ensure we didn't add any loops here, but instead
+	// It would be great to ensure we didn't add any graph cycles here, but
-	// of checking now, we'll move the check into the main loop.
+	// instead of checking now, we'll move the check into the main loop.
 	return nil
 }
 
@@ -291,5 +291,10 @@ func (obj *State) ReversalDelete() error {
 	}
 	file := path.Join(dir, ReverseFile) // return a unique file
 
-	return errwrap.Wrapf(os.Remove(file), "could not remove reverse state file")
+	// FIXME: why do we see these removals when there isn't a state file?
+	if err = os.Remove(file); os.IsNotExist(err) {
+		return nil // ignore missing files
+	}
+
+	return errwrap.Wrapf(err, "could not remove reverse state file")
 }

engine/graph/semaphore.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/semaphore_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/sendrecv.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/state.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -32,7 +32,7 @@ import (
 // State stores some state about the resource it is mapped to.
 type State struct {
 	// Graph is a pointer to the graph that this vertex is part of.
-	//Graph pgraph.Graph
+	Graph *pgraph.Graph
 
 	// Vertex is the pointer in the graph that this state corresponds to. It
 	// can be converted to a `Res` if necessary.
@@ -169,25 +169,63 @@ func (obj *State) Init() error {
 			}
 			return res.Refresh()
 		},
-		Send: func(st interface{}) error {
-			res, ok := obj.Vertex.(engine.SendableRes)
-			if !ok {
-				panic("res does not support the Sendable trait")
-			}
-			// XXX: type check this
-			//expected := res.Sends()
-			//if err := XXX_TYPE_CHECK(expected, st); err != nil {
-			//	return err
-			//}
 
-			return res.Send(st) // send the struct
+		Send: engine.GenerateSendFunc(res),
-		},
+		Recv: engine.GenerateRecvFunc(res),
-		Recv: func() map[string]*engine.Send { // TODO: change this API?
+
-			res, ok := obj.Vertex.(engine.RecvableRes)
+		// FIXME: pass in a safe, limited query func instead?
-			if !ok {
+		// TODO: not implemented, use FilteredGraph
-				panic("res does not support the Recvable trait")
+		//Graph: func() *pgraph.Graph {
+		//	_, ok := obj.Vertex.(engine.CanGraphQueryRes)
+		//	if !ok {
+		//		panic("res does not support the GraphQuery trait")
+		//	}
+		//	return obj.Graph // we return in a func so it's fresh!
+		//},
+
+		FilteredGraph: func() (*pgraph.Graph, error) {
+			graph, err := pgraph.NewGraph("filtered")
+			if err != nil {
+				return nil, errwrap.Wrapf(err, "could not create graph")
 			}
-			return res.Recv()
+
+			// filter graph and build a new one...
+			adjacency := obj.Graph.Adjacency()
+			for v1 := range adjacency {
+				// check we're allowed
+				r1, ok := v1.(engine.GraphQueryableRes)
+				if !ok {
+					continue
+				}
+				// pass in information on requestor...
+				if err := r1.GraphQueryAllowed(
+					engine.GraphQueryableOptionKind(res.Kind()),
+					engine.GraphQueryableOptionName(res.Name()),
+					// TODO: add more information...
+				); err != nil {
+					continue
+				}
+				graph.AddVertex(v1)
+
+				for v2, edge := range adjacency[v1] {
+					r2, ok := v2.(engine.GraphQueryableRes)
+					if !ok {
+						continue
+					}
+					// pass in information on requestor...
+					if err := r2.GraphQueryAllowed(
+						engine.GraphQueryableOptionKind(res.Kind()),
+						engine.GraphQueryableOptionName(res.Name()),
+						// TODO: add more information...
+					); err != nil {
+						continue
+					}
+					//graph.AddVertex(v2) // redundant
+					graph.AddEdge(v1, v2, edge)
+				}
+			}
+
+			return graph, nil // we return in a func so it's fresh!
 		},
 
 		World:  obj.World,

engine/graph/vardir.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graphqueryable.go

@@ -0,0 +1,70 @@
+// Mgmt
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package engine
+
+// GraphQueryableRes is the interface that must be implemented if you want your
+// resource to be allowed to be queried from another resource in the graph. This
+// is done as a form of explicit authorization tracking so that we can consider
+// security aspects more easily. Ultimately, all resource code should be
+// trusted, but it's still a good idea to know if a particular resource is even
+// able to access information about another one, and if your resource doesn't
+// add the trait supporting this, then it won't be allowed.
+type GraphQueryableRes interface {
+	Res // implement everything in Res but add the additional requirements
+
+	// GraphQueryAllowed returns nil if you're allowed to query the graph.
+	GraphQueryAllowed(...GraphQueryableOption) error
+}
+
+// GraphQueryableOption is an option that can be used to specify the
+// authentication.
+type GraphQueryableOption func(*GraphQueryableOptions)
+
+// GraphQueryableOptions represents the different possible configurable options.
+type GraphQueryableOptions struct {
+	// Kind is the kind of the resource making the access.
+	Kind string
+	// Name is the name of the resource making the access.
+	Name string
+	// TODO: add more options if needed
+}
+
+// Apply is a helper function to apply a list of options to the struct. You
+// should initialize it with defaults you want, and then apply any you've
+// received like this.
+func (obj *GraphQueryableOptions) Apply(opts ...GraphQueryableOption) {
+	for _, optionFunc := range opts { // apply the options
+		optionFunc(obj)
+	}
+}
+
+// GraphQueryableOptionKind tells the GraphQueryAllowed function what the
+// resource kind is.
+func GraphQueryableOptionKind(kind string) GraphQueryableOption {
+	return func(gqo *GraphQueryableOptions) {
+		gqo.Kind = kind
+	}
+}
+
+// GraphQueryableOptionName tells the GraphQueryAllowed function what the
+// resource name is.
+func GraphQueryableOptionName(name string) GraphQueryableOption {
+	return func(gqo *GraphQueryableOptions) {
+		gqo.Name = name
+	}
+}

engine/metaparams.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/metaparams_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/refresh.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/resources.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -21,6 +21,7 @@ import (
 	"encoding/gob"
 	"fmt"
 
+	"github.com/purpleidea/mgmt/pgraph"
 	"github.com/purpleidea/mgmt/util/errwrap"
 
 	"gopkg.in/yaml.v2"
@@ -29,8 +30,8 @@ import (
 // TODO: should each resource be a sub-package?
 var registeredResources = map[string]func() Res{}
 
-// RegisterResource registers a new resource by providing a constructor
+// RegisterResource registers a new resource by providing a constructor function
-// function that returns a resource object ready to be unmarshalled from YAML.
+// that returns a resource object ready to be unmarshalled from YAML.
 func RegisterResource(kind string, fn func() Res) {
 	f := fn()
 	if kind == "" {
@@ -120,6 +121,20 @@ type Init struct {
 
 	// Other functionality:
 
+	// Graph is a function that returns the current graph. The returned
+	// value won't be valid after a graphsync so make sure to call this when
+	// you are about to use it, and discard it right after.
+	// FIXME: it might be better to offer a safer, more limited, GraphQuery?
+	//Graph func() *pgraph.Graph // TODO: not implemented, use FilteredGraph
+
+	// FilteredGraph is a function that returns a filtered variant of the
+	// current graph. Only resource that have allowed themselves to be added
+	// into this graph will appear. If they did not consent, then those
+	// vertices and any associated edges, will not be present.
+	FilteredGraph func() (*pgraph.Graph, error)
+
+	// TODO: GraphQuery offers an interface to query the resource graph.
+
 	// World provides a connection to the outside world. This is most often
 	// used for communicating with the distributed database.
 	World World
@@ -227,8 +242,8 @@ func Validate(res Res) error {
 // the Interrupt method to shutdown the resource quickly. Running this method
 // may leave the resource in a partial state, however this may be desired if you
 // want a faster exit or if you'd prefer a partial state over letting the
-// resource complete in a situation where you made an error and you wish to
+// resource complete in a situation where you made an error and you wish to exit
-// exit quickly to avoid data loss. It is usually triggered after multiple ^C
+// quickly to avoid data loss. It is usually triggered after multiple ^C
 // signals.
 type InterruptableRes interface {
 	Res

engine/resources/augeas.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -124,8 +124,8 @@ func (obj *AugeasRes) Close() error {
 	return nil
 }
 
-// Watch is the primary listener for this resource and it outputs events.
+// Watch is the primary listener for this resource and it outputs events. This
-// Taken from the File resource.
+// was taken from the File resource.
 // FIXME: DRY - This is taken from the file resource
 func (obj *AugeasRes) Watch() error {
 	var err error
@@ -301,8 +301,8 @@ func (obj *AugeasRes) UIDs() []engine.ResUID {
 	return []engine.ResUID{x}
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
-// It is primarily useful for setting the defaults.
+// primarily useful for setting the defaults.
 func (obj *AugeasRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes AugeasRes // indirection to avoid infinite recursion
 

engine/resources/aws_ec2.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -121,8 +121,8 @@ const (
 )
 
 // AwsRegions is a list of all AWS regions generated using ec2.DescribeRegions.
-// cn-north-1 and us-gov-west-1 are not returned, probably due to security.
+// cn-north-1 and us-gov-west-1 are not returned, probably due to security. List
-// List available at http://docs.aws.amazon.com/general/latest/gr/rande.html
+// available at http://docs.aws.amazon.com/general/latest/gr/rande.html
 var AwsRegions = []string{
 	"ap-northeast-1",
 	"ap-northeast-2",
@@ -187,7 +187,8 @@ type AwsEc2Res struct {
 	InstanceID  string
 }
 
-// chanStruct defines the type for a channel used to pass events and errors to watch.
+// chanStruct defines the type for a channel used to pass events and errors to
+// watch.
 type chanStruct struct {
 	event awsEc2Event
 	state string
@@ -233,7 +234,8 @@ type ruleDetail struct {
 	State []string `json:"state"`
 }
 
-// postData is the format of the messages received and decoded by snsPostHandler().
+// postData is the format of the messages received and decoded by
+// snsPostHandler().
 type postData struct {
 	Type             string `json:"Type"`
 	MessageID        string `json:"MessageId"`
@@ -247,7 +249,8 @@ type postData struct {
 	SigningCertURL   string `json:"SigningCertURL"`
 }
 
-// postMsg is used to unmarshal the postData message if it's an event notification.
+// postMsg is used to unmarshal the postData message if it's an event
+// notification.
 type postMsg struct {
 	InstanceID string `json:"instance-id"`
 	State      string `json:"state"`
@@ -413,7 +416,8 @@ func (obj *AwsEc2Res) Watch() error {
 	return obj.longpollWatch()
 }
 
-// longpollWatch uses the ec2 api's built in methods to watch ec2 resource state.
+// longpollWatch uses the ec2 api's built in methods to watch ec2 resource
+// state.
 func (obj *AwsEc2Res) longpollWatch() error {
 	send := false
 
@@ -510,10 +514,10 @@ func (obj *AwsEc2Res) longpollWatch() error {
 }
 
 // snsWatch uses amazon's SNS and CloudWatchEvents APIs to get instance state-
-// change notifications pushed to the http endpoint (snsServer) set up below.
+// change notifications pushed to the http endpoint (snsServer) set up below. In
-// In Init() a CloudWatch rule is created along with a corresponding SNS topic
+// Init() a CloudWatch rule is created along with a corresponding SNS topic that
-// that it can publish to. snsWatch creates an http server which listens for
+// it can publish to. snsWatch creates an http server which listens for messages
-// messages published to the topic and processes them accordingly.
+// published to the topic and processes them accordingly.
 func (obj *AwsEc2Res) snsWatch() error {
 	send := false
 	defer obj.wg.Wait()
@@ -795,8 +799,8 @@ type AwsEc2UID struct {
 	name string
 }
 
-// UIDs includes all params to make a unique identification of this object.
+// UIDs includes all params to make a unique identification of this object. Most
-// Most resources only return one, although some resources can return multiple.
+// resources only return one, although some resources can return multiple.
 func (obj *AwsEc2Res) UIDs() []engine.ResUID {
 	x := &AwsEc2UID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
@@ -805,8 +809,8 @@ func (obj *AwsEc2Res) UIDs() []engine.ResUID {
 	return []engine.ResUID{x}
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
-// It is primarily useful for setting the defaults.
+// primarily useful for setting the defaults.
 func (obj *AwsEc2Res) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes AwsEc2Res // indirection to avoid infinite recursion
 
@@ -942,8 +946,8 @@ func (obj *AwsEc2Res) snsVerifySignature(post postData) error {
 	return nil
 }
 
-// snsGetCert downloads and parses the signing certificate from the provided
+// snsGetCert downloads and parses the signing certificate from the provided URL
-// URL for message verification.
+// for message verification.
 func (obj *AwsEc2Res) snsGetCert(url string) (*x509.Certificate, error) {
 	// only download valid certificates from amazon
 	matchURL, err := regexp.MatchString(SnsCertURLRegex, url)
@@ -1035,8 +1039,8 @@ func (obj *AwsEc2Res) snsDeleteTopic(topicArn string) error {
 	return nil
 }
 
-// snsSubscribe subscribes the endpoint to the sns topic.
+// snsSubscribe subscribes the endpoint to the sns topic. Returning
-// Returning SubscriptionArn here is useless as it is still pending confirmation.
+// SubscriptionArn here is useless as it is still pending confirmation.
 func (obj *AwsEc2Res) snsSubscribe(endpoint string, topicArn string) error {
 	// subscribe to the topic
 	subInput := &sns.SubscribeInput{
@@ -1052,8 +1056,8 @@ func (obj *AwsEc2Res) snsSubscribe(endpoint string, topicArn string) error {
 	return nil
 }
 
-// snsConfirmSubscription confirms the sns subscription.
+// snsConfirmSubscription confirms the sns subscription. Returning
-// Returning SubscriptionArn here is useless as it is still pending confirmation.
+// SubscriptionArn here is useless as it is still pending confirmation.
 func (obj *AwsEc2Res) snsConfirmSubscription(topicArn string, token string) error {
 	// confirm the subscription
 	csInput := &sns.ConfirmSubscriptionInput{
@@ -1105,7 +1109,8 @@ func (obj *AwsEc2Res) snsProcessEvent(message, instanceName string) (awsEc2Event
 	return awsEc2EventNone, nil
 }
 
-// snsAuthorize adds the necessary permission for cloudwatch to publish to the SNS topic.
+// snsAuthorize adds the necessary permission for cloudwatch to publish to the
+// SNS topic.
 func (obj *AwsEc2Res) snsAuthorizeCloudWatch(topicArn string) error {
 	// get the topic attributes, including the security policy
 	gaInput := &sns.GetTopicAttributesInput{

engine/resources/config_etcd.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -229,8 +229,8 @@ func (obj *ConfigEtcdRes) Interrupt() error {
 	return nil
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
-// It is primarily useful for setting the defaults.
+// primarily useful for setting the defaults.
 func (obj *ConfigEtcdRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes ConfigEtcdRes // indirection to avoid infinite recursion
 

engine/resources/consul_kv.go

@@ -0,0 +1,283 @@
+// Mgmt
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"sync"
+
+	"github.com/purpleidea/mgmt/engine"
+	"github.com/purpleidea/mgmt/engine/traits"
+	"github.com/purpleidea/mgmt/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
+
+	"github.com/hashicorp/consul/api"
+)
+
+func init() {
+	engine.RegisterResource("consul:kv", func() engine.Res { return &ConsulKVRes{} })
+}
+
+// ConsulKVRes is a resource that writes a value into a Consul datastore. The
+// name of the resource can either be the key name, or the concatenation of the
+// server address and the key name: http://127.0.0.1:8500/my-key. If the param
+// keys are specified, then those are used. If the Name cannot be properly
+// parsed by url.Parse, then it will be considered as the Key's value. If the
+// Key is specified explicitly, then we won't use anything from the Name.
+type ConsulKVRes struct {
+	traits.Base
+	init *engine.Init
+
+	// Key is the name of the key. Defaults to the name of the resource.
+	Key string `lang:"key" yaml:"key"`
+
+	// Value is the value for the key.
+	Value string `lang:"value" yaml:"value"`
+
+	// Scheme is the URI scheme for the Consul server. Default: http.
+	Scheme string `lang:"scheme" yaml:"scheme"`
+
+	// Address is the address of the Consul server. Default: 127.0.0.1:8500.
+	Address string `lang:"address" yaml:"address"`
+
+	// Token is used to provide an ACL token to use for this resource.
+	Token string `lang:"token" yaml:"token"`
+
+	client *api.Client
+	config *api.Config // needed to close the idle connections
+	once   bool        // safety token
+	key    string      // cache the key name to avoid re-running the parser
+}
+
+// Default returns some sensible defaults for this resource.
+func (obj *ConsulKVRes) Default() engine.Res {
+	return &ConsulKVRes{}
+}
+
+// Validate if the params passed in are valid data.
+func (obj *ConsulKVRes) Validate() error {
+	s, _, k := obj.inputParser()
+	if k == "" {
+		return fmt.Errorf("the Key is empty")
+	}
+	if s != "" && s != "http" && s != "https" {
+		return fmt.Errorf("unknown Scheme")
+	}
+	return nil
+}
+
+// Init runs some startup code for this resource.
+func (obj *ConsulKVRes) Init(init *engine.Init) error {
+	obj.init = init // save for later
+
+	s, a, k := obj.inputParser()
+
+	obj.config = api.DefaultConfig()
+	if s != "" {
+		obj.config.Scheme = s
+	}
+	if a != "" {
+		obj.config.Address = obj.Address
+	}
+	obj.key = k // store the key
+	obj.init.Logf("using consul key: %s", obj.key)
+
+	if obj.Token != "" {
+		obj.config.Token = obj.Token
+	}
+
+	var err error
+	obj.client, err = api.NewClient(obj.config)
+	return errwrap.Wrapf(err, "could not create Consul client")
+}
+
+// Close is run by the engine to clean up after the resource is done.
+func (obj *ConsulKVRes) Close() error {
+	if obj.config != nil && obj.config.Transport != nil {
+		obj.config.Transport.CloseIdleConnections()
+	}
+	return nil
+}
+
+// Watch is the listener and main loop for this resource and it outputs events.
+func (obj *ConsulKVRes) Watch() error {
+	wg := &sync.WaitGroup{}
+	defer wg.Wait()
+
+	ch := make(chan error)
+	exit := make(chan struct{})
+
+	kv := obj.client.KV()
+
+	wg.Add(1)
+	go func() {
+		defer close(ch)
+		defer wg.Done()
+
+		opts := &api.QueryOptions{RequireConsistent: true}
+		ctx, cancel := util.ContextWithCloser(context.Background(), exit)
+		defer cancel()
+		opts = opts.WithContext(ctx)
+
+		for {
+			_, meta, err := kv.Get(obj.key, opts)
+			select {
+			case ch <- err: // send
+				if err != nil {
+					return
+				}
+
+				// WaitIndex = 0, which means that it is the
+				// first time we run the query, as we are about
+				// to change the WaitIndex to make a blocking
+				// query, we can consider the watch started.
+				opts.WaitIndex = meta.LastIndex
+				if opts.WaitIndex != 0 {
+					continue
+				}
+
+				if !obj.once {
+					obj.init.Running()
+					obj.once = true
+					continue
+				}
+
+				// Unexpected situation, bug in consul API...
+				select {
+				case ch <- fmt.Errorf("unexpected behaviour in Consul API"):
+				case <-obj.init.Done: // signal for shutdown request
+				}
+
+			case <-obj.init.Done: // signal for shutdown request
+			}
+			return
+		}
+	}()
+
+	defer close(exit)
+	for {
+		select {
+		case err, ok := <-ch:
+			if !ok { // channel shutdown
+				return nil
+			}
+			if err != nil {
+				return errwrap.Wrapf(err, "unknown %s watcher error", obj)
+			}
+			if obj.init.Debug {
+				obj.init.Logf("event!")
+			}
+			obj.init.Event()
+
+		case <-obj.init.Done: // signal for shutdown request
+			return nil
+		}
+	}
+}
+
+// CheckApply is run to check the state and, if apply is true, to apply the
+// necessary changes to reach the desired state. This is run before Watch and
+// again if Watch finds a change occurring to the state.
+func (obj *ConsulKVRes) CheckApply(apply bool) (bool, error) {
+	if obj.init.Debug {
+		obj.init.Logf("consul key: %s", obj.key)
+	}
+	kv := obj.client.KV()
+	pair, _, err := kv.Get(obj.key, nil)
+	if err != nil {
+		return false, err
+	}
+
+	if pair != nil && string(pair.Value) == obj.Value {
+		return true, nil
+	}
+
+	if !apply {
+		return false, nil
+	}
+
+	p := &api.KVPair{Key: obj.key, Value: []byte(obj.Value)}
+	_, err = kv.Put(p, nil)
+	return false, err
+}
+
+// Cmp compares two resources and return if they are equivalent.
+func (obj *ConsulKVRes) Cmp(r engine.Res) error {
+	res, ok := r.(*ConsulKVRes)
+	if !ok {
+		return fmt.Errorf("not a %s", obj.Kind())
+	}
+
+	if obj.Key != res.Key {
+		return fmt.Errorf("the Key param differs")
+	}
+	if obj.Value != res.Value {
+		return fmt.Errorf("the Value param differs")
+	}
+	if obj.Scheme != res.Scheme {
+		return fmt.Errorf("the Scheme param differs")
+	}
+	if obj.Address != res.Address {
+		return fmt.Errorf("the Address param differs")
+	}
+	if obj.Token != res.Token {
+		return fmt.Errorf("the Token param differs")
+	}
+
+	return nil
+}
+
+// inputParser parses the Name() of a resource and extracts the scheme, address,
+// and key name of a consul key. We don't have an error, because if we have one,
+// then it means the input must be a raw key. Output of this function is scheme,
+// address (includes hostname and port), and key. This also takes our parameters
+// in to account, and applies the correct overrides if they are specified there.
+func (obj *ConsulKVRes) inputParser() (string, string, string) {
+	// If the key is specified explicitly, then we're not going to parse the
+	// resource name for a pattern, and we use our given params as they are.
+	if obj.Key != "" {
+		return obj.Scheme, obj.Address, obj.Key
+	}
+
+	// Now we parse...
+	u, err := url.Parse(obj.Name())
+	if err != nil {
+		// If this didn't work, then we know it's explicitly a raw key.
+		return obj.Scheme, obj.Address, obj.Name()
+	}
+
+	// Otherwise, we use the parse result, and we overwrite any of the
+	// fields if we have an explicit param that was specified.
+	k := u.Path
+	s := u.Scheme
+	a := u.Host
+
+	//if obj.Key != "" { // this is now guaranteed to never happen
+	//	k = obj.Key
+	//}
+	if obj.Scheme != "" {
+		s = obj.Scheme
+	}
+	if obj.Address != "" {
+		a = obj.Address
+	}
+
+	return s, a, k
+}

engine/resources/consul_kv_test.go

@@ -0,0 +1,71 @@
+// Mgmt
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/purpleidea/mgmt/engine"
+)
+
+func createConsulRes(name string) *ConsulKVRes {
+	r, err := engine.NewNamedResource("consul:kv", name)
+	if err != nil {
+		panic(fmt.Sprintf("could not create resource: %+v", err))
+	}
+
+	res := r.(*ConsulKVRes) // if this panics, the test will panic
+	return res
+}
+
+func TestParseConsulName(t *testing.T) {
+	n1 := "test"
+	r1 := createConsulRes(n1)
+	if s, a, k := r1.inputParser(); s != "" || a != "" || k != "test" {
+		t.Errorf("unexpected output while parsing `%s`: %s, %s, %s", n1, s, a, k)
+	}
+
+	n2 := "http://127.0.0.1:8500/test"
+	r2 := createConsulRes(n2)
+	if s, a, k := r2.inputParser(); s != "http" || a != "127.0.0.1:8500" || k != "/test" {
+		t.Errorf("unexpected output while parsing `%s`: %s, %s, %s", n2, s, a, k)
+	}
+
+	n3 := "http://127.0.0.1:8500/test"
+	r3 := createConsulRes(n3)
+	r3.Scheme = "https"
+	r3.Address = "example.com"
+	if s, a, k := r3.inputParser(); s != "https" || a != "example.com" || k != "/test" {
+		t.Errorf("unexpected output while parsing `%s`: %s, %s, %s", n3, s, a, k)
+	}
+
+	n4 := "http:://127.0.0.1..5:8500/test" // wtf, url.Parse is on drugs...
+	r4 := createConsulRes(n4)
+	//if s, a, k := r4.inputParser(); s != "" || a != "" || k != n4 { // what i really expect
+	if s, a, k := r4.inputParser(); s != "http" || a != "" || k != "" { // what i get
+		t.Errorf("unexpected output while parsing `%s`: %s, %s, %s", n4, s, a, k)
+	}
+
+	n5 := "http://127.0.0.1:8500/test" // whatever, it's ignored
+	r5 := createConsulRes(n3)
+	r5.Key = "some key"
+	if s, a, k := r5.inputParser(); s != "" || a != "" || k != "some key" {
+		t.Errorf("unexpected output while parsing `%s`: %s, %s, %s", n5, s, a, k)
+	}
+}

engine/resources/cron.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -139,9 +139,9 @@ func (obj *CronRes) Default() engine.Res {
 	}
 }
 
-// makeComposite creates a pointer to a FileRes. The pointer is used to
+// makeComposite creates a pointer to a FileRes. The pointer is used to validate
-// validate and initialize the nested file resource and to apply the file state
+// and initialize the nested file resource and to apply the file state in
-// in CheckApply.
+// CheckApply.
 func (obj *CronRes) makeComposite() (*FileRes, error) {
 	p, err := obj.UnitFilePath()
 	if err != nil {
@@ -466,8 +466,8 @@ func (obj *CronRes) AutoEdges() (engine.AutoEdge, error) {
 	return nil, nil
 }
 
-// UIDs includes all params to make a unique identification of this object.
+// UIDs includes all params to make a unique identification of this object. Most
-// Most resources only return one although some resources can return multiple.
+// resources only return one although some resources can return multiple.
 func (obj *CronRes) UIDs() []engine.ResUID {
 	unit := fmt.Sprintf("%s.service", obj.Name())
 	if obj.Unit != "" {
@@ -486,8 +486,8 @@ func (obj *CronRes) UIDs() []engine.ResUID {
 	return uids
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
-// It is primarily useful for setting the defaults.
+// primarily useful for setting the defaults.
 func (obj *CronRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes CronRes // indirection to avoid infinite recursion
 

engine/resources/docker_container.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -50,8 +50,8 @@ const (
 	// initCtxTimeout is the length of time, in seconds, before requests are
 	// cancelled in Init.
 	initCtxTimeout = 20
-	// checkApplyCtxTimeout is the length of time, in seconds, before requests
+	// checkApplyCtxTimeout is the length of time, in seconds, before
-	// are cancelled in CheckApply.
+	// requests are cancelled in CheckApply.
 	checkApplyCtxTimeout = 120
 )
 
@@ -74,11 +74,12 @@ type DockerContainerRes struct {
 	Env []string `yaml:"env"`
 	// Ports is a map of port bindings. E.g. {"tcp" => {80 => 8080},}.
 	Ports map[string]map[int64]int64 `yaml:"ports"`
-	// APIVersion allows you to override the host's default client API version.
+	// APIVersion allows you to override the host's default client API
+	// version.
 	APIVersion string `yaml:"apiversion"`
 
-	// Force, if true, will destroy and redeploy the container if the image is
+	// Force, if true, this will destroy and redeploy the container if the
-	// incorrect.
+	// image is incorrect.
 	Force bool `yaml:"force"`
 
 	client *client.Client // docker api client
@@ -88,7 +89,9 @@ type DockerContainerRes struct {
 
 // Default returns some sensible defaults for this resource.
 func (obj *DockerContainerRes) Default() engine.Res {
-	return &DockerContainerRes{}
+	return &DockerContainerRes{
+		State: "running",
+	}
 }
 
 // Validate if the params passed in are valid data.
@@ -98,6 +101,11 @@ func (obj *DockerContainerRes) Validate() error {
 		return fmt.Errorf("state must be running, stopped or removed")
 	}
 
+	// make sure an image is specified
+	if obj.Image == "" {
+		return fmt.Errorf("image must be specified")
+	}
+
 	// validate env
 	for _, env := range obj.Env {
 		if !strings.Contains(env, "=") || strings.Contains(env, " ") {
@@ -140,7 +148,7 @@ func (obj *DockerContainerRes) Init(init *engine.Init) error {
 	defer cancel()
 
 	// Initialize the docker client.
-	obj.client, err = client.NewClient(client.DefaultDockerHost, obj.APIVersion, nil, nil)
+	obj.client, err = client.NewClientWithOpts(client.WithVersion(obj.APIVersion))
 	if err != nil {
 		return errwrap.Wrapf(err, "error creating docker client")
 	}
@@ -302,7 +310,7 @@ func (obj *DockerContainerRes) CheckApply(apply bool) (bool, error) {
 			}
 		}
 
-		c, err := obj.client.ContainerCreate(ctx, containerConfig, hostConfig, nil, obj.Name())
+		c, err := obj.client.ContainerCreate(ctx, containerConfig, hostConfig, nil, nil, obj.Name())
 		if err != nil {
 			return false, errwrap.Wrapf(err, "error creating container")
 		}
@@ -367,52 +375,105 @@ func (obj *DockerContainerRes) Cmp(r engine.Res) error {
 	if !ok {
 		return fmt.Errorf("error casting r to *DockerContainerRes")
 	}
-	if obj.Name() != res.Name() {
+
-		return fmt.Errorf("names differ")
+	if obj.State != res.State {
+		return fmt.Errorf("the State differs")
+	}
+	if obj.Image != res.Image {
+		return fmt.Errorf("the Image differs")
 	}
 	if err := util.SortedStrSliceCompare(obj.Cmd, res.Cmd); err != nil {
-		return errwrap.Wrapf(err, "cmd differs")
+		return errwrap.Wrapf(err, "the Cmd field differs")
 	}
 	if err := util.SortedStrSliceCompare(obj.Env, res.Env); err != nil {
-		return errwrap.Wrapf(err, "env differs")
+		return errwrap.Wrapf(err, "tne Env field differs")
 	}
 	if len(obj.Ports) != len(res.Ports) {
-		return fmt.Errorf("ports length differs")
+		return fmt.Errorf("the Ports length differs")
 	}
 	for k, v := range obj.Ports {
 		for p, q := range v {
 			if w, ok := res.Ports[k][p]; !ok || q != w {
-				return fmt.Errorf("ports differ")
+				return fmt.Errorf("the Ports field differs")
 			}
 		}
 	}
 	if obj.APIVersion != res.APIVersion {
-		return fmt.Errorf("apiversions differ")
+		return fmt.Errorf("the APIVersion differs")
 	}
 	if obj.Force != res.Force {
-		return fmt.Errorf("forces differ")
+		return fmt.Errorf("the Force field differs")
 	}
 	return nil
 }
 
-// DockerUID is the UID struct for DockerContainerRes.
+// DockerContainerUID is the UID struct for DockerContainerRes.
-type DockerUID struct {
+type DockerContainerUID struct {
 	engine.BaseUID
+
 	name string
 }
 
-// UIDs includes all params to make a unique identification of this object.
+// DockerContainerResAutoEdges holds the state of the auto edge generator.
-// Most resources only return one, although some resources can return multiple.
+type DockerContainerResAutoEdges struct {
+	UIDs    []engine.ResUID
+	pointer int
+}
+
+// AutoEdges returns edges to any docker:image resource that matches the image
+// specified in the docker:container resource definition.
+func (obj *DockerContainerRes) AutoEdges() (engine.AutoEdge, error) {
+	var result []engine.ResUID
+	var reversed bool
+	if obj.State != "removed" {
+		reversed = true
+	}
+	result = append(result, &DockerImageUID{
+		BaseUID: engine.BaseUID{
+			Reversed: &reversed,
+		},
+		image: dockerImageNameTag(obj.Image),
+	})
+	return &DockerContainerResAutoEdges{
+		UIDs:    result,
+		pointer: 0,
+	}, nil
+}
+
+// Next returnes the next automatic edge.
+func (obj *DockerContainerResAutoEdges) Next() []engine.ResUID {
+	if len(obj.UIDs) == 0 {
+		return nil
+	}
+	value := obj.UIDs[obj.pointer]
+	obj.pointer++
+	return []engine.ResUID{value}
+}
+
+// Test gets results of the earlier Next() call, & returns if we should
+// continue.
+func (obj *DockerContainerResAutoEdges) Test(input []bool) bool {
+	if len(obj.UIDs) <= obj.pointer {
+		return false
+	}
+	if len(input) != 1 { // in case we get given bad data
+		panic(fmt.Sprintf("Expecting a single value!"))
+	}
+	return true // keep going
+}
+
+// UIDs includes all params to make a unique identification of this object. Most
+// resources only return one, although some resources can return multiple.
 func (obj *DockerContainerRes) UIDs() []engine.ResUID {
-	x := &DockerUID{
+	x := &DockerContainerUID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
 		name:    obj.Name(),
 	}
 	return []engine.ResUID{x}
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
-// It is primarily useful for setting the defaults.
+// primarily useful for setting the defaults.
 func (obj *DockerContainerRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes DockerContainerRes // indirection to avoid infinite recursion
 

engine/resources/docker_container_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -165,6 +165,7 @@ func setup() error {
 		},
 		&container.HostConfig{},
 		nil,
+		nil,
 		"mgmt-test",
 	)
 	if err != nil {

engine/resources/docker_image.go

@@ -0,0 +1,295 @@
+// Mgmt
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// +build !nodocker
+
+package resources
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/purpleidea/mgmt/engine"
+	"github.com/purpleidea/mgmt/engine/traits"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/client"
+	errwrap "github.com/pkg/errors"
+)
+
+const (
+	// dockerImageInitCtxTimeout is the length of time, in seconds, before
+	// requests are cancelled in Init.
+	dockerImageInitCtxTimeout = 20
+	// dockerImageCheckApplyCtxTimeout is the length of time, in seconds,
+	// before requests are cancelled in CheckApply.
+	dockerImageCheckApplyCtxTimeout = 120
+)
+
+func init() {
+	engine.RegisterResource("docker:image", func() engine.Res { return &DockerImageRes{} })
+}
+
+// DockerImageRes is a docker image resource. The resource's name must be a
+// docker image in any supported format (url, image, or image:tag).
+type DockerImageRes struct {
+	traits.Base // add the base methods without re-implementation
+	traits.Edgeable
+
+	// State of the image must be exists or absent.
+	State string `yaml:"state"`
+	// APIVersion allows you to override the host's default client API
+	// version.
+	APIVersion string `yaml:"apiversion"`
+
+	image  string         // full image:tag format
+	client *client.Client // docker api client
+
+	init *engine.Init
+}
+
+// Default returns some sensible defaults for this resource.
+func (obj *DockerImageRes) Default() engine.Res {
+	return &DockerImageRes{
+		// TODO: eventually if image supports other properties, this can
+		// be left out and we could have the state be "unmanaged".
+		State: "exists",
+	}
+}
+
+// Validate if the params passed in are valid data.
+func (obj *DockerImageRes) Validate() error {
+	// validate state
+	if obj.State != "exists" && obj.State != "absent" {
+		return fmt.Errorf("state must be exists or absent")
+	}
+
+	// validate APIVersion
+	if obj.APIVersion != "" {
+		verOK, err := regexp.MatchString(`^(v)[1-9]\.[0-9]\d*$`, obj.APIVersion)
+		if err != nil {
+			return errwrap.Wrapf(err, "error matching apiversion string")
+		}
+		if !verOK {
+			return fmt.Errorf("invalid apiversion: %s", obj.APIVersion)
+		}
+	}
+
+	return nil
+}
+
+// Init runs some startup code for this resource.
+func (obj *DockerImageRes) Init(init *engine.Init) error {
+	var err error
+	obj.init = init // save for later
+
+	// Save the full image name and tag.
+	obj.image = dockerImageNameTag(obj.Name())
+
+	ctx, cancel := context.WithTimeout(context.Background(), dockerImageInitCtxTimeout*time.Second)
+	defer cancel()
+
+	// Initialize the docker client.
+	obj.client, err = client.NewClientWithOpts(client.WithVersion(obj.APIVersion))
+	if err != nil {
+		return errwrap.Wrapf(err, "error creating docker client")
+	}
+
+	// Validate the image.
+	resp, err := obj.client.ImageSearch(ctx, obj.image, types.ImageSearchOptions{Limit: 1})
+	if err != nil {
+		return errwrap.Wrapf(err, "error searching for image")
+	}
+	if len(resp) == 0 {
+		return fmt.Errorf("image: %s not found", obj.image)
+	}
+	return nil
+}
+
+// Close is run by the engine to clean up after the resource is done.
+func (obj *DockerImageRes) Close() error {
+	return obj.client.Close() // close the docker client
+}
+
+// Watch is the primary listener for this resource and it outputs events.
+func (obj *DockerImageRes) Watch() error {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	eventChan, errChan := obj.client.Events(ctx, types.EventsOptions{})
+
+	// notify engine that we're running
+	obj.init.Running()
+
+	var send = false // send event?
+	for {
+		select {
+		case event, ok := <-eventChan:
+			if !ok { // channel shutdown
+				return nil
+			}
+			if obj.init.Debug {
+				obj.init.Logf("%+v", event)
+			}
+			send = true
+
+		case err, ok := <-errChan:
+			if !ok {
+				return nil
+			}
+			return err
+
+		case <-obj.init.Done: // closed by the engine to signal shutdown
+			return nil
+		}
+
+		// do all our event sending all together to avoid duplicate msgs
+		if send {
+			send = false
+			obj.init.Event() // notify engine of an event (this can block)
+		}
+	}
+}
+
+// CheckApply method for Docker resource.
+func (obj *DockerImageRes) CheckApply(apply bool) (checkOK bool, err error) {
+	ctx, cancel := context.WithTimeout(context.Background(), dockerImageCheckApplyCtxTimeout*time.Second)
+	defer cancel()
+
+	s, err := obj.client.ImageList(ctx, types.ImageListOptions{
+		Filters: filters.NewArgs(filters.Arg("reference", obj.image)),
+	})
+	if err != nil {
+		return false, errwrap.Wrapf(err, "error listing images")
+	}
+	if len(s) > 1 {
+		return false, fmt.Errorf("more than one image found")
+	}
+
+	if obj.State == "absent" && len(s) == 0 {
+		return true, nil
+	}
+	if obj.State == "exists" && len(s) == 1 {
+		return true, nil
+	}
+
+	if !apply {
+		return false, nil
+	}
+
+	if obj.State == "absent" {
+		// TODO: force? prune children?
+		if _, err := obj.client.ImageRemove(ctx, obj.image, types.ImageRemoveOptions{}); err != nil {
+			return false, errwrap.Wrapf(err, "error removing image")
+		}
+		return false, nil
+	}
+
+	// pull the image
+	p, err := obj.client.ImagePull(ctx, obj.image, types.ImagePullOptions{})
+	if err != nil {
+		return false, errwrap.Wrapf(err, "error pulling image")
+	}
+	// Wait for the image to download, EOF signals that it's done.
+	if _, err := ioutil.ReadAll(p); err != nil {
+		return false, errwrap.Wrapf(err, "error reading image pull result")
+	}
+
+	return false, nil
+}
+
+// Cmp compares two resources and returns an error if they are not equivalent.
+func (obj *DockerImageRes) Cmp(r engine.Res) error {
+	// we can only compare DockerImageRes to others of the same resource kind
+	res, ok := r.(*DockerImageRes)
+	if !ok {
+		return fmt.Errorf("error casting r to *DockerImageRes")
+	}
+	if obj.State != res.State {
+		return fmt.Errorf("the State differs")
+	}
+
+	if obj.APIVersion != res.APIVersion {
+		return fmt.Errorf("the APIVersion differs")
+	}
+	return nil
+}
+
+// DockerImageUID is the UID struct for DockerImageRes.
+type DockerImageUID struct {
+	engine.BaseUID
+
+	image string
+}
+
+// UIDs includes all params to make a unique identification of this object. Most
+// resources only return one, although some resources can return multiple.
+func (obj *DockerImageRes) UIDs() []engine.ResUID {
+	x := &DockerImageUID{
+		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
+		image:   dockerImageNameTag(obj.Name()),
+	}
+	return []engine.ResUID{x}
+}
+
+// AutoEdges returns the AutoEdge interface.
+func (obj *DockerImageRes) AutoEdges() (engine.AutoEdge, error) {
+	return nil, nil
+}
+
+// IFF aka if and only if they are equivalent, return true. If not, false.
+func (obj *DockerImageUID) IFF(uid engine.ResUID) bool {
+	res, ok := uid.(*DockerImageUID)
+	if !ok {
+		return false
+	}
+	return obj.image == res.image
+}
+
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
+// primarily useful for setting the defaults.
+func (obj *DockerImageRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type rawRes DockerImageRes // indirection to avoid infinite recursion
+
+	def := obj.Default()             // get the default
+	res, ok := def.(*DockerImageRes) // put in the right format
+	if !ok {
+		return fmt.Errorf("could not convert to DockerImageRes")
+	}
+	raw := rawRes(*res) // convert; the defaults go here
+
+	if err := unmarshal(&raw); err != nil {
+		return err
+	}
+
+	*obj = DockerImageRes(raw) // restore from indirection with type conversion!
+	return nil
+}
+
+// dockerImageNameTag does a naive check to see if the input includes a tag or
+// is a url, and if not, appends the `:latest` tag to ensure disambiguation.
+func dockerImageNameTag(image string) string {
+	if strings.Contains(image, ":") {
+		return image
+	}
+	return image + ":latest"
+}

engine/resources/exec.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -24,6 +24,7 @@ import (
 	"fmt"
 	"os/exec"
 	"os/user"
+	"sort"
 	"strings"
 	"sync"
 	"syscall"
@@ -54,7 +55,7 @@ type ExecRes struct {
 	// only be used when a Shell is *not* specified. The advantage of this
 	// is that you don't have to worry about escape characters.
 	Args []string `yaml:"args"`
-	// Cmd is the dir to run the command in. If empty, then this will use
+	// Cwd is the dir to run the command in. If empty, then this will use
 	// the working directory of the calling process. (This process is mgmt,
 	// not the process being run here.)
 	Cwd string `yaml:"cwd"`
@@ -65,6 +66,9 @@ type ExecRes struct {
 	// running command. If the Kill is received before the process exits,
 	// then this be treated as an error.
 	Timeout uint64 `yaml:"timeout"`
+	// Env allows the user to specify environment variables for script
+	// execution. These are taken using a map of format of VAR_NAME -> value.
+	Env map[string]string `yaml:"env"`
 
 	// Watch is the command to run to detect event changes. Each line of
 	// output from this command is treated as an event.
@@ -138,6 +142,12 @@ func (obj *ExecRes) Validate() error {
 		}
 	}
 
+	// check that environment variables' format is valid
+	for key := range obj.Env {
+		if err := isNameValid(key); err != nil {
+			return errwrap.Wrapf(err, "invalid variable name")
+		}
+	}
 	return nil
 }
 
@@ -214,19 +224,21 @@ func (obj *ExecRes) Watch() error {
 				exitErr, ok := err.(*exec.ExitError) // embeds an os.ProcessState
 				if !ok {
 					// command failed in some bad way
-					return errwrap.Wrapf(err, "unknown error")
+					return errwrap.Wrapf(err, "watchcmd failed in some bad way")
 				}
 				pStateSys := exitErr.Sys() // (*os.ProcessState) Sys
 				wStatus, ok := pStateSys.(syscall.WaitStatus)
 				if !ok {
-					return errwrap.Wrapf(err, "error running cmd")
+					return errwrap.Wrapf(err, "could not get exit status of watchcmd")
 				}
 				exitStatus := wStatus.ExitStatus()
-				obj.init.Logf("watchcmd exited with: %d", exitStatus)
+				if exitStatus == 0 {
-				if exitStatus != 0 {
+					// i'm not sure if this could happen
-					return errwrap.Wrapf(err, "unexpected exit status of zero")
+					return errwrap.Wrapf(err, "unexpected watchcmd exit status of zero")
 				}
-				return err // i'm not sure if this could happen
+
+				obj.init.Logf("watchcmd exited with: %d", exitStatus)
+				return errwrap.Wrapf(err, "watchcmd errored")
 			}
 
 			// each time we get a line of output, we loop!
@@ -298,16 +310,17 @@ func (obj *ExecRes) CheckApply(apply bool) (bool, error) {
 			exitErr, ok := err.(*exec.ExitError) // embeds an os.ProcessState
 			if !ok {
 				// command failed in some bad way
-				return false, err
+				return false, errwrap.Wrapf(err, "ifcmd failed in some bad way")
 			}
 			pStateSys := exitErr.Sys() // (*os.ProcessState) Sys
 			wStatus, ok := pStateSys.(syscall.WaitStatus)
 			if !ok {
-				return false, errwrap.Wrapf(err, "error running cmd")
+				return false, errwrap.Wrapf(err, "could not get exit status of ifcmd")
 			}
 			exitStatus := wStatus.ExitStatus()
 			if exitStatus == 0 {
-				return false, fmt.Errorf("unexpected exit status of zero")
+				// i'm not sure if this could happen
+				return false, errwrap.Wrapf(err, "unexpected ifcmd exit status of zero")
 			}
 
 			obj.init.Logf("ifcmd exited with: %d", exitStatus)
@@ -368,6 +381,18 @@ func (obj *ExecRes) CheckApply(apply bool) (bool, error) {
 	defer cancel()
 	cmd := exec.CommandContext(ctx, cmdName, cmdArgs...)
 	cmd.Dir = obj.Cwd // run program in pwd if ""
+
+	envKeys := []string{}
+	for key := range obj.Env {
+		envKeys = append(envKeys, key)
+	}
+	sort.Strings(envKeys)
+	cmdEnv := []string{}
+	for _, k := range envKeys {
+		cmdEnv = append(cmdEnv, k+"="+obj.Env[k])
+	}
+	cmd.Env = cmdEnv
+
 	// ignore signals sent to parent process (we're in our own group)
 	cmd.SysProcAttr = &syscall.SysProcAttr{
 		Setpgid: true,
@@ -438,7 +463,7 @@ func (obj *ExecRes) CheckApply(apply bool) (bool, error) {
 			return false, errwrap.Wrapf(err, "cmd timeout, exit status: %d", exitStatus)
 		}
 
-		return false, fmt.Errorf("unknown cmd error, signal: %s, exit status: %d", sig, exitStatus)
+		return false, errwrap.Wrapf(err, "unknown cmd error, signal: %s, exit status: %d", sig, exitStatus)
 
 	} else if err != nil {
 		return false, errwrap.Wrapf(err, "general cmd error")
@@ -545,25 +570,38 @@ type ExecUID struct {
 
 // ExecResAutoEdges holds the state of the auto edge generator.
 type ExecResAutoEdges struct {
-	edges []engine.ResUID
+	edges   []engine.ResUID
+	pointer int
 }
 
 // Next returns the next automatic edge.
 func (obj *ExecResAutoEdges) Next() []engine.ResUID {
-	return obj.edges
+	if len(obj.edges) == 0 {
+		return nil
+	}
+	value := obj.edges[obj.pointer]
+	obj.pointer++
+	return []engine.ResUID{value}
 }
 
-// Test gets results of the earlier Next() call, & returns if we should continue!
+// Test gets results of the earlier Next() call, & returns if we should
+// continue!
 func (obj *ExecResAutoEdges) Test(input []bool) bool {
-	return false // never keep going
+	if len(obj.edges) <= obj.pointer {
-	// TODO: we could return false if we find as many edges as the number of different path's in cmdFiles()
+		return false
+	}
+	if len(input) != 1 { // in case we get given bad data
+		panic(fmt.Sprintf("Expecting a single value!"))
+	}
+	return true // keep going
 }
 
 // AutoEdges returns the AutoEdge interface. In this case the systemd units.
 func (obj *ExecRes) AutoEdges() (engine.AutoEdge, error) {
 	var data []engine.ResUID
+	var reversed = true
+
 	for _, x := range obj.cmdFiles() {
-		var reversed = true
 		data = append(data, &PkgFileUID{
 			BaseUID: engine.BaseUID{
 				Name:     obj.Name(),
@@ -572,14 +610,44 @@ func (obj *ExecRes) AutoEdges() (engine.AutoEdge, error) {
 			},
 			path: x, // what matters
 		})
+		data = append(data, &FileUID{
+			BaseUID: engine.BaseUID{
+				Name:     obj.Name(),
+				Kind:     obj.Kind(),
+				Reversed: &reversed,
+			},
+			path: x,
+		})
 	}
+	if obj.User != "" {
+		data = append(data, &UserUID{
+			BaseUID: engine.BaseUID{
+				Name:     obj.Name(),
+				Kind:     obj.Kind(),
+				Reversed: &reversed,
+			},
+			name: obj.User,
+		})
+	}
+	if obj.Group != "" {
+		data = append(data, &GroupUID{
+			BaseUID: engine.BaseUID{
+				Name:     obj.Name(),
+				Kind:     obj.Kind(),
+				Reversed: &reversed,
+			},
+			name: obj.Group,
+		})
+	}
+
 	return &ExecResAutoEdges{
-		edges: data,
+		edges:   data,
+		pointer: 0,
 	}, nil
 }
 
-// UIDs includes all params to make a unique identification of this object.
+// UIDs includes all params to make a unique identification of this object. Most
-// Most resources only return one, although some resources can return multiple.
+// resources only return one, although some resources can return multiple.
 func (obj *ExecRes) UIDs() []engine.ResUID {
 	x := &ExecUID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
@@ -609,8 +677,8 @@ func (obj *ExecRes) Sends() interface{} {
 	}
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
-// It is primarily useful for setting the defaults.
+// primarily useful for setting the defaults.
 func (obj *ExecRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes ExecRes // indirection to avoid infinite recursion
 
@@ -690,9 +758,9 @@ type cmdOutput struct {
 }
 
 // cmdOutputRunner wraps the Cmd in with a StdoutPipe scanner and reads for
-// errors. It runs Start and Wait, and errors runtime things in the channel.
+// errors. It runs Start and Wait, and errors runtime things in the channel. If
-// If it can't start up the command, it will fail early. Once it's running, it
+// it can't start up the command, it will fail early. Once it's running, it will
-// will return the channel which can be used for the duration of the process.
+// return the channel which can be used for the duration of the process.
 // Cancelling the context merely unblocks the sending on the output channel, it
 // does not Kill the cmd process. For that you must do it yourself elsewhere.
 func (obj *ExecRes) cmdOutputRunner(ctx context.Context, cmd *exec.Cmd) (chan *cmdOutput, error) {
@@ -800,3 +868,20 @@ func (obj *wrapWriter) Write(p []byte) (int, error) {
 func (obj *wrapWriter) String() string {
 	return obj.Buffer.String()
 }
+
+// isNameValid checks that environment variable name is valid.
+func isNameValid(varName string) error {
+	if varName == "" {
+		return fmt.Errorf("variable name cannot be an empty string")
+	}
+	for i := range varName {
+		c := varName[i]
+		if i == 0 && '0' <= c && c <= '9' {
+			return fmt.Errorf("variable name cannot begin with number")
+		}
+		if !(c == '_' || '0' <= c && c <= '9' || 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z') {
+			return fmt.Errorf("invalid character in variable name")
+		}
+	}
+	return nil
+}

engine/resources/exec_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -28,6 +28,8 @@ import (
 	"time"
 
 	"github.com/purpleidea/mgmt/engine"
+	"github.com/purpleidea/mgmt/engine/graph/autoedge"
+	"github.com/purpleidea/mgmt/pgraph"
 )
 
 func fakeExecInit(t *testing.T) (*engine.Init, *ExecSends) {
@@ -257,3 +259,77 @@ func TestExecTimeoutBehaviour(t *testing.T) {
 
 	// no error
 }
+
+func TestExecAutoEdge1(t *testing.T) {
+	g, err := pgraph.NewGraph("TestGraph")
+	if err != nil {
+		t.Errorf("error creating graph: %v", err)
+		return
+	}
+
+	resUser, err := engine.NewNamedResource("user", "someuser")
+	if err != nil {
+		t.Errorf("error creating user resource: %v", err)
+		return
+	}
+
+	resGroup, err := engine.NewNamedResource("group", "somegroup")
+	if err != nil {
+		t.Errorf("error creating group resource: %v", err)
+		return
+	}
+
+	resFile, err := engine.NewNamedResource("file", "/somefile")
+	if err != nil {
+		t.Errorf("error creating group resource: %v", err)
+		return
+	}
+
+	resExec, err := engine.NewNamedResource("exec", "somefile")
+	if err != nil {
+		t.Errorf("error creating exec resource: %v", err)
+		return
+	}
+	exc := resExec.(*ExecRes)
+	exc.Cmd = resFile.Name()
+	exc.User = resUser.Name()
+	exc.Group = resGroup.Name()
+
+	g.AddVertex(resUser, resGroup, resFile, resExec)
+
+	if i := g.NumEdges(); i != 0 {
+		t.Errorf("should have 0 edges instead of: %d", i)
+		return
+	}
+
+	debug := testing.Verbose() // set via the -test.v flag to `go test`
+	logf := func(format string, v ...interface{}) {
+		t.Logf("test: "+format, v...)
+	}
+	if err := autoedge.AutoEdge(g, debug, logf); err != nil {
+		t.Errorf("error running autoedges: %v", err)
+		return
+	}
+
+	expected, err := pgraph.NewGraph("Expected")
+	if err != nil {
+		t.Errorf("error creating graph: %v", err)
+		return
+	}
+
+	expectEdge := func(from, to pgraph.Vertex) {
+		edge := &engine.Edge{Name: fmt.Sprintf("%s -> %s (expected)", from, to)}
+		expected.AddEdge(from, to, edge)
+	}
+	expectEdge(resFile, resExec)
+	expectEdge(resUser, resExec)
+	expectEdge(resGroup, resExec)
+
+	vertexCmp := func(v1, v2 pgraph.Vertex) (bool, error) { return v1 == v2, nil } // pointer compare is sufficient
+	edgeCmp := func(e1, e2 pgraph.Edge) (bool, error) { return true, nil }         // we don't care about edges here
+
+	if err := expected.GraphCmp(g, vertexCmp, edgeCmp); err != nil {
+		t.Errorf("graph doesn't match expected: %s", err)
+		return
+	}
+}

engine/resources/file.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -29,21 +29,52 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"sync"
 	"syscall"
 
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/engine/traits"
 	engineUtil "github.com/purpleidea/mgmt/engine/util"
+	"github.com/purpleidea/mgmt/lang/funcs/vars"
+	"github.com/purpleidea/mgmt/lang/interfaces"
+	"github.com/purpleidea/mgmt/lang/types"
 	"github.com/purpleidea/mgmt/recwatch"
 	"github.com/purpleidea/mgmt/util"
 	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 func init() {
-	engine.RegisterResource("file", func() engine.Res { return &FileRes{} })
+	engine.RegisterResource(KindFile, func() engine.Res { return &FileRes{} })
+
+	// const.res.file.state.exists = "exists"
+	// const.res.file.state.absent = "absent"
+	vars.RegisterResourceParams(KindFile, map[string]map[string]func() interfaces.Var{
+		ParamFileState: {
+			FileStateExists: func() interfaces.Var {
+				return &types.StrValue{
+					V: FileStateExists,
+				}
+			},
+			FileStateAbsent: func() interfaces.Var {
+				return &types.StrValue{
+					V: FileStateAbsent,
+				}
+			},
+			// TODO: consider removing this field entirely
+			"undefined": func() interfaces.Var {
+				return &types.StrValue{
+					V: FileStateUndefined, // empty string
+				}
+			},
+		},
+	})
 }
 
 const (
+	// KindFile is the kind string used to identify this resource.
+	KindFile = "file"
+	// ParamFileState is the name of the state field parameter.
+	ParamFileState = "state"
 	// FileStateExists is the string that represents that the file should be
 	// present.
 	FileStateExists = "exists"
@@ -53,13 +84,20 @@ const (
 	// FileStateUndefined means the file state has not been specified.
 	// TODO: consider moving to *string and express this state as a nil.
 	FileStateUndefined = ""
+
+	// FileModeAllowAssign specifies whether we only use ugo=rwx style
+	// assignment (false) or if we also allow ugo+-rwx style too (true). I
+	// think that it's possibly illogical to allow imperative mode
+	// specifiers in a declarative language, so let's leave it off for now.
+	FileModeAllowAssign = false
 )
 
-// FileRes is a file and directory resource. Dirs are defined by names ending
+// FileRes is a file and directory resource. Dirs are defined by names ending in
-// in a slash.
+// a slash.
 type FileRes struct {
 	traits.Base // add the base methods without re-implementation
 	traits.Edgeable
+	traits.GraphQueryable // allow others to query this res in the res graph
 	//traits.Groupable // TODO: implement this
 	traits.Recvable
 	traits.Reversible
@@ -81,11 +119,38 @@ type FileRes struct {
 	State string `lang:"state" yaml:"state"`
 
 	// Content specifies the file contents to use. If this is nil, they are
-	// left undefined. It cannot be combined with Source.
+	// left undefined. It cannot be combined with the Source or Fragments
+	// parameters.
 	Content *string `lang:"content" yaml:"content"`
 	// Source specifies the source contents for the file resource. It cannot
-	// be combined with the Content parameter.
+	// be combined with the Content or Fragments parameters. It must be an
+	// absolute path, and it can point to a file or a directory. If it
+	// points to a file, then that will will be copied throuh directly. If
+	// it points to a directory, then it will copy the directory "rsync
+	// style" onto the file destination. As a result, if this is a file,
+	// then the main file res must be a file, and if it is a directory, then
+	// this must be a directory. To meaningfully copy a full directory, you
+	// also need to specify the Recurse parameter, which is currently
+	// required. If you want an existing dir to be turned into a file (or
+	// vice-versa) instead of erroring, then you'll also need to specify the
+	// Force parameter. If source is undefined and the file path is a
+	// directory, then a directory will be created. If left undefined, and
+	// combined with the Purge option too, then any unmanaged file in this
+	// dir will be removed.
 	Source string `lang:"source" yaml:"source"`
+	// Fragments specifies that the file is built from a list of individual
+	// files. If one of the files is a directory, then the list of files in
+	// that directory are the fragments to combine. Multiple of these can be
+	// used together, although most simple cases will probably only either
+	// involve a single directory path or a fixed list of individual files.
+	// All paths are absolute and as a result must start with a slash. The
+	// directories (if any) must end with a slash as well. This cannot be
+	// combined with the Content or Source parameters. If a file with param
+	// is reversed, the reversed file is one that has `Content` set instead.
+	// Automatic edges will be added from these fragments. This currently
+	// isn't recursive in that if a fragment is a directory, this only
+	// searches one level deep at the moment.
+	Fragments []string `lang:"fragments" yaml:"fragments"`
 
 	// Owner specifies the file owner. You can specify either the string
 	// name, or a string representation of the owner integer uid.
@@ -94,14 +159,17 @@ type FileRes struct {
 	// name, or a string representation of the group integer gid.
 	Group string `lang:"group" yaml:"group"`
 	// Mode is the mode of the file as a string representation of the octal
-	// form.
+	// form or symbolic form.
-	// TODO: add symbolic representations
 	Mode    string `lang:"mode" yaml:"mode"`
 	Recurse bool   `lang:"recurse" yaml:"recurse"`
 	Force   bool   `lang:"force" yaml:"force"`
+	// Purge specifies that when true, any unmanaged file in this file
+	// directory will be removed. As a result, this file resource must be a
+	// directory. This isn't particularly meaningful if you don't also set
+	// Recurse to true. This doesn't work with Content or Fragments.
+	Purge bool `lang:"purge" yaml:"purge"`
 
-	sha256sum  string
+	sha256sum string
-	recWatcher *recwatch.RecWatcher
 }
 
 // getPath returns the actual path to use for this resource. It computes this
@@ -137,10 +205,22 @@ func (obj *FileRes) isDir() bool {
 // the case where the mode is not specified. The caller should check obj.Mode is
 // not empty.
 func (obj *FileRes) mode() (os.FileMode, error) {
-	m, err := strconv.ParseInt(obj.Mode, 8, 32)
+	if n, err := strconv.ParseInt(obj.Mode, 8, 32); err == nil {
-	if err != nil {
+		return os.FileMode(n), nil
-		return os.FileMode(0), errwrap.Wrapf(err, "mode should be an octal number (%s)", obj.Mode)
 	}
+
+	// Try parsing symbolically by first getting the files current mode.
+	stat, err := os.Stat(obj.getPath())
+	if err != nil {
+		return os.FileMode(0), errwrap.Wrapf(err, "failed to get the current file mode")
+	}
+
+	modes := strings.Split(obj.Mode, ",")
+	m, err := engineUtil.ParseSymbolicModes(modes, stat.Mode(), FileModeAllowAssign)
+	if err != nil {
+		return os.FileMode(0), errwrap.Wrapf(err, "mode should be an octal number or symbolic mode (%s)", obj.Mode)
+	}
+
 	return os.FileMode(m), nil
 }
 
@@ -173,18 +253,55 @@ func (obj *FileRes) Validate() error {
 		return fmt.Errorf("the State is invalid")
 	}
 
-	if obj.State == FileStateAbsent && obj.Content != nil {
+	isContent := obj.Content != nil
-		return fmt.Errorf("can't specify Content for an absent file")
+	isSrc := obj.Source != ""
+	isFrag := len(obj.Fragments) > 0
+	if (isContent && isSrc) || (isSrc && isFrag) || (isFrag && isContent) {
+		return fmt.Errorf("can only specify one of Content, Source, and Fragments")
 	}
 
-	if obj.Content != nil && obj.Source != "" {
+	if obj.State == FileStateAbsent && (isContent || isSrc || isFrag) {
-		return fmt.Errorf("can't specify both Content and Source")
+		return fmt.Errorf("can't specify file Content, Source, or Fragments when State is %s", FileStateAbsent)
 	}
 
-	if obj.isDir() && obj.Content != nil { // makes no sense
+	// The path and Source must either both be dirs or both not be.
-		return fmt.Errorf("can't specify Content when creating a Dir")
+	srcIsDir := strings.HasSuffix(obj.Source, "/")
+	if isSrc && (obj.isDir() != srcIsDir) {
+		return fmt.Errorf("the path and Source must either both be dirs or both not be")
 	}
 
+	if obj.isDir() && (isContent || isFrag) { // makes no sense
+		return fmt.Errorf("can't specify Content or Fragments when creating a Dir")
+	}
+
+	// TODO: is this really a requirement that we want to enforce?
+	if isSrc && obj.isDir() && srcIsDir && !obj.Recurse {
+		return fmt.Errorf("you'll want to Recurse when you have a Source dir to copy")
+	}
+	// TODO: do we want to enforce this sort of thing?
+	if obj.Purge && !obj.Recurse {
+		return fmt.Errorf("you'll want to Recurse when you have a Purge to do")
+	}
+
+	if isSrc && !obj.isDir() && !srcIsDir && obj.Recurse {
+		return fmt.Errorf("you can't recurse when copying a single file")
+	}
+
+	for _, frag := range obj.Fragments {
+		// absolute paths begin with a slash
+		if !strings.HasPrefix(frag, "/") {
+			return fmt.Errorf("the frag (`%s`) isn't an absolute path", frag)
+		}
+	}
+
+	if obj.Purge && (isContent || isFrag) {
+		return fmt.Errorf("can't combine Purge with Content or Fragments")
+	}
+	// XXX: should this work with obj.Purge && obj.Source != "" or not?
+	//if obj.Purge && obj.Source != "" {
+	//	return fmt.Errorf("can't Purge when Source is specified")
+	//}
+
 	// TODO: should we silently ignore these errors or include them?
 	//if obj.State == FileStateAbsent && obj.Owner != "" {
 	//	return fmt.Errorf("can't specify Owner for an absent file")
@@ -220,11 +337,6 @@ func (obj *FileRes) Validate() error {
 		}
 	}
 
-	// XXX: should this specify that we create an empty directory instead?
-	//if obj.Source == "" && obj.isDir() {
-	//	return fmt.Errorf("can't specify an empty source when creating a Dir.")
-	//}
-
 	return nil
 }
 
@@ -242,19 +354,112 @@ func (obj *FileRes) Close() error {
 	return nil
 }
 
-// Watch is the primary listener for this resource and it outputs events.
+// Watch is the primary listener for this resource and it outputs events. This
-// This one is a file watcher for files and directories.
+// one is a file watcher for files and directories. Modify with caution, it is
-// Modify with caution, it is probably important to write some test cases first!
+// probably important to write some test cases first! If the Watch returns an
-// If the Watch returns an error, it means that something has gone wrong, and it
+// error, it means that something has gone wrong, and it must be restarted. On a
-// must be restarted. On a clean exit it returns nil.
+// clean exit it returns nil.
-// FIXME: Also watch the source directory when using obj.Source !!!
 func (obj *FileRes) Watch() error {
-	var err error
+	// TODO: chan *recwatch.Event instead?
-	obj.recWatcher, err = recwatch.NewRecWatcher(obj.getPath(), obj.Recurse)
+	inputEvents := make(chan recwatch.Event)
+	defer close(inputEvents)
+
+	wg := &sync.WaitGroup{}
+	defer wg.Wait()
+
+	exit := make(chan struct{})
+	// TODO: should this be after (later in the file) than the `defer recWatcher.Close()` ?
+	// TODO: should this be after (later in the file) the `defer recWatcher.Close()` ?
+	defer close(exit)
+
+	recWatcher, err := recwatch.NewRecWatcher(obj.getPath(), obj.Recurse)
 	if err != nil {
 		return err
 	}
-	defer obj.recWatcher.Close()
+	defer recWatcher.Close()
+
+	// watch the various inputs to this file resource too!
+	if obj.Source != "" {
+		// This block is virtually identical to the below one.
+		recurse := strings.HasSuffix(obj.Source, "/") // isDir
+		rw, err := recwatch.NewRecWatcher(obj.Source, recurse)
+		if err != nil {
+			return err
+		}
+		defer rw.Close()
+
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for {
+				// TODO: *recwatch.Event instead?
+				var event recwatch.Event
+				var ok bool
+				var shutdown bool
+				select {
+				case event, ok = <-rw.Events(): // recv
+				case <-exit: // unblock
+					return
+				}
+
+				if !ok {
+					err := fmt.Errorf("channel shutdown")
+					event = recwatch.Event{Error: err}
+					shutdown = true
+				}
+
+				select {
+				case inputEvents <- event: // send
+					if shutdown { // optimization to free early
+						return
+					}
+				case <-exit: // unblock
+					return
+				}
+			}
+		}()
+	}
+	for _, frag := range obj.Fragments {
+		// This block is virtually identical to the above one.
+		recurse := false // TODO: is it okay for depth==1 dirs?
+		//recurse := strings.HasSuffix(frag, "/") // isDir
+		rw, err := recwatch.NewRecWatcher(frag, recurse)
+		if err != nil {
+			return err
+		}
+		defer rw.Close()
+
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for {
+				// TODO: *recwatch.Event instead?
+				var event recwatch.Event
+				var ok bool
+				var shutdown bool
+				select {
+				case event, ok = <-rw.Events(): // recv
+				case <-exit: // unblock
+					return
+				}
+
+				if !ok {
+					err := fmt.Errorf("channel shutdown")
+					event = recwatch.Event{Error: err}
+					shutdown = true
+				}
+
+				select {
+				case inputEvents <- event: // send
+					if shutdown { // optimization to free early
+						return
+					}
+				case <-exit: // unblock
+					return
+				}
+			}
+		}()
+	}
 
 	obj.init.Running() // when started, notify engine that we're running
 
@@ -265,9 +470,12 @@ func (obj *FileRes) Watch() error {
 		}
 
 		select {
-		case event, ok := <-obj.recWatcher.Events():
+		case event, ok := <-recWatcher.Events():
 			if !ok { // channel shutdown
-				return nil
+				// TODO: Should this be an error? Previously it
+				// was a `return nil`, and i'm not sure why...
+				//return nil
+				return fmt.Errorf("unexpected close")
 			}
 			if err := event.Error; err != nil {
 				return errwrap.Wrapf(err, "unknown %s watcher error", obj)
@@ -277,6 +485,18 @@ func (obj *FileRes) Watch() error {
 			}
 			send = true
 
+		case event, ok := <-inputEvents:
+			if !ok {
+				return fmt.Errorf("unexpected close")
+			}
+			if err := event.Error; err != nil {
+				return errwrap.Wrapf(err, "unknown %s input watcher error", obj)
+			}
+			if obj.init.Debug { // don't access event.Body if event.Error isn't nil
+				obj.init.Logf("input event(%s): %v", event.Body.Name, event.Body.Op)
+			}
+			send = true
+
 		case <-obj.init.Done: // closed by the engine to signal shutdown
 			return nil
 		}
@@ -485,11 +705,14 @@ func (obj *FileRes) dirCheckApply(apply bool) (bool, error) {
 // syncCheckApply is the CheckApply operation for a source and destination dir.
 // It is recursive and can create directories directly, and files via the usual
 // fileCheckApply method. It returns checkOK and error as is normally expected.
-func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
+// If excludes is specified, none of those files there will be deleted by this,
+// with the exception that a sync *can* convert a file to a dir, or vice-versa.
+func (obj *FileRes) syncCheckApply(apply bool, src, dst string, excludes []string) (bool, error) {
 	if obj.init.Debug {
 		obj.init.Logf("syncCheckApply: %s -> %s", src, dst)
 	}
-	if src == "" || dst == "" {
+	// an src of "" is now supported, if dst is a dir
+	if dst == "" {
 		return false, fmt.Errorf("the src and dst must not be empty")
 	}
 
@@ -499,11 +722,14 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 	srcIsDir := strings.HasSuffix(src, "/")
 	dstIsDir := strings.HasSuffix(dst, "/")
 
-	if srcIsDir != dstIsDir {
+	if srcIsDir != dstIsDir && src != "" {
 		return false, fmt.Errorf("the src and dst must be both either files or directories")
 	}
+	if src == "" && !dstIsDir {
+		return false, fmt.Errorf("dst must be a dir if we have an empty src")
+	}
 
-	if !srcIsDir && !dstIsDir {
+	if !srcIsDir && !dstIsDir && src != "" {
 		if obj.init.Debug {
 			obj.init.Logf("syncCheckApply: %s -> %s", src, dst)
 		}
@@ -524,18 +750,23 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 	}
 
 	// else: if srcIsDir && dstIsDir
-	srcFiles, err := ReadDir(src)          // if src does not exist...
+
-	if err != nil && !os.IsNotExist(err) { // an empty map comes out below!
+	smartSrc := make(map[string]FileInfo)
-		return false, err
+	if src != "" {
+		srcFiles, err := ReadDir(src)          // if src does not exist...
+		if err != nil && !os.IsNotExist(err) { // an empty map comes out below!
+			return false, err
+		}
+		smartSrc = mapPaths(srcFiles)
+		obj.init.Logf("syncCheckApply: srcFiles: %v", srcFiles)
 	}
+
 	dstFiles, err := ReadDir(dst)
 	if err != nil && !os.IsNotExist(err) {
 		return false, err
 	}
-	//obj.init.Logf("syncCheckApply: srcFiles: %v", srcFiles)
-	//obj.init.Logf("syncCheckApply: dstFiles: %v", dstFiles)
-	smartSrc := mapPaths(srcFiles)
 	smartDst := mapPaths(dstFiles)
+	obj.init.Logf("syncCheckApply: dstFiles: %v", dstFiles)
 
 	for relPath, fileInfo := range smartSrc {
 		absSrc := fileInfo.AbsPath // absolute path
@@ -581,7 +812,7 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 			obj.init.Logf("syncCheckApply: recurse: %s -> %s", absSrc, absDst)
 		}
 		if obj.Recurse {
-			if c, err := obj.syncCheckApply(apply, absSrc, absDst); err != nil { // recurse
+			if c, err := obj.syncCheckApply(apply, absSrc, absDst, excludes); err != nil { // recurse
 				return false, errwrap.Wrapf(err, "syncCheckApply: recurse failed")
 			} else if !c { // don't let subsequent passes make this true
 				checkOK = false
@@ -596,6 +827,19 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 	if !apply && len(smartDst) > 0 { // we know there are files to remove!
 		return false, nil // so just exit now
 	}
+
+	// isExcluded specifies if the path is part of an excluded path. For
+	// example, if we exclude /tmp/foo/bar from deletion, then we don't want
+	// to delete /tmp/foo/bar *or* /tmp/foo/ *or* /tmp/ b/c they're parents.
+	isExcluded := func(p string) bool {
+		for _, x := range excludes {
+			if util.HasPathPrefix(x, p) {
+				return true
+			}
+		}
+		return false
+	}
+
 	// any files that now remain in smartDst need to be removed...
 	for relPath, fileInfo := range smartDst {
 		absSrc := src + relPath    // absolute dest (should not exist!)
@@ -611,6 +855,9 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 		// think the symmetry is more elegant and correct here for now
 		// Avoiding this is also useful if we had a recurse limit arg!
 		if true { // switch
+			if isExcluded(absDst) { // skip removing excluded files
+				continue
+			}
 			obj.init.Logf("syncCheckApply: removing: %s", absCleanDst)
 			if apply {
 				if err := os.RemoveAll(absCleanDst); err != nil { // dangerous ;)
@@ -622,11 +869,14 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 		}
 		_ = absSrc
 		//obj.init.Logf("syncCheckApply: recurse rm: %s -> %s", absSrc, absDst)
-		//if c, err := obj.syncCheckApply(apply, absSrc, absDst); err != nil {
+		//if c, err := obj.syncCheckApply(apply, absSrc, absDst, excludes); err != nil {
 		//	return false, errwrap.Wrapf(err, "syncCheckApply: recurse rm failed")
 		//} else if !c { // don't let subsequent passes make this true
 		//	checkOK = false
 		//}
+		//if isExcluded(absDst) { // skip removing excluded files
+		//	continue
+		//}
 		//obj.init.Logf("syncCheckApply: removing: %s", absCleanDst)
 		//if apply { // safety
 		//	if err := os.Remove(absCleanDst); err != nil {
@@ -690,7 +940,7 @@ func (obj *FileRes) stateCheckApply(apply bool) (bool, error) {
 	// Optimization: we shouldn't even look at obj.Content here, but we can
 	// skip this empty file creation here since we know we're going to be
 	// making it there anyways. This way we save the extra fopen noise.
-	if obj.Content != nil {
+	if obj.Content != nil || len(obj.Fragments) > 0 {
 		return false, nil // pretend we actually made it
 	}
 
@@ -718,6 +968,7 @@ func (obj *FileRes) contentCheckApply(apply bool) (bool, error) {
 		return true, nil
 	}
 
+	// Actually write the file. This is similar to fragmentsCheckApply.
 	bufferSrc := bytes.NewReader([]byte(*obj.Content))
 	sha256sum, checkOK, err := obj.fileCheckApply(apply, bufferSrc, obj.getPath(), obj.sha256sum)
 	if sha256sum != "" { // empty values mean errored or didn't hash
@@ -736,11 +987,48 @@ func (obj *FileRes) sourceCheckApply(apply bool) (bool, error) {
 	obj.init.Logf("sourceCheckApply(%t)", apply)
 
 	// source is not defined, leave it alone...
-	if obj.Source == "" {
+	if obj.Source == "" && !obj.Purge {
 		return true, nil
 	}
 
-	checkOK, err := obj.syncCheckApply(apply, obj.Source, obj.getPath())
+	excludes := []string{}
+
+	// If we're running a purge, do it here.
+	if obj.Purge {
+		graph, err := obj.init.FilteredGraph()
+		if err != nil {
+			return false, errwrap.Wrapf(err, "can't read filtered graph")
+		}
+		for _, vertex := range graph.Vertices() {
+			res, ok := vertex.(engine.Res)
+			if !ok {
+				// programming error
+				return false, fmt.Errorf("not a Res")
+			}
+			if res.Kind() != KindFile {
+				continue // only interested in files
+			}
+			if res.Name() == obj.Name() {
+				continue // skip me!
+			}
+			fileRes, ok := res.(*FileRes)
+			if !ok {
+				// programming error
+				return false, fmt.Errorf("not a FileRes")
+			}
+			p := fileRes.getPath() // if others use it, make public!
+			if !util.HasPathPrefix(p, obj.getPath()) {
+				continue
+			}
+			excludes = append(excludes, p)
+		}
+	}
+	if obj.init.Debug {
+		obj.init.Logf("syncCheckApply: excludes: %+v", excludes)
+	}
+
+	// XXX: should this work with obj.Purge && obj.Source != "" or not?
+	checkOK, err := obj.syncCheckApply(apply, obj.Source, obj.getPath(), excludes)
 	if err != nil {
 		obj.init.Logf("syncCheckApply: error: %v", err)
 		return false, err
@@ -749,6 +1037,66 @@ func (obj *FileRes) sourceCheckApply(apply bool) (bool, error) {
 	return checkOK, nil
 }
 
+// fragmentsCheckApply performs a CheckApply for the file fragments.
+func (obj *FileRes) fragmentsCheckApply(apply bool) (bool, error) {
+	obj.init.Logf("fragmentsCheckApply(%t)", apply)
+
+	// fragments is not defined, leave it alone...
+	if len(obj.Fragments) == 0 {
+		return true, nil
+	}
+
+	content := ""
+	// TODO: In the future we could have a flag that merges and then sorts
+	// all the individual files in each directory before they are combined.
+	for _, frag := range obj.Fragments {
+		// It's a single file. Add it to what we're building...
+		if isDir := strings.HasSuffix(frag, "/"); !isDir {
+			out, err := ioutil.ReadFile(frag)
+			if err != nil {
+				return false, errwrap.Wrapf(err, "could not read file fragment")
+			}
+			content += string(out)
+			continue
+		}
+
+		// We're a dir, peer inside...
+		files, err := ioutil.ReadDir(frag)
+		if err != nil {
+			return false, errwrap.Wrapf(err, "could not read fragment directory")
+		}
+		// TODO: Add a sort and filter option so that we can choose the
+		// way we iterate through this directory to build out the file.
+		for _, file := range files {
+			if file.IsDir() { // skip recursive solutions for now...
+				continue
+			}
+			f := path.Join(frag, file.Name())
+			out, err := ioutil.ReadFile(f)
+			if err != nil {
+				return false, errwrap.Wrapf(err, "could not read directory file fragment")
+			}
+			content += string(out)
+		}
+	}
+
+	// Actually write the file. This is similar to contentCheckApply.
+	bufferSrc := bytes.NewReader([]byte(content))
+	// NOTE: We pass in an invalidated sha256sum cache since we don't cache
+	// all the individual files, and it could all change without us knowing.
+	// TODO: Is the sha256sum caching even having an effect at all here ???
+	sha256sum, checkOK, err := obj.fileCheckApply(apply, bufferSrc, obj.getPath(), "")
+	if sha256sum != "" { // empty values mean errored or didn't hash
+		// this can be valid even when the whole function errors
+		obj.sha256sum = sha256sum // cache value
+	}
+	if err != nil {
+		return false, err
+	}
+	// if no err, but !ok, then...
+	return checkOK, nil // success
+}
+
 // chownCheckApply performs a CheckApply for the file ownership.
 func (obj *FileRes) chownCheckApply(apply bool) (bool, error) {
 	obj.init.Logf("chownCheckApply(%t)", apply)
@@ -858,7 +1206,8 @@ func (obj *FileRes) CheckApply(apply bool) (bool, error) {
 
 	checkOK := true
 
-	// run stateCheckApply before contentCheckApply and sourceCheckApply
+	// Run stateCheckApply before contentCheckApply, sourceCheckApply, and
+	// fragmentsCheckApply.
 	if c, err := obj.stateCheckApply(apply); err != nil {
 		return false, err
 	} else if !c {
@@ -874,6 +1223,11 @@ func (obj *FileRes) CheckApply(apply bool) (bool, error) {
 	} else if !c {
 		checkOK = false
 	}
+	if c, err := obj.fragmentsCheckApply(apply); err != nil {
+		return false, err
+	} else if !c {
+		checkOK = false
+	}
 
 	if c, err := obj.chownCheckApply(apply); err != nil {
 		return false, err
@@ -918,6 +1272,14 @@ func (obj *FileRes) Cmp(r engine.Res) error {
 	if obj.Source != res.Source {
 		return fmt.Errorf("the Source differs")
 	}
+	if len(obj.Fragments) != len(res.Fragments) {
+		return fmt.Errorf("the number of Fragments differs")
+	}
+	for i, x := range obj.Fragments {
+		if frag := res.Fragments[i]; x != frag {
+			return fmt.Errorf("the fragment at index %d differs", i)
+		}
+	}
 
 	if obj.Owner != res.Owner {
 		return fmt.Errorf("the Owner differs")
@@ -937,6 +1299,9 @@ func (obj *FileRes) Cmp(r engine.Res) error {
 	if obj.Force != res.Force {
 		return fmt.Errorf("the Force option differs")
 	}
+	if obj.Purge != res.Purge {
+		return fmt.Errorf("the Purge option differs")
+	}
 
 	return nil
 }
@@ -958,6 +1323,11 @@ func (obj *FileUID) IFF(uid engine.ResUID) bool {
 
 // FileResAutoEdges holds the state of the auto edge generator.
 type FileResAutoEdges struct {
+	// We do all of these first...
+	frags []engine.ResUID
+	fdone bool
+
+	// Then this is the second part...
 	data    []engine.ResUID
 	pointer int
 	found   bool
@@ -965,6 +1335,12 @@ type FileResAutoEdges struct {
 
 // Next returns the next automatic edge.
 func (obj *FileResAutoEdges) Next() []engine.ResUID {
+	// We do all of these first...
+	if !obj.fdone && len(obj.frags) > 0 {
+		return obj.frags // return them all at the same time
+	}
+
+	// Then this is the second part...
 	if obj.found {
 		panic("Shouldn't be called anymore!")
 	}
@@ -976,8 +1352,16 @@ func (obj *FileResAutoEdges) Next() []engine.ResUID {
 	return []engine.ResUID{value} // we return one, even though api supports N
 }
 
-// Test gets results of the earlier Next() call, & returns if we should continue!
+// Test gets results of the earlier Next() call, & returns if we should
+// continue!
 func (obj *FileResAutoEdges) Test(input []bool) bool {
+	// We do all of these first...
+	if !obj.fdone && len(obj.frags) > 0 {
+		obj.fdone = true // mark as done
+		return true      // keep going
+	}
+
+	// Then this is the second part...
 	// if there aren't any more remaining
 	if len(obj.data) <= obj.pointer {
 		return false
@@ -1013,15 +1397,32 @@ func (obj *FileRes) AutoEdges() (engine.AutoEdge, error) {
 			path: x, // what matters
 		}) // build list
 	}
+
+	// Ensure any file or dir fragments come first.
+	frags := []engine.ResUID{}
+	for _, frag := range obj.Fragments {
+		var reversed = true // cheat by passing a pointer
+		frags = append(frags, &FileUID{
+			BaseUID: engine.BaseUID{
+				Name:     obj.Name(),
+				Kind:     obj.Kind(),
+				Reversed: &reversed,
+			},
+			path: frag, // what matters
+		}) // build list
+
+	}
+
 	return &FileResAutoEdges{
+		frags:   frags,
 		data:    data,
 		pointer: 0,
 		found:   false,
 	}, nil
 }
 
-// UIDs includes all params to make a unique identification of this object.
+// UIDs includes all params to make a unique identification of this object. Most
-// Most resources only return one, although some resources can return multiple.
+// resources only return one, although some resources can return multiple.
 func (obj *FileRes) UIDs() []engine.ResUID {
 	x := &FileUID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
@@ -1047,8 +1448,8 @@ func (obj *FileRes) CollectPattern(pattern string) {
 	obj.Dirname = pattern // XXX: simplistic for now
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
-// It is primarily useful for setting the defaults.
+// primarily useful for setting the defaults.
 func (obj *FileRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes FileRes // indirection to avoid infinite recursion
 
@@ -1075,18 +1476,24 @@ func (obj *FileRes) Copy() engine.CopyableRes {
 		s := *obj.Content
 		content = &s
 	}
+	fragments := []string{}
+	for _, frag := range obj.Fragments {
+		fragments = append(fragments, frag)
+	}
 	return &FileRes{
-		Path:     obj.Path,
+		Path:      obj.Path,
-		Dirname:  obj.Dirname,
+		Dirname:   obj.Dirname,
-		Basename: obj.Basename,
+		Basename:  obj.Basename,
-		State:    obj.State, // TODO: if this becomes a pointer, copy the string!
+		State:     obj.State, // TODO: if this becomes a pointer, copy the string!
-		Content:  content,
+		Content:   content,
-		Source:   obj.Source,
+		Source:    obj.Source,
-		Owner:    obj.Owner,
+		Fragments: fragments,
-		Group:    obj.Group,
+		Owner:     obj.Owner,
-		Mode:     obj.Mode,
+		Group:     obj.Group,
-		Recurse:  obj.Recurse,
+		Mode:      obj.Mode,
-		Force:    obj.Force,
+		Recurse:   obj.Recurse,
+		Force:     obj.Force,
+		Purge:     obj.Purge,
 	}
 }
 
@@ -1133,8 +1540,9 @@ func (obj *FileRes) Reversed() (engine.ReversibleRes, error) {
 
 	// If we've specified content, we might need to restore the original, OR
 	// if we're removing the file with a `state => "absent"`, save it too...
+	// We do this whether we specified content with Content or w/ Fragments.
 	// The `res.State != FileStateAbsent` check is an optional optimization.
-	if (obj.Content != nil || obj.State == FileStateAbsent) && res.State != FileStateAbsent {
+	if ((obj.Content != nil || len(obj.Fragments) > 0) || obj.State == FileStateAbsent) && res.State != FileStateAbsent {
 		content, err := ioutil.ReadFile(obj.getPath())
 		if err != nil && !os.IsNotExist(err) {
 			return nil, errwrap.Wrapf(err, "could not read file for reversal storage")
@@ -1154,6 +1562,13 @@ func (obj *FileRes) Reversed() (engine.ReversibleRes, error) {
 		return nil, fmt.Errorf("can't reverse with Source yet")
 	}
 
+	// We suck in the previous file contents above when Fragments is used...
+	// This is basically the very same code path as when we reverse Content.
+	// TODO: Do we want to do it this way or is there a better reverse path?
+	if len(obj.Fragments) > 0 {
+		res.Fragments = []string{}
+	}
+
 	// There is a race if the operating system is adding/changing/removing
 	// the file between the ioutil.Readfile at the top and here. If there is
 	// a discrepancy between the two, then you might get an unexpected
@@ -1191,6 +1606,18 @@ func (obj *FileRes) Reversed() (engine.ReversibleRes, error) {
 	return res, nil
 }
 
+// GraphQueryAllowed returns nil if you're allowed to query the graph. This
+// function accepts information about the requesting resource so we can
+// determine the access with some form of fine-grained control.
+func (obj *FileRes) GraphQueryAllowed(opts ...engine.GraphQueryableOption) error {
+	options := &engine.GraphQueryableOptions{} // default options
+	options.Apply(opts...)                     // apply the options
+	if options.Kind != KindFile {
+		return fmt.Errorf("only other files can access my information")
+	}
+	return nil
+}
+
 // smartPath adds a trailing slash to the path if it is a directory.
 func smartPath(fileInfo os.FileInfo) string {
 	smartPath := fileInfo.Name() // absolute path

engine/resources/file_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/resources/group.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -271,8 +271,8 @@ func (obj *GroupUID) IFF(uid engine.ResUID) bool {
 	return true
 }
 
-// UIDs includes all params to make a unique identification of this object.
+// UIDs includes all params to make a unique identification of this object. Most
-// Most resources only return one, although some resources can return multiple.
+// resources only return one, although some resources can return multiple.
 func (obj *GroupRes) UIDs() []engine.ResUID {
 	x := &GroupUID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
@@ -282,8 +282,8 @@ func (obj *GroupRes) UIDs() []engine.ResUID {
 	return []engine.ResUID{x}
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
-// It is primarily useful for setting the defaults.
+// primarily useful for setting the defaults.
 func (obj *GroupRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes GroupRes // indirection to avoid infinite recursion
 

engine/resources/hostname.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -46,12 +46,12 @@ var ErrResourceInsufficientParameters = errors.New("insufficient parameters for
 
 // HostnameRes is a resource that allows setting and watching the hostname.
 //
-// StaticHostname is the one configured in /etc/hostname or a similar file.
+// StaticHostname is the one configured in /etc/hostname or a similar file. It
-// It is chosen by the local user. It is not always in sync with the current
+// is chosen by the local user. It is not always in sync with the current host
-// host name as returned by the gethostname() system call.
+// name as returned by the gethostname() system call.
 //
-// TransientHostname is the one configured via the kernel's sethostbyname().
+// TransientHostname is the one configured via the kernel's sethostbyname(). It
-// It can be different from the static hostname in case DHCP or mDNS have been
+// can be different from the static hostname in case DHCP or mDNS have been
 // configured to change the name based on network information.
 //
 // PrettyHostname is a free-form UTF8 host name for presentation to the user.
@@ -248,8 +248,8 @@ type HostnameUID struct {
 	transientHostname string
 }
 
-// UIDs includes all params to make a unique identification of this object.
+// UIDs includes all params to make a unique identification of this object. Most
-// Most resources only return one, although some resources can return multiple.
+// resources only return one, although some resources can return multiple.
 func (obj *HostnameRes) UIDs() []engine.ResUID {
 	x := &HostnameUID{
 		BaseUID:           engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
@@ -261,8 +261,8 @@ func (obj *HostnameRes) UIDs() []engine.ResUID {
 	return []engine.ResUID{x}
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
-// It is primarily useful for setting the defaults.
+// primarily useful for setting the defaults.
 func (obj *HostnameRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes HostnameRes // indirection to avoid infinite recursion
 

engine/resources/http.go

@@ -0,0 +1,808 @@
+// Mgmt
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/purpleidea/mgmt/engine"
+	"github.com/purpleidea/mgmt/engine/traits"
+	"github.com/purpleidea/mgmt/util/errwrap"
+
+	securefilepath "github.com/cyphar/filepath-securejoin"
+)
+
+func init() {
+	engine.RegisterResource("http:server", func() engine.Res { return &HTTPServerRes{} })
+	engine.RegisterResource("http:file", func() engine.Res { return &HTTPFileRes{} })
+}
+
+const (
+	// HTTPUseSecureJoin specifies that we should add in a "secure join" lib
+	// so that we avoid the ../../etc/passwd and symlink problems.
+	HTTPUseSecureJoin = true
+)
+
+// HTTPServerRes is an http server resource. It serves files, but does not
+// actually apply any state. The name is used as the address to listen on,
+// unless the Address field is specified, and in that case it is used instead.
+// This resource can offer up files for serving that are specified either inline
+// in this resource by specifying an http root, or as http:file resources which
+// will get autogrouped into this resource at runtime. The two methods can be
+// combined as well.
+//
+// This server also supports autogrouping some more magical resources into it.
+// For example, the http:flag and http:ui resources add in magic endpoints.
+//
+// This server is not meant as a featureful replacement for the venerable and
+// modern httpd servers out there, but rather as a simple, dynamic, integrated
+// alternative for bootstrapping new machines and clusters in an elegant way.
+//
+// TODO: add support for TLS
+// XXX: Add an http:flag resource that lets an http client set a flag somewhere!
+// XXX: Add a http:ui resource that functions can read data from!
+// XXX: The http:ui resource can also take in values from those functions!
+type HTTPServerRes struct {
+	traits.Base      // add the base methods without re-implementation
+	traits.Edgeable  // XXX: add autoedge support
+	traits.Groupable // can have HTTPFileRes grouped into it
+
+	init *engine.Init
+
+	// Address is the listen address to use for the http server. It is
+	// common to use `:80` (the standard) to listen on TCP port 80 on all
+	// addresses.
+	Address string `lang:"address" yaml:"address"`
+
+	// Timeout is the maximum duration in seconds to use for unspecified
+	// timeouts. In other words, when this value is specified, it is used as
+	// the value for the other *Timeout values when they aren't used. Put
+	// another way, this makes it easy to set all the different timeouts
+	// with a single parameter.
+	Timeout *uint64 `lang:"timeout" yaml:"timeout"`
+
+	// ReadTimeout is the maximum duration in seconds for reading during the
+	// http request. If it is zero, then there is no timeout. If this is
+	// unspecified, then the value of Timeout is used instead if it is set.
+	// For more information, see the golang net/http Server documentation.
+	ReadTimeout *uint64 `lang:"read_timeout" yaml:"read_timeout"`
+
+	// WriteTimeout is the maximum duration in seconds for writing during
+	// the http request. If it is zero, then there is no timeout. If this is
+	// unspecified, then the value of Timeout is used instead if it is set.
+	// For more information, see the golang net/http Server documentation.
+	WriteTimeout *uint64 `lang:"write_timeout" yaml:"write_timeout"`
+
+	// ShutdownTimeout is the maximum duration in seconds to wait for the
+	// server to shutdown gracefully before calling Close. By default it is
+	// nice to let client connections terminate gracefully, however it might
+	// take longer than we are willing to wait, particularly if one is long
+	// polling or running a very long download. As a result, you can set a
+	// timeout here. The default is zero which means it will wait
+	// indefinitely. The shutdown process can also be cancelled by the
+	// interrupt handler which this resource supports. If this is
+	// unspecified, then the value of Timeout is used instead if it is set.
+	ShutdownTimeout *uint64 `lang:"shutdown_timeout" yaml:"shutdown_timeout"`
+
+	// Root is the root directory that we should serve files from. If it is
+	// not specified, then it is not used. Any http file resources will have
+	// precedence over anything in here, in case the same path exists twice.
+	// TODO: should we have a flag to determine the precedence rules here?
+	Root string `lang:"root" yaml:"root"`
+
+	// TODO: should we allow adding a list of one-of files directly here?
+
+	interruptChan chan struct{}
+
+	conn     net.Listener
+	serveMux *http.ServeMux // can't share the global one between resources!
+	server   *http.Server
+}
+
+// Default returns some sensible defaults for this resource.
+func (obj *HTTPServerRes) Default() engine.Res {
+	return &HTTPServerRes{}
+}
+
+// getAddress returns the actual address to use. When Address is not specified,
+// we use the Name.
+func (obj *HTTPServerRes) getAddress() string {
+	if obj.Address != "" {
+		return obj.Address
+	}
+	return obj.Name()
+}
+
+// getReadTimeout determines the value for ReadTimeout, because if unspecified,
+// this will default to the value of Timeout.
+func (obj *HTTPServerRes) getReadTimeout() *uint64 {
+	if obj.ReadTimeout != nil {
+		return obj.ReadTimeout
+	}
+	return obj.Timeout // might be nil
+}
+
+// getWriteTimeout determines the value for WriteTimeout, because if
+// unspecified, this will default to the value of Timeout.
+func (obj *HTTPServerRes) getWriteTimeout() *uint64 {
+	if obj.WriteTimeout != nil {
+		return obj.WriteTimeout
+	}
+	return obj.Timeout // might be nil
+}
+
+// getShutdownTimeout determines the value for ShutdownTimeout, because if
+// unspecified, this will default to the value of Timeout.
+func (obj *HTTPServerRes) getShutdownTimeout() *uint64 {
+	if obj.ShutdownTimeout != nil {
+		return obj.ShutdownTimeout
+	}
+	return obj.Timeout // might be nil
+}
+
+// Validate checks if the resource data structure was populated correctly.
+func (obj *HTTPServerRes) Validate() error {
+	if obj.getAddress() == "" {
+		return fmt.Errorf("empty address")
+	}
+
+	host, _, err := net.SplitHostPort(obj.getAddress())
+	if err != nil {
+		return errwrap.Wrapf(err, "the Address is in an invalid format: %s", obj.getAddress())
+	}
+	if host != "" {
+		// TODO: should we allow fqdn's here?
+		ip := net.ParseIP(host)
+		if ip == nil {
+			return fmt.Errorf("the Address is not a valid IP: %s", host)
+		}
+	}
+
+	if obj.Root != "" && !strings.HasPrefix(obj.Root, "/") {
+		return fmt.Errorf("the Root must be absolute")
+	}
+	if obj.Root != "" && !strings.HasSuffix(obj.Root, "/") {
+		return fmt.Errorf("the Root must be a dir")
+	}
+
+	// XXX: validate that the autogrouped resources don't have paths that
+	// conflict with each other. We can only have a single unique entry for
+	// what handles a /whatever URL.
+
+	return nil
+}
+
+// Init runs some startup code for this resource.
+func (obj *HTTPServerRes) Init(init *engine.Init) error {
+	obj.init = init // save for later
+
+	// No need to error in Validate if Timeout is ignored, but log it.
+	// These are all specified, so Timeout effectively does nothing.
+	a := obj.ReadTimeout != nil
+	b := obj.WriteTimeout != nil
+	c := obj.ShutdownTimeout != nil
+	if obj.Timeout != nil && (a && b && c) {
+		obj.init.Logf("the Timeout param is being ignored")
+	}
+
+	// NOTE: If we don't Init anything that's autogrouped, then it won't
+	// even get an Init call on it.
+	// TODO: should we do this in the engine? Do we want to decide it here?
+	for _, res := range obj.GetGroup() { // grouped elements
+		if err := res.Init(init); err != nil {
+			return errwrap.Wrapf(err, "autogrouped Init failed")
+		}
+	}
+
+	obj.interruptChan = make(chan struct{})
+
+	return nil
+}
+
+// Close is run by the engine to clean up after the resource is done.
+func (obj *HTTPServerRes) Close() error {
+	return nil
+}
+
+// Watch is the primary listener for this resource and it outputs events.
+func (obj *HTTPServerRes) Watch() error {
+	// TODO: I think we could replace all this with:
+	//obj.conn, err := net.Listen("tcp", obj.getAddress())
+	// ...but what is the advantage?
+	addr, err := net.ResolveTCPAddr("tcp", obj.getAddress())
+	if err != nil {
+		return errwrap.Wrapf(err, "could not resolve address")
+	}
+
+	obj.conn, err = net.ListenTCP("tcp", addr)
+	if err != nil {
+		return errwrap.Wrapf(err, "could not start listener")
+	}
+	defer obj.conn.Close()
+
+	obj.serveMux = http.NewServeMux() // do it here in case Watch restarts!
+	obj.serveMux.HandleFunc("/", obj.handler())
+
+	readTimeout := uint64(0)
+	if i := obj.getReadTimeout(); i != nil {
+		readTimeout = *i
+	}
+	writeTimeout := uint64(0)
+	if i := obj.getWriteTimeout(); i != nil {
+		writeTimeout = *i
+	}
+	obj.server = &http.Server{
+		Addr:         obj.getAddress(),
+		Handler:      obj.serveMux,
+		ReadTimeout:  time.Duration(readTimeout) * time.Second,
+		WriteTimeout: time.Duration(writeTimeout) * time.Second,
+		//MaxHeaderBytes: 1 << 20, XXX: should we add a param for this?
+	}
+
+	obj.init.Running() // when started, notify engine that we're running
+
+	var closeError error
+	closeSignal := make(chan struct{})
+
+	wg := &sync.WaitGroup{}
+	defer wg.Wait()
+
+	shutdownChan := make(chan struct{}) // server shutdown finished signal
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		select {
+		case <-obj.interruptChan:
+			// TODO: should we bubble up the error from Close?
+			// TODO: do we need a mutex around this Close?
+			obj.server.Close() // kill it quickly!
+		case <-shutdownChan:
+			// let this exit
+		}
+	}()
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		defer close(closeSignal)
+
+		err := obj.server.Serve(obj.conn) // blocks until Shutdown() is called!
+		if err == nil || err == http.ErrServerClosed {
+			return
+		}
+		// if this returned on its own, then closeSignal can be used...
+		closeError = errwrap.Wrapf(err, "the server errored")
+	}()
+
+	// When Shutdown is called, Serve, ListenAndServe, and ListenAndServeTLS
+	// immediately return ErrServerClosed. Make sure the program doesn't
+	// exit and waits instead for Shutdown to return.
+	defer func() {
+		defer close(shutdownChan) // signal that shutdown is finished
+		ctx := context.Background()
+		if i := obj.getShutdownTimeout(); i != nil && *i > 0 {
+			var cancel context.CancelFunc
+			ctx, cancel = context.WithTimeout(ctx, time.Duration(*i)*time.Second)
+			defer cancel()
+		}
+		err := obj.server.Shutdown(ctx) // shutdown gracefully
+		if err == context.DeadlineExceeded {
+			// TODO: should we bubble up the error from Close?
+			// TODO: do we need a mutex around this Close?
+			obj.server.Close() // kill it now
+		}
+	}()
+
+	startupChan := make(chan struct{})
+	close(startupChan) // send one initial signal
+
+	var send = false // send event?
+	for {
+		if obj.init.Debug {
+			obj.init.Logf("Looping...")
+		}
+
+		select {
+		case <-startupChan:
+			startupChan = nil
+			send = true
+
+		case <-closeSignal: // something shut us down early
+			return closeError
+
+		case <-obj.init.Done: // closed by the engine to signal shutdown
+			return nil
+		}
+
+		// do all our event sending all together to avoid duplicate msgs
+		if send {
+			send = false
+			obj.init.Event() // notify engine of an event (this can block)
+		}
+	}
+}
+
+// CheckApply never has anything to do for this resource, so it always succeeds.
+// It does however check that certain runtime requirements (such as the Root dir
+// existing if one was specified) are fulfilled.
+func (obj *HTTPServerRes) CheckApply(apply bool) (bool, error) {
+	if obj.init.Debug {
+		obj.init.Logf("CheckApply")
+	}
+
+	// XXX: We don't want the initial CheckApply to return true until the
+	// Watch has started up, so we must block here until that's the case...
+
+	// Cheap runtime validation!
+	if obj.Root != "" {
+		fileInfo, err := os.Stat(obj.Root)
+		if err != nil {
+			return false, errwrap.Wrapf(err, "can't stat Root dir")
+		}
+		if !fileInfo.IsDir() {
+			return false, fmt.Errorf("the Root path is not a dir")
+		}
+	}
+
+	return true, nil // always succeeds, with nothing to do!
+}
+
+// Cmp compares two resources and returns an error if they are not equivalent.
+func (obj *HTTPServerRes) Cmp(r engine.Res) error {
+	// we can only compare HTTPServerRes to others of the same resource kind
+	res, ok := r.(*HTTPServerRes)
+	if !ok {
+		return fmt.Errorf("res is not the same kind")
+	}
+
+	if obj.Address != res.Address {
+		return fmt.Errorf("the Address differs")
+	}
+
+	if (obj.Timeout == nil) != (res.Timeout == nil) { // xor
+		return fmt.Errorf("the Timeout differs")
+	}
+	if obj.Timeout != nil && res.Timeout != nil {
+		if *obj.Timeout != *res.Timeout { // compare the values
+			return fmt.Errorf("the value of Timeout differs")
+		}
+	}
+	if (obj.ReadTimeout == nil) != (res.ReadTimeout == nil) {
+		return fmt.Errorf("the ReadTimeout differs")
+	}
+	if obj.ReadTimeout != nil && res.ReadTimeout != nil {
+		if *obj.ReadTimeout != *res.ReadTimeout {
+			return fmt.Errorf("the value of ReadTimeout differs")
+		}
+	}
+	if (obj.WriteTimeout == nil) != (res.WriteTimeout == nil) {
+		return fmt.Errorf("the WriteTimeout differs")
+	}
+	if obj.WriteTimeout != nil && res.WriteTimeout != nil {
+		if *obj.WriteTimeout != *res.WriteTimeout {
+			return fmt.Errorf("the value of WriteTimeout differs")
+		}
+	}
+	if (obj.ShutdownTimeout == nil) != (res.ShutdownTimeout == nil) {
+		return fmt.Errorf("the ShutdownTimeout differs")
+	}
+	if obj.ShutdownTimeout != nil && res.ShutdownTimeout != nil {
+		if *obj.ShutdownTimeout != *res.ShutdownTimeout {
+			return fmt.Errorf("the value of ShutdownTimeout differs")
+		}
+	}
+
+	// TODO: We could do this sort of thing to skip checking Timeout when it
+	// is not used, but for the moment, this is overkill and not needed yet.
+	//a := obj.ReadTimeout != nil
+	//b := obj.WriteTimeout != nil
+	//c := obj.ShutdownTimeout != nil
+	//if !(obj.Timeout != nil && (a && b && c)) {
+	//	// the Timeout param is not being ignored
+	//}
+
+	if obj.Root != res.Root {
+		return fmt.Errorf("the Root differs")
+	}
+
+	return nil
+}
+
+// Interrupt is called to ask the execution of this resource to end early. It
+// will cause the server Shutdown to end abruptly instead of leading open client
+// connections terminate gracefully. It does this by causing the server Close
+// method to run.
+func (obj *HTTPServerRes) Interrupt() error {
+	close(obj.interruptChan) // this should cause obj.server.Close() to run!
+	return nil
+}
+
+// Copy copies the resource. Don't call it directly, use engine.ResCopy instead.
+// TODO: should this copy internal state?
+func (obj *HTTPServerRes) Copy() engine.CopyableRes {
+	var timeout, readTimeout, writeTimeout, shutdownTimeout *uint64
+	if obj.Timeout != nil {
+		x := *obj.Timeout
+		timeout = &x
+	}
+	if obj.ReadTimeout != nil {
+		x := *obj.ReadTimeout
+		readTimeout = &x
+	}
+	if obj.WriteTimeout != nil {
+		x := *obj.WriteTimeout
+		writeTimeout = &x
+	}
+	if obj.ShutdownTimeout != nil {
+		x := *obj.ShutdownTimeout
+		shutdownTimeout = &x
+	}
+	return &HTTPServerRes{
+		Address:         obj.Address,
+		Timeout:         timeout,
+		ReadTimeout:     readTimeout,
+		WriteTimeout:    writeTimeout,
+		ShutdownTimeout: shutdownTimeout,
+		Root:            obj.Root,
+	}
+}
+
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
+// primarily useful for setting the defaults.
+func (obj *HTTPServerRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type rawRes HTTPServerRes // indirection to avoid infinite recursion
+
+	def := obj.Default()            // get the default
+	res, ok := def.(*HTTPServerRes) // put in the right format
+	if !ok {
+		return fmt.Errorf("could not convert to HTTPServerRes")
+	}
+	raw := rawRes(*res) // convert; the defaults go here
+
+	if err := unmarshal(&raw); err != nil {
+		return err
+	}
+
+	*obj = HTTPServerRes(raw) // restore from indirection with type conversion!
+	return nil
+}
+
+// GroupCmp returns whether two resources can be grouped together or not. Can
+// these two resources be merged, aka, does this resource support doing so? Will
+// resource allow itself to be grouped _into_ this obj?
+func (obj *HTTPServerRes) GroupCmp(r engine.GroupableRes) error {
+	res1, ok1 := r.(*HTTPFileRes) // different from what we usually do!
+	if ok1 {
+		// If the http file resource has the Server field specified,
+		// then it must match against our name field if we want it to
+		// group with us.
+		if res1.Server != "" && res1.Server != obj.Name() {
+			return fmt.Errorf("resource groups with a different server name")
+		}
+
+		return nil
+	}
+
+	return fmt.Errorf("resource is not the right kind")
+}
+
+// readHandler handles all the incoming download requests from clients.
+func (obj *HTTPServerRes) handler() func(http.ResponseWriter, *http.Request) {
+	// TODO: we could statically pre-compute some stuff here...
+
+	return func(w http.ResponseWriter, req *http.Request) {
+
+		if obj.init.Debug {
+			obj.init.Logf("Client: %s", req.RemoteAddr)
+		}
+		// TODO: would this leak anything security sensitive in our log?
+		obj.init.Logf("URL: %s", req.URL)
+		if obj.init.Debug {
+			obj.init.Logf("Path: %s", req.URL.Path)
+		}
+
+		// We only allow GET at the moment.
+		if req.Method != http.MethodGet {
+			w.WriteHeader(http.StatusMethodNotAllowed)
+			return
+		}
+
+		requestPath := req.URL.Path // TODO: is this what we want here?
+
+		//var handle io.Reader // TODO: simplify?
+		var handle io.ReadSeeker
+
+		// Look through the autogrouped resources!
+		// TODO: can we improve performance by only searching here once?
+		for _, x := range obj.GetGroup() { // grouped elements
+			res, ok := x.(*HTTPFileRes) // convert from Res
+			if !ok {
+				continue
+			}
+			if requestPath != res.getPath() {
+				continue // not me
+			}
+
+			if obj.init.Debug {
+				obj.init.Logf("Got grouped file: %s", res.String())
+			}
+			var err error
+			handle, err = res.getContent()
+			if err != nil {
+				obj.init.Logf("could not get content for: %s", requestPath)
+				msg, httpStatus := toHTTPError(err)
+				http.Error(w, msg, httpStatus)
+				return
+			}
+			break
+		}
+
+		// Look in root if we have one, and we haven't got a file yet...
+		if obj.Root != "" && handle == nil {
+
+			p := filepath.Join(obj.Root, requestPath) // normal unsafe!
+			if !strings.HasPrefix(p, obj.Root) {      // root ends with /
+				// user might have tried a ../../etc/passwd hack
+				obj.init.Logf("join inconsistency: %s", p)
+				http.NotFound(w, req) // lie to them...
+				return
+			}
+			if HTTPUseSecureJoin {
+				var err error
+				p, err = securefilepath.SecureJoin(obj.Root, requestPath)
+				if err != nil {
+					obj.init.Logf("secure join fail: %s", p)
+					http.NotFound(w, req) // lie to them...
+					return
+				}
+			}
+			if obj.init.Debug {
+				obj.init.Logf("Got file at root: %s", p)
+			}
+			var err error
+			handle, err = os.Open(p)
+			if err != nil {
+				obj.init.Logf("could not open: %s", p)
+				msg, httpStatus := toHTTPError(err)
+				http.Error(w, msg, httpStatus)
+				return
+			}
+		}
+
+		// We never found a file...
+		if handle == nil {
+			if obj.init.Debug || true { // XXX: maybe we should always do this?
+				obj.init.Logf("File not found: %s", requestPath)
+			}
+			http.NotFound(w, req)
+			return
+		}
+
+		// Determine the last-modified time if we can.
+		modtime := time.Now()
+		if f, ok := handle.(*os.File); ok {
+			fi, err := f.Stat()
+			if err == nil {
+				modtime = fi.ModTime()
+			}
+			// TODO: if Stat errors, should we fail the whole thing?
+		}
+
+		// XXX: is requestPath what we want for the name field?
+		http.ServeContent(w, req, requestPath, modtime, handle)
+		//obj.init.Logf("%d bytes sent", n) // XXX: how do we know (on the server-side) if it worked?
+
+		return
+	}
+}
+
+// HTTPFileRes is a file that exists within an http server. The name is used as
+// the public path of the file, unless the filename field is specified, and in
+// that case it is used instead. The way this works is that it autogroups at
+// runtime with an existing http resource, and in doing so makes the file
+// associated with this resource available for serving from that http server.
+type HTTPFileRes struct {
+	traits.Base      // add the base methods without re-implementation
+	traits.Edgeable  // XXX: add autoedge support
+	traits.Groupable // can be grouped into HTTPServerRes
+
+	init *engine.Init
+
+	// Server is the name of the http server resource to group this into. If
+	// it is omitted, and there is only a single http resource, then it will
+	// be grouped into it automatically. If there is more than one main http
+	// resource being used, then the grouping behaviour is *undefined* when
+	// this is not specified, and it is not recommended to leave this blank!
+	Server string `lang:"server" yaml:"server"`
+
+	// Filename is the name of the file this data should appear as on the
+	// http server.
+	Filename string `lang:"filename" yaml:"filename"`
+
+	// Path is the absolute path to a file that should be used as the source
+	// for this file resource. It must not be combined with the data field.
+	Path string `lang:"path" yaml:"path"`
+
+	// Data is the file content that should be used as the source for this
+	// file resource. It must not be combined with the path field.
+	// TODO: should this be []byte instead?
+	Data string `lang:"data" yaml:"data"`
+}
+
+// Default returns some sensible defaults for this resource.
+func (obj *HTTPFileRes) Default() engine.Res {
+	return &HTTPFileRes{}
+}
+
+// getPath returns the actual path we respond to. When Filename is not
+// specified, we use the Name. Note that this is the filename that will be seen
+// on the http server, it is *not* the source path to the actual file contents
+// being sent by the server.
+func (obj *HTTPFileRes) getPath() string {
+	if obj.Filename != "" {
+		return obj.Filename
+	}
+	return obj.Name()
+}
+
+// getContent returns the content that we expect from this resource. It depends
+// on whether the user specified the Path or Data fields, and whether the Path
+// exists or not.
+func (obj *HTTPFileRes) getContent() (io.ReadSeeker, error) {
+	if obj.Path != "" && obj.Data != "" {
+		// programming error! this should have been caught in Validate!
+		return nil, fmt.Errorf("must not specify Path and Data")
+	}
+
+	if obj.Path != "" {
+		return os.Open(obj.Path)
+	}
+
+	return bytes.NewReader([]byte(obj.Data)), nil
+}
+
+// Validate checks if the resource data structure was populated correctly.
+func (obj *HTTPFileRes) Validate() error {
+	if obj.getPath() == "" {
+		return fmt.Errorf("empty filename")
+	}
+	// FIXME: does getPath need to start with a slash?
+
+	if obj.Path != "" && !strings.HasPrefix(obj.Path, "/") {
+		return fmt.Errorf("the Path must be absolute")
+	}
+
+	if obj.Path != "" && obj.Data != "" {
+		return fmt.Errorf("must not specify Path and Data")
+	}
+
+	// NOTE: if obj.Path == "" && obj.Data == "" then we have an empty file!
+
+	return nil
+}
+
+// Init runs some startup code for this resource.
+func (obj *HTTPFileRes) Init(init *engine.Init) error {
+	obj.init = init // save for later
+
+	return nil
+}
+
+// Close is run by the engine to clean up after the resource is done.
+func (obj *HTTPFileRes) Close() error {
+	return nil
+}
+
+// Watch is the primary listener for this resource and it outputs events. This
+// particular one does absolutely nothing but block until we've received a done
+// signal.
+func (obj *HTTPFileRes) Watch() error {
+	obj.init.Running() // when started, notify engine that we're running
+
+	select {
+	case <-obj.init.Done: // closed by the engine to signal shutdown
+	}
+
+	//obj.init.Event() // notify engine of an event (this can block)
+
+	return nil
+}
+
+// CheckApply never has anything to do for this resource, so it always succeeds.
+func (obj *HTTPFileRes) CheckApply(apply bool) (bool, error) {
+	if obj.init.Debug {
+		obj.init.Logf("CheckApply")
+	}
+
+	return true, nil // always succeeds, with nothing to do!
+}
+
+// Cmp compares two resources and returns an error if they are not equivalent.
+func (obj *HTTPFileRes) Cmp(r engine.Res) error {
+	// we can only compare HTTPFileRes to others of the same resource kind
+	res, ok := r.(*HTTPFileRes)
+	if !ok {
+		return fmt.Errorf("res is not the same kind")
+	}
+
+	if obj.Server != res.Server {
+		return fmt.Errorf("the Server field differs")
+	}
+	if obj.Filename != res.Filename {
+		return fmt.Errorf("the Filename differs")
+	}
+	if obj.Path != res.Path {
+		return fmt.Errorf("the Path differs")
+	}
+	if obj.Data != res.Data {
+		return fmt.Errorf("the Data differs")
+	}
+
+	return nil
+}
+
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
+// primarily useful for setting the defaults.
+func (obj *HTTPFileRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type rawRes HTTPFileRes // indirection to avoid infinite recursion
+
+	def := obj.Default()          // get the default
+	res, ok := def.(*HTTPFileRes) // put in the right format
+	if !ok {
+		return fmt.Errorf("could not convert to HTTPFileRes")
+	}
+	raw := rawRes(*res) // convert; the defaults go here
+
+	if err := unmarshal(&raw); err != nil {
+		return err
+	}
+
+	*obj = HTTPFileRes(raw) // restore from indirection with type conversion!
+	return nil
+}
+
+// toHTTPError returns a non-specific HTTP error message and status code for a
+// given non-nil error value. It's important that toHTTPError does not actually
+// return err.Error(), since msg and httpStatus are returned to users, and
+// historically Go's ServeContent always returned just "404 Not Found" for all
+// errors. We don't want to start leaking information in error messages.
+// NOTE: This was copied and modified slightly from the golang net/http package.
+// See: https://github.com/golang/go/issues/38375
+func toHTTPError(err error) (msg string, httpStatus int) {
+	if os.IsNotExist(err) {
+		//return "404 page not found", http.StatusNotFound
+		return http.StatusText(http.StatusNotFound), http.StatusNotFound
+	}
+	if os.IsPermission(err) {
+		//return "403 Forbidden", http.StatusForbidden
+		return http.StatusText(http.StatusForbidden), http.StatusForbidden
+	}
+	// Default:
+	//return "500 Internal Server Error", http.StatusInternalServerError
+	return http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError
+}

engine/resources/kv.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -34,10 +34,12 @@ func init() {
 	engine.RegisterResource("kv", func() engine.Res { return &KVRes{} })
 }
 
-// KVResSkipCmpStyle represents the different styles of comparison when using SkipLessThan.
+// KVResSkipCmpStyle represents the different styles of comparison when using
+// SkipLessThan.
 type KVResSkipCmpStyle int
 
-// These are the different allowed comparison styles. Most folks will want SkipCmpStyleInt.
+// These are the different allowed comparison styles. Most folks will want
+// SkipCmpStyleInt.
 const (
 	SkipCmpStyleInt KVResSkipCmpStyle = iota
 	SkipCmpStyleString
@@ -308,8 +310,8 @@ type KVUID struct {
 	name string
 }
 
-// UIDs includes all params to make a unique identification of this object.
+// UIDs includes all params to make a unique identification of this object. Most
-// Most resources only return one, although some resources can return multiple.
+// resources only return one, although some resources can return multiple.
 func (obj *KVRes) UIDs() []engine.ResUID {
 	x := &KVUID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
@@ -318,8 +320,8 @@ func (obj *KVRes) UIDs() []engine.ResUID {
 	return []engine.ResUID{x}
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
-// It is primarily useful for setting the defaults.
+// primarily useful for setting the defaults.
 func (obj *KVRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes KVRes // indirection to avoid infinite recursion
 

engine/resources/mount.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -403,8 +403,8 @@ func (obj *MountUID) IFF(uid engine.ResUID) bool {
 	return obj.name == res.name
 }
 
-// UIDs includes all params to make a unique identification of this object.
+// UIDs includes all params to make a unique identification of this object. Most
-// Most resources only return one although some resources can return multiple.
+// resources only return one although some resources can return multiple.
 func (obj *MountRes) UIDs() []engine.ResUID {
 	x := &MountUID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
@@ -413,8 +413,8 @@ func (obj *MountRes) UIDs() []engine.ResUID {
 	return []engine.ResUID{x}
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
-// It is primarily useful for setting the defaults.
+// primarily useful for setting the defaults.
 func (obj *MountRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes MountRes // indirection to avoid infinite recursion
 
@@ -499,8 +499,8 @@ func (obj *MountRes) fstabEntryRemove(file string, mount *fstab.Mount) error {
 	return obj.fstabWrite(file, mounts)
 }
 
-// fstabWrite generates an fstab file with the given mounts, and writes them
+// fstabWrite generates an fstab file with the given mounts, and writes them to
-// to the provided fstab file.
+// the provided fstab file.
 func (obj *MountRes) fstabWrite(file string, mounts fstab.Mounts) error {
 	// build the file contents
 	contents := fmt.Sprintf("# Generated by %s at %d", obj.init.Program, time.Now().UnixNano()) + "\n"
@@ -541,9 +541,9 @@ func mountExists(file string, mount *fstab.Mount) (bool, error) {
 	return false, nil
 }
 
-// mountCompare compares two mounts. It is assumed that the first comes from
+// mountCompare compares two mounts. It is assumed that the first comes from a
-// a resource definition, and the second comes from /proc/mounts. It compares
+// resource definition, and the second comes from /proc/mounts. It compares the
-// the two after resolving the loopback device's file path (if necessary,) and
+// two after resolving the loopback device's file path (if necessary,) and
 // ignores freq and passno, as they may differ between the definition and
 // /proc/mounts.
 func mountCompare(def, proc *fstab.Mount) (bool, error) {
@@ -599,8 +599,8 @@ func mountReload() error {
 	return nil
 }
 
-// restartUnit restarts the given dbus unit and waits for it to finish
+// restartUnit restarts the given dbus unit and waits for it to finish starting
-// starting up. If restartTimeout is exceeded, it will return an error.
+// up. If restartTimeout is exceeded, it will return an error.
 func restartUnit(conn *dbus.Conn, unit string) error {
 	// timeout if we don't get the JobRemoved event
 	ctx, cancel := context.WithTimeout(context.TODO(), dbusRestartCtxTimeout*time.Second)

engine/resources/mount_linux_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/resources/mount_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/resources/msg.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -231,8 +231,8 @@ type MsgUID struct {
 	body string
 }
 
-// UIDs includes all params to make a unique identification of this object.
+// UIDs includes all params to make a unique identification of this object. Most
-// Most resources only return one, although some resources can return multiple.
+// resources only return one, although some resources can return multiple.
 func (obj *MsgRes) UIDs() []engine.ResUID {
 	x := &MsgUID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
@@ -241,8 +241,8 @@ func (obj *MsgRes) UIDs() []engine.ResUID {
 	return []engine.ResUID{x}
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
-// It is primarily useful for setting the defaults.
+// primarily useful for setting the defaults.
 func (obj *MsgRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes MsgRes // indirection to avoid infinite recursion
 

engine/resources/msg_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/resources/net.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -81,17 +81,32 @@ const (
 	socketFile = "pipe.sock" // path in vardir to store our socket file
 )
 
-// NetRes is a network interface resource based on netlink. It manages the
+// NetRes is a network interface resource based on netlink. It manages the state
-// state of a network link. Configuration is also stored in a networkd
+// of a network link. Configuration is also stored in a networkd configuration
-// configuration file, so the network is available upon reboot.
+// file, so the network is available upon reboot. The name of the resource is
+// the string representing the network interface name. This could be "eth0" for
+// example.
 type NetRes struct {
 	traits.Base // add the base methods without re-implementation
 
 	init *engine.Init
 
-	State   string   `yaml:"state"`   // up, down, or empty
+	// State is the desired state of the interface. It can be "up", "down",
-	Addrs   []string `yaml:"addrs"`   // list of addresses in cidr format
+	// or the empty string to leave that unspecified.
-	Gateway string   `yaml:"gateway"` // gateway address
+	State string `lang:"state" yaml:"state"`
+
+	// Addrs is the list of addresses to set on the interface. They must
+	// each be in CIDR notation such as: 192.0.2.42/24 for example.
+	Addrs []string `lang:"addrs" yaml:"addrs"`
+
+	// Gateway represents the default route to set for the interface.
+	Gateway string `lang:"gateway" yaml:"gateway"`
+
+	// IPForward is a boolean that sets whether we should forward incoming
+	// packets onward when this is set. It default to unspecified, which
+	// downstream (in the systemd-networkd configuration) defaults to false.
+	// XXX: this could also be "ipv4" or "ipv6", add those as a second option?
+	IPForward *bool `lang:"ip_forward" yaml:"ip_forward"`
 
 	iface        *iface // a struct containing the net.Interface and netlink.Link
 	unitFilePath string // the interface unit file path
@@ -99,8 +114,8 @@ type NetRes struct {
 	socketFile string // path for storing the pipe socket file
 }
 
-// nlChanStruct defines the channel used to send netlink messages and errors
+// nlChanStruct defines the channel used to send netlink messages and errors to
-// to the event processing loop in Watch.
+// the event processing loop in Watch.
 type nlChanStruct struct {
 	msg []syscall.NetlinkMessage
 	err error
@@ -371,8 +386,8 @@ func (obj *NetRes) addrCheckApply(apply bool) (bool, error) {
 	return false, nil
 }
 
-// gatewayCheckApply checks if the interface has the correct default gateway
+// gatewayCheckApply checks if the interface has the correct default gateway and
-// and adds/deletes routes as necessary.
+// adds/deletes routes as necessary.
 func (obj *NetRes) gatewayCheckApply(apply bool) (bool, error) {
 	// get all routes from the interface
 	routes, err := netlink.RouteList(obj.iface.link, netlink.FAMILY_V4)
@@ -548,8 +563,8 @@ func (obj *NetUID) IFF(uid engine.ResUID) bool {
 	return obj.name == res.name
 }
 
-// UIDs includes all params to make a unique identification of this object.
+// UIDs includes all params to make a unique identification of this object. Most
-// Most resources only return one although some resources can return multiple.
+// resources only return one although some resources can return multiple.
 func (obj *NetRes) UIDs() []engine.ResUID {
 	x := &NetUID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
@@ -558,8 +573,8 @@ func (obj *NetRes) UIDs() []engine.ResUID {
 	return []engine.ResUID{x}
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
-// It is primarily useful for setting the defaults.
+// primarily useful for setting the defaults.
 func (obj *NetRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes NetRes // indirection to avoid infinite recursion
 
@@ -590,6 +605,13 @@ func (obj *NetRes) unitFileContents() []byte {
 	if obj.Gateway != "" {
 		u = append(u, fmt.Sprintf("Gateway=%s", obj.Gateway))
 	}
+	if obj.IPForward != nil {
+		b := "false"
+		if *obj.IPForward {
+			b = "true"
+		}
+		u = append(u, fmt.Sprintf("IPForward=%s", b))
+	}
 	c := strings.Join(u, "\n")
 	return []byte(c)
 }
@@ -625,8 +647,8 @@ func (obj *iface) linkUpDown(state string) error {
 	return netlink.LinkSetDown(obj.link)
 }
 
-// getAddrs returns a list of strings containing all of the interface's
+// getAddrs returns a list of strings containing all of the interface's IP
-// IP addresses in CIDR format.
+// addresses in CIDR format.
 func (obj *iface) getAddrs() ([]string, error) {
 	var ifaceAddrs []string
 	a, err := obj.iface.Addrs()
@@ -694,8 +716,8 @@ func (obj *iface) kernelApply(addrs []string) error {
 	return nil
 }
 
-// addrApplyDelete, checks the interface's addresses and deletes any that are not
+// addrApplyDelete, checks the interface's addresses and deletes any that are
-// in the list/definition.
+// not in the list/definition.
 func (obj *iface) addrApplyDelete(objAddrs []string) error {
 	ifaceAddrs, err := obj.getAddrs()
 	if err != nil {

engine/resources/net_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/resources/noop.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -103,8 +103,8 @@ type NoopUID struct {
 	name string
 }
 
-// UIDs includes all params to make a unique identification of this object.
+// UIDs includes all params to make a unique identification of this object. Most
-// Most resources only return one, although some resources can return multiple.
+// resources only return one, although some resources can return multiple.
 func (obj *NoopRes) UIDs() []engine.ResUID {
 	x := &NoopUID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
@@ -126,8 +126,8 @@ func (obj *NoopRes) GroupCmp(r engine.GroupableRes) error {
 	return nil // noop resources can always be grouped together!
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
-// It is primarily useful for setting the defaults.
+// primarily useful for setting the defaults.
 func (obj *NoopRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes NoopRes // indirection to avoid infinite recursion
 

engine/resources/noop_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/resources/nspawn.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -73,8 +73,8 @@ func (obj *NspawnRes) Default() engine.Res {
 	}
 }
 
-// makeComposite creates a pointer to a SvcRes. The pointer is used to
+// makeComposite creates a pointer to a SvcRes. The pointer is used to validate
-// validate and initialize the nested svc.
+// and initialize the nested svc.
 func (obj *NspawnRes) makeComposite() (*SvcRes, error) {
 	res, err := engine.NewNamedResource("svc", fmt.Sprintf(nspawnServiceTmpl, obj.Name()))
 	if err != nil {
@@ -113,7 +113,7 @@ func (obj *NspawnRes) Validate() error {
 		return errwrap.Wrapf(err, "makeComposite failed in validate")
 	}
 	if err := svc.Validate(); err != nil { // composite resource
-		return errwrap.Wrapf(err, "validate failed for embedded svc: %s", obj.svc)
+		return errwrap.Wrapf(err, "validate failed for embedded svc: %s", svc)
 	}
 	return nil
 }
@@ -128,10 +128,7 @@ func (obj *NspawnRes) Init(init *engine.Init) error {
 	}
 	obj.svc = svc
 	// TODO: we could build a new init that adds a prefix to the logger...
-	if err := obj.svc.Init(init); err != nil {
+	return obj.svc.Init(init)
-		return err
-	}
-	return nil
 }
 
 // Close is run by the engine to clean up after the resource is done.
@@ -304,8 +301,8 @@ func (obj *NspawnUID) IFF(uid engine.ResUID) bool {
 	return obj.name == res.name
 }
 
-// UIDs includes all params to make a unique identification of this object.
+// UIDs includes all params to make a unique identification of this object. Most
-// Most resources only return one although some resources can return multiple.
+// resources only return one although some resources can return multiple.
 func (obj *NspawnRes) UIDs() []engine.ResUID {
 	x := &NspawnUID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
@@ -314,8 +311,8 @@ func (obj *NspawnRes) UIDs() []engine.ResUID {
 	return append([]engine.ResUID{x}, obj.svc.UIDs()...)
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
-// It is primarily useful for setting the defaults.
+// primarily useful for setting the defaults.
 func (obj *NspawnRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes NspawnRes // indirection to avoid infinite recursion
 

engine/resources/packagekit/packagekit.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -37,7 +37,8 @@ const (
 	Paranoid = false // enable if you see any ghosts
 )
 
-// constants which might need to be tweaked or which contain special dbus strings.
+// constants which might need to be tweaked or which contain special dbus
+// strings.
 const (
 	// FIXME: if PkBufferSize is too low, install seems to drop signals
 	PkBufferSize = 1000
@@ -71,7 +72,7 @@ var (
 	}
 )
 
-//type enum_filter uint64
+// type enum_filter uint64
 // https://github.com/hughsie/PackageKit/blob/master/lib/packagekit-glib2/pk-enum.c
 const ( //static const PkEnumMatch enum_filter[]
 	PkFilterEnumUnknown        uint64 = 1 << iota // "unknown"
@@ -154,7 +155,8 @@ type Conn struct {
 	Logf  func(format string, v ...interface{})
 }
 
-// PkPackageIDActionData is a struct that is returned by PackagesToPackageIDs in the map values.
+// PkPackageIDActionData is a struct that is returned by PackagesToPackageIDs in
+// the map values.
 type PkPackageIDActionData struct {
 	Found     bool
 	Installed bool
@@ -185,7 +187,8 @@ func (obj *Conn) Close() error {
 	return obj.conn.Close()
 }
 
-// internal helper to add signal matches to the bus, should only be called once
+// matchSignal is an internal helper to add signal matches to the bus. It should
+// only be called once.
 func (obj *Conn) matchSignal(ch chan *dbus.Signal, path dbus.ObjectPath, iface string, signals []string) (func() error, error) {
 	if obj.Debug {
 		obj.Logf("matchSignal(%v, %v, %s, %v)", ch, path, iface, signals)
@@ -565,7 +568,8 @@ loop:
 	return nil
 }
 
-// GetFilesByPackageID gets the list of files that are contained inside a list of packageIDs.
+// GetFilesByPackageID gets the list of files that are contained inside a list
+// of packageIDs.
 func (obj *Conn) GetFilesByPackageID(packageIDs []string) (files map[string][]string, err error) {
 	// NOTE: the maximum number of files in an RPM is 52116 in Fedora 23
 	// https://gist.github.com/purpleidea/b98e60dcd449e1ac3b8a
@@ -634,7 +638,8 @@ loop:
 	return
 }
 
-// GetUpdates gets a list of packages that are installed and which can be updated, mod filter.
+// GetUpdates gets a list of packages that are installed and which can be
+// updated, mod filter.
 func (obj *Conn) GetUpdates(filter uint64) ([]string, error) {
 	if obj.Debug {
 		obj.Logf("GetUpdates()")
@@ -876,7 +881,8 @@ func (obj *Conn) PackagesToPackageIDs(packageMap map[string]string, filter uint6
 	return result, nil
 }
 
-// FilterPackageIDs returns a list of packageIDs which match the set of package names in packages.
+// FilterPackageIDs returns a list of packageIDs which match the set of package
+// names in packages.
 func FilterPackageIDs(m map[string]*PkPackageIDActionData, packages []string) ([]string, error) {
 	result := []string{}
 	for _, k := range packages {
@@ -890,7 +896,8 @@ func FilterPackageIDs(m map[string]*PkPackageIDActionData, packages []string) ([
 	return result, nil
 }
 
-// FilterState returns a map of whether each package queried matches the particular state.
+// FilterState returns a map of whether each package queried matches the
+// particular state.
 func FilterState(m map[string]*PkPackageIDActionData, packages []string, state string) (result map[string]bool, err error) {
 	result = make(map[string]bool)
 	pkgs := []string{} // bad pkgs that don't have a bool state
@@ -920,7 +927,8 @@ func FilterState(m map[string]*PkPackageIDActionData, packages []string, state s
 	return result, err
 }
 
-// FilterPackageState returns all packages that are in package and match the specific state.
+// FilterPackageState returns all packages that are in package and match the
+// specific state.
 func FilterPackageState(m map[string]*PkPackageIDActionData, packages []string, state string) (result []string, err error) {
 	result = []string{}
 	for _, k := range packages {
@@ -946,7 +954,8 @@ func FilterPackageState(m map[string]*PkPackageIDActionData, packages []string,
 	return result, err
 }
 
-// FlagInData asks whether a flag exists inside the data portion of a packageID field?
+// FlagInData asks whether a flag exists inside the data portion of a packageID
+// field?
 func FlagInData(flag, data string) bool {
 	flags := strings.Split(data, ":")
 	for _, f := range flags {

engine/resources/password.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -322,8 +322,8 @@ type PasswordUID struct {
 	name string
 }
 
-// UIDs includes all params to make a unique identification of this object.
+// UIDs includes all params to make a unique identification of this object. Most
-// Most resources only return one, although some resources can return multiple.
+// resources only return one, although some resources can return multiple.
 func (obj *PasswordRes) UIDs() []engine.ResUID {
 	x := &PasswordUID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
@@ -347,8 +347,8 @@ func (obj *PasswordRes) Sends() interface{} {
 	}
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
-// It is primarily useful for setting the defaults.
+// primarily useful for setting the defaults.
 func (obj *PasswordRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes PasswordRes // indirection to avoid infinite recursion
 

engine/resources/pippet.go

@@ -0,0 +1,329 @@
+// Mgmt
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os/exec"
+	"sync"
+
+	"github.com/purpleidea/mgmt/engine"
+	"github.com/purpleidea/mgmt/engine/traits"
+	"github.com/purpleidea/mgmt/util/errwrap"
+)
+
+var pippetReceiverInstance *pippetReceiver
+var pippetReceiverOnce sync.Once
+
+func init() {
+	engine.RegisterResource("pippet", func() engine.Res { return &PippetRes{} })
+}
+
+// PippetRes is a wrapper resource for puppet. It implements the functional
+// equivalent of an exec resource that calls "puppet resource <type> <title>
+// <params>", but offers superior performance through a long-running Puppet
+// process that receives resources through a pipe (hence the name).
+type PippetRes struct {
+	traits.Base // add the base methods without re-implementation
+	traits.Refreshable
+
+	init *engine.Init
+
+	// Type is the exact name of the wrapped Puppet resource type, e.g.
+	// "file", "mount". This needs not be a core type. It can be a type
+	// from a module. The Puppet installation local to the mgmt agent
+	// machine must be able recognize it. It has to be a native type though,
+	// as opposed to defined types from your Puppet manifest code.
+	Type string `yaml:"type" json:"type"`
+	// Title is used by Puppet as the resource title. Puppet will often
+	// assign special meaning to the title, e.g. use it as the path for a
+	// file resource, or the name of a package.
+	Title string `yaml:"title" json:"title"`
+	// Params is expected to be a hash in YAML format, pairing resource
+	// parameter names with their respective values, e.g. { ensure: present
+	// }
+	Params string `yaml:"params" json:"params"`
+
+	runner *pippetReceiver
+}
+
+// Default returns an example Pippet resource.
+func (obj *PippetRes) Default() engine.Res {
+	return &PippetRes{
+		Params: "{}", // use an empty params hash per default
+	}
+}
+
+// Validate never errors out. We don't know the set of potential types that
+// Puppet supports. Resource names are arbitrary. We cannot really validate the
+// parameter YAML, because we cannot assume that it can be unmarshalled into a
+// map[string]string; Puppet supports complex parameter values.
+func (obj *PippetRes) Validate() error {
+	return nil
+}
+
+// Init makes sure that the PippetReceiver object is initialized.
+func (obj *PippetRes) Init(init *engine.Init) error {
+	obj.init = init // save for later
+	obj.runner = getPippetReceiverInstance()
+	return obj.runner.Register()
+}
+
+// Close is run by the engine to clean up after the resource is done.
+func (obj *PippetRes) Close() error {
+	return obj.runner.Unregister()
+}
+
+// Watch is the primary listener for this resource and it outputs events.
+func (obj *PippetRes) Watch() error {
+	obj.init.Running() // when started, notify engine that we're running
+
+	select {
+	case <-obj.init.Done: // closed by the engine to signal shutdown
+	}
+
+	//obj.init.Event() // notify engine of an event (this can block)
+
+	return nil
+}
+
+// CheckApply synchronizes the resource if required.
+func (obj *PippetRes) CheckApply(apply bool) (bool, error) {
+	changed, err := applyPippetRes(obj.runner, obj)
+	if err != nil {
+		return false, fmt.Errorf("pippet: %s[%s]: ERROR - %v", obj.Type, obj.Title, err)
+	}
+	return !changed, nil
+}
+
+// Cmp compares two resources and returns an error if they are not equivalent.
+func (obj *PippetRes) Cmp(r engine.Res) error {
+	res, ok := r.(*PippetRes)
+	if !ok {
+		return fmt.Errorf("not a %s", obj.Kind())
+	}
+
+	if obj.Type != res.Type {
+		return fmt.Errorf("the Type param differs")
+	}
+
+	if obj.Title != res.Title {
+		return fmt.Errorf("the Title param differs")
+	}
+
+	// FIXME: This is a lie. Parameter lists can be equivalent but not
+	// lexically identical (e.g. whitespace differences, parameter order).
+	// This is difficult to handle because we cannot casually unmarshall the
+	// YAML content.
+	if obj.Params != res.Params {
+		return fmt.Errorf("the Param param differs")
+	}
+
+	return nil
+}
+
+// PippetUID is the UID struct for PippetRes.
+type PippetUID struct {
+	engine.BaseUID
+	resourceType  string
+	resourceTitle string
+}
+
+// UIDs includes all params to make a unique identification of this object. Most
+// resources only return one, although some resources can return multiple.
+func (obj *PippetRes) UIDs() []engine.ResUID {
+	x := &PippetUID{
+		BaseUID:       engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
+		resourceType:  obj.Type,
+		resourceTitle: obj.Title,
+	}
+	return []engine.ResUID{x}
+}
+
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
+// primarily useful for setting the defaults.
+func (obj *PippetRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type rawRes PippetRes // indirection to avoid infinite recursion
+
+	def := obj.Default()        // get the default
+	res, ok := def.(*PippetRes) // put in the right format
+	if !ok {
+		return fmt.Errorf("could not convert to PippetRes")
+	}
+	raw := rawRes(*res) // convert; the defaults go here
+
+	if err := unmarshal(&raw); err != nil {
+		return err
+	}
+
+	*obj = PippetRes(raw) // restore from indirection with type conversion!
+	return nil
+}
+
+// PippetRunner is the interface used to communicate with the PippetReceiver
+// object. Its main purpose is dependency injection.
+type PippetRunner interface {
+	LockApply()
+	UnlockApply()
+	InputStream() io.WriteCloser
+	OutputStream() io.ReadCloser
+}
+
+// PippetResult is the structured return value type for the PippetReceiver's
+// Apply function.
+type PippetResult struct {
+	Error     bool
+	Failed    bool
+	Changed   bool
+	Exception string
+}
+
+// GetPippetReceiverInstance returns a pointer to the PippetReceiver object. The
+// PippetReceiver is supposed to be a singleton object. The pippet resource code
+// should always use the PippetReceiverInstance function to gain access to the
+// pippetReceiver object. Other objects of type pippetReceiver should not be
+// created.
+func getPippetReceiverInstance() *pippetReceiver {
+	for pippetReceiverInstance == nil {
+		pippetReceiverOnce.Do(func() { pippetReceiverInstance = &pippetReceiver{} })
+	}
+	return pippetReceiverInstance
+}
+
+type pippetReceiver struct {
+	stdin         io.WriteCloser
+	stdout        io.ReadCloser
+	registerMutex sync.Mutex
+	applyMutex    sync.Mutex
+	registered    int
+}
+
+// Init runs the Puppet process that will perform the work of synchronizing
+// resources that are sent to its stdin. The process will keep running until
+// Close is called. Init should not be called directly. It is implicitly called
+// by the Register function.
+func (obj *pippetReceiver) Init() error {
+	cmd := exec.Command("puppet", "yamlresource", "receive", "--color=no")
+	var err error
+	obj.stdin, err = cmd.StdinPipe()
+	if err != nil {
+		return err
+	}
+	obj.stdout, err = cmd.StdoutPipe()
+	if err != nil {
+		return errwrap.Append(err, obj.stdin.Close())
+	}
+	if err = cmd.Start(); err != nil {
+		return errwrap.Append(err, obj.stdin.Close())
+	}
+	buf := make([]byte, 80)
+	if _, err = obj.stdout.Read(buf); err != nil {
+		return errwrap.Append(err, obj.stdin.Close())
+	}
+	return nil
+}
+
+// Register should be called by any user (i.e., any pippet resource) before
+// using the PippetRunner functions on this receiver object. Register implicitly
+// takes care of calling Init if required.
+func (obj *pippetReceiver) Register() error {
+	obj.registerMutex.Lock()
+	defer obj.registerMutex.Unlock()
+	obj.registered = obj.registered + 1
+	if obj.registered > 1 {
+		return nil
+	}
+	// count was increased from 0 to 1, we need to (re-)init
+	var err error
+	if err = obj.Init(); err != nil {
+		obj.registered = 0
+	}
+	return err
+}
+
+// Unregister should be called by any object that registered itself using the
+// Register function, and which no longer needs the receiver. This should
+// typically happen at closing time of the pippet resource that registered
+// itself. Unregister implicitly calls Close in case all registered resources
+// have unregistered.
+func (obj *pippetReceiver) Unregister() error {
+	obj.registerMutex.Lock()
+	defer obj.registerMutex.Unlock()
+	obj.registered = obj.registered - 1
+	if obj.registered == 0 {
+		return obj.Close()
+	}
+	if obj.registered < 0 {
+		return fmt.Errorf("pippet runner: ERROR: unregistered more resources than were registered")
+	}
+	return nil
+}
+
+// LockApply locks the pippetReceiver's mutex for an "Apply" transaction.
+func (obj *pippetReceiver) LockApply() {
+	obj.applyMutex.Lock()
+}
+
+// UnlockApply unlocks the pippetReceiver's mutex for an "Apply" transaction.
+func (obj *pippetReceiver) UnlockApply() {
+	obj.applyMutex.Unlock()
+}
+
+// InputStream returns the pippetReceiver's pipe writer.
+func (obj *pippetReceiver) InputStream() io.WriteCloser {
+	return obj.stdin
+}
+
+// OutputStream returns the pippetReceiver's pipe reader.
+func (obj *pippetReceiver) OutputStream() io.ReadCloser {
+	return obj.stdout
+}
+
+// Close stops the backend puppet process by closing its stdin handle. It should
+// not be called directly. It is implicitly called by the Unregister function if
+// appropriate.
+func (obj *pippetReceiver) Close() error {
+	return obj.stdin.Close()
+}
+
+// applyPippetRes does the actual work of making Puppet synchronize a resource.
+func applyPippetRes(runner PippetRunner, resource *PippetRes) (bool, error) {
+	runner.LockApply()
+	defer runner.UnlockApply()
+	if err := json.NewEncoder(runner.InputStream()).Encode(resource); err != nil {
+		return false, errwrap.Wrapf(err, "failed to send resource to puppet")
+	}
+
+	result := PippetResult{
+		Error:     true,
+		Exception: "missing output fields",
+	}
+	if err := json.NewDecoder(runner.OutputStream()).Decode(&result); err != nil {
+		return false, errwrap.Wrapf(err, "failed to read response from puppet")
+	}
+
+	if result.Error {
+		return false, fmt.Errorf("puppet did not sync: %s", result.Exception)
+	}
+	if result.Failed {
+		return false, fmt.Errorf("puppet failed to sync")
+	}
+	return result.Changed, nil
+}

engine/resources/pippet_test.go

@@ -0,0 +1,136 @@
+// Mgmt
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// +build !root
+
+package resources
+
+import (
+	"io"
+	"testing"
+)
+
+type nullWriteCloser struct {
+}
+
+type fakePippetReceiver struct {
+	stdin  nullWriteCloser
+	stdout *io.PipeReader
+	Locked bool
+}
+
+func (obj nullWriteCloser) Write(data []byte) (int, error) {
+	return len(data), nil
+}
+
+func (obj nullWriteCloser) Close() error {
+	return nil
+}
+
+func (obj *fakePippetReceiver) LockApply() {
+	obj.Locked = true
+}
+
+func (obj *fakePippetReceiver) UnlockApply() {
+	obj.Locked = false
+}
+
+func (obj *fakePippetReceiver) InputStream() io.WriteCloser {
+	return obj.stdin
+}
+
+func (obj *fakePippetReceiver) OutputStream() io.ReadCloser {
+	return obj.stdout
+}
+
+func newFakePippetReceiver(jsonTestOutput string) *fakePippetReceiver {
+	output, input := io.Pipe()
+
+	result := &fakePippetReceiver{
+		stdout: output,
+	}
+
+	go func() {
+		// this will appear on the fake stdout
+		input.Write([]byte(jsonTestOutput))
+	}()
+
+	return result
+}
+
+var pippetTestRes = &PippetRes{
+	Type:   "notify",
+	Title:  "testmessage",
+	Params: `{msg: "This is a test"}`,
+}
+
+func TestNormalPuppetOutput(t *testing.T) {
+	r := newFakePippetReceiver(`{"resource":"Notify[test]","failed":false,"changed":true,"noop":false,"error":false,"exception":null}`)
+	changed, err := applyPippetRes(r, pippetTestRes)
+
+	if err != nil {
+		t.Errorf("normal Puppet output led to an apply error: %v", err)
+	}
+
+	if !changed {
+		t.Errorf("return values of applyPippetRes did not reflect the changed state")
+	}
+}
+
+func TestUnchangedPuppetOutput(t *testing.T) {
+	r := newFakePippetReceiver(`{"resource":"Notify[test]","failed":false,"changed":false,"noop":false,"error":false,"exception":null}`)
+	changed, err := applyPippetRes(r, pippetTestRes)
+
+	if err != nil {
+		t.Errorf("normal Puppet output led to an apply error: %v", err)
+	}
+
+	if changed {
+		t.Errorf("return values of applyPippetRes did not reflect the changed state")
+	}
+}
+
+func TestFailingPuppetOutput(t *testing.T) {
+	r := newFakePippetReceiver(`{"resource":"Notify[test]","failed":false,"changed":false,"noop":false,"error":true,"exception":"I failed!"}`)
+	_, err := applyPippetRes(r, pippetTestRes)
+
+	if err == nil {
+		t.Errorf("failing Puppet output led to an apply error: %v", err)
+	}
+}
+
+func TestEmptyPuppetOutput(t *testing.T) {
+	t.Skip("empty output will currently make the application (and the test) hang")
+}
+
+func TestPartialPuppetOutput(t *testing.T) {
+	r := newFakePippetReceiver(`{"resource":"Notify[test]","failed":false,"changed":true}`)
+	_, err := applyPippetRes(r, pippetTestRes)
+
+	if err == nil {
+		t.Errorf("partial Puppet output did not lead to an apply error")
+	}
+}
+
+func TestMalformedPuppetOutput(t *testing.T) {
+	r := newFakePippetReceiver(`oops something went wrong!!1!eleven`)
+	_, err := applyPippetRes(r, pippetTestRes)
+
+	if err == nil {
+		t.Errorf("malformed Puppet output did not lead to an apply error")
+	}
+}

engine/resources/pkg.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -100,8 +100,8 @@ func (obj *PkgRes) Close() error {
 	return nil
 }
 
-// Watch is the primary listener for this resource and it outputs events.
+// Watch is the primary listener for this resource and it outputs events. It
-// It uses the PackageKit UpdatesChanged signal to watch for changes.
+// uses the PackageKit UpdatesChanged signal to watch for changes.
 // TODO: https://github.com/hughsie/PackageKit/issues/109
 // TODO: https://github.com/hughsie/PackageKit/issues/110
 func (obj *PkgRes) Watch() error {
@@ -504,7 +504,8 @@ func (obj *PkgResAutoEdges) Next() []engine.ResUID {
 	return result
 }
 
-// Test gets results of the earlier Next() call, & returns if we should continue!
+// Test gets results of the earlier Next() call, & returns if we should
+// continue!
 func (obj *PkgResAutoEdges) Test(input []bool) bool {
 	if !obj.testIsNext {
 		panic("expecting a call to Next()")
@@ -591,8 +592,8 @@ func (obj *PkgRes) AutoEdges() (engine.AutoEdge, error) {
 	}, nil
 }
 
-// UIDs includes all params to make a unique identification of this object.
+// UIDs includes all params to make a unique identification of this object. Most
-// Most resources only return one, although some resources can return multiple.
+// resources only return one, although some resources can return multiple.
 func (obj *PkgRes) UIDs() []engine.ResUID {
 	x := &PkgUID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
@@ -611,9 +612,9 @@ func (obj *PkgRes) UIDs() []engine.ResUID {
 	return result
 }
 
-// GroupCmp returns whether two resources can be grouped together or not.
+// GroupCmp returns whether two resources can be grouped together or not. Can
-// Can these two resources be merged, aka, does this resource support doing so?
+// these two resources be merged, aka, does this resource support doing so? Will
-// Will resource allow itself to be grouped _into_ this obj?
+// resource allow itself to be grouped _into_ this obj?
 func (obj *PkgRes) GroupCmp(r engine.GroupableRes) error {
 	res, ok := r.(*PkgRes)
 	if !ok {
@@ -631,8 +632,8 @@ func (obj *PkgRes) GroupCmp(r engine.GroupableRes) error {
 	return nil
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
-// It is primarily useful for setting the defaults.
+// primarily useful for setting the defaults.
 func (obj *PkgRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes PkgRes // indirection to avoid infinite recursion
 
@@ -651,7 +652,8 @@ func (obj *PkgRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	return nil
 }
 
-// ReturnSvcInFileList returns a list of svc names for matches like: `/usr/lib/systemd/system/*.service`.
+// ReturnSvcInFileList returns a list of svc names for matches like:
+// `/usr/lib/systemd/system/*.service`.
 func ReturnSvcInFileList(fileList []string) []string {
 	result := []string{}
 	for _, x := range fileList {

engine/resources/pkg_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/resources/print.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -133,8 +133,8 @@ type PrintUID struct {
 	name string
 }
 
-// UIDs includes all params to make a unique identification of this object.
+// UIDs includes all params to make a unique identification of this object. Most
-// Most resources only return one, although some resources can return multiple.
+// resources only return one, although some resources can return multiple.
 func (obj *PrintRes) UIDs() []engine.ResUID {
 	x := &PrintUID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
@@ -156,8 +156,8 @@ func (obj *PrintRes) GroupCmp(r engine.GroupableRes) error {
 	return nil // grouped together if we were asked to
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
-// It is primarily useful for setting the defaults.
+// primarily useful for setting the defaults.
 func (obj *PrintRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes PrintRes // indirection to avoid infinite recursion
 

engine/resources/resources_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -23,12 +23,17 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"os/user"
+	"path"
+	"strconv"
 	"strings"
 	"sync"
+	"syscall"
 	"testing"
 	"time"
 
 	"github.com/purpleidea/mgmt/engine"
+	"github.com/purpleidea/mgmt/pgraph"
 	"github.com/purpleidea/mgmt/util"
 	"github.com/purpleidea/mgmt/util/errwrap"
 )
@@ -116,7 +121,8 @@ func (obj *changedStep) Action() error {
 }
 func (obj *changedStep) Expect() error { return nil }
 
-// NewChangedStep waits up to this many ms for a CheckApply action to occur. Watch function to startup.
+// NewChangedStep waits up to this many ms for a CheckApply action to occur.
+// Watch function to startup.
 func NewChangedStep(ms uint, expect bool) Step {
 	return &changedStep{
 		ms:     ms,
@@ -171,7 +177,29 @@ func FileExpect(p, s string) Step { // path & string
 	}
 }
 
-// FileExpect takes a path and a string to write to that file, and builds a Step
+// FileOwnerExpect takes a path and a uid to expect from that file, and builds a
+// Step that checks that out of them.
+func FileOwnerExpect(p, o string) Step { // path & owner
+	return &manualStep{
+		action: func() error { return nil },
+		expect: func() error {
+			var stat syscall.Stat_t
+			if err := syscall.Stat(p, &stat); err != nil {
+				return err
+			}
+			i, err := strconv.ParseUint(o, 10, 32)
+			if err != nil {
+				return err
+			}
+			if i != uint64(stat.Uid) {
+				return fmt.Errorf("file uid did not match in %s", p)
+			}
+			return nil
+		},
+	}
+}
+
+// FileWrite takes a path and a string to write to that file, and builds a Step
 // that does that to them.
 func FileWrite(p, s string) Step { // path & string
 	return &manualStep{
@@ -192,6 +220,15 @@ func ErrIsNotExistOK(e error) error {
 	return errwrap.Wrapf(e, "unexpected error")
 }
 
+// GetUID returns the UID of the user running this test.
+func GetUID() (string, error) {
+	u, err := user.Lookup(os.Getenv("USER"))
+	if err != nil {
+		return "", err
+	}
+	return u.Uid, nil
+}
+
 func TestResources1(t *testing.T) {
 	type test struct { // an individual test
 		name      string
@@ -225,7 +262,7 @@ func TestResources1(t *testing.T) {
 		p := "/tmp/whatever"
 		s := "hello, world\n"
 		res.Path = p
-		res.State = "exists"
+		res.State = FileStateExists
 		contents := s
 		res.Content = &contents
 
@@ -284,12 +321,48 @@ func TestResources1(t *testing.T) {
 			cleanup: func() error { return os.Remove(f) },
 		})
 	}
+	{
+		r := makeRes("exec", "x2")
+		res := r.(*ExecRes) // if this panics, the test will panic
+		res.Env = map[string]string{
+			"boiling": "one hundred",
+		}
+		f := "/tmp/whatever"
+		res.Cmd = fmt.Sprintf("env | grep boiling > %s", f)
+		res.Shell = "/bin/bash"
+		res.IfCmd = "! diff <(cat /tmp/whatever) <(echo boiling=one hundred)"
+		res.IfShell = "/bin/bash"
+		res.WatchCmd = fmt.Sprintf("/usr/bin/inotifywait -e modify -m %s", f)
+		res.WatchShell = "/bin/bash"
+
+		timeline := []Step{
+			NewStartupStep(1000 * 60),              // startup
+			NewChangedStep(1000*60, false),         // did we do something?
+			FileExpect(f, "boiling=one hundred\n"), // check initial state
+			NewClearChangedStep(1000 * 15),         // did we do something?
+			FileWrite(f, "this is stuff!\n"),       // change state
+			NewChangedStep(1000*60, false),         // did we do something?
+			FileExpect(f, "boiling=one hundred\n"), // check again
+			sleep(1),                               // we can sleep too!
+		}
+
+		testCases = append(testCases, test{
+			name:     "exec with env",
+			res:      res,
+			fail:     false,
+			timeline: timeline,
+			expect:   func() error { return nil },
+			// build file for inotifywait
+			startup: func() error { return ioutil.WriteFile(f, []byte("starting...\n"), 0666) },
+			cleanup: func() error { return os.Remove(f) },
+		})
+	}
 	{
 		r := makeRes("file", "r1")
 		res := r.(*FileRes) // if this panics, the test will panic
 		p := "/tmp/emptyfile"
 		res.Path = p
-		res.State = "exists"
+		res.State = FileStateExists
 
 		timeline := []Step{
 			NewStartupStep(1000 * 60),      // startup
@@ -313,7 +386,7 @@ func TestResources1(t *testing.T) {
 		res := r.(*FileRes) // if this panics, the test will panic
 		p := "/tmp/existingfile"
 		res.Path = p
-		res.State = "exists"
+		res.State = FileStateExists
 		content := "some existing text\n"
 
 		timeline := []Step{
@@ -332,6 +405,33 @@ func TestResources1(t *testing.T) {
 			cleanup:  func() error { return os.Remove(p) },
 		})
 	}
+	{
+		r := makeRes("file", "r1")
+		res := r.(*FileRes) // if this panics, the test will panic
+		p := "/tmp/ownerfile"
+		uid, _ := GetUID()
+		res.Path = p
+		res.State = FileStateExists
+		res.Owner = uid
+		content := "some test file owned by uid " + uid
+
+		timeline := []Step{
+			NewStartupStep(1000 * 60),     // startup
+			NewChangedStep(1000*60, true), // did we do something?
+			FileExpect(p, content),        // check file content
+			FileOwnerExpect(p, uid),       // check uid of the file
+		}
+
+		testCases = append(testCases, test{
+			name:     "uid test file",
+			res:      res,
+			fail:     false,
+			timeline: timeline,
+			expect:   func() error { return nil },
+			startup:  func() error { return ioutil.WriteFile(p, []byte(content), 0666) },
+			cleanup:  func() error { return os.Remove(p) },
+		})
+	}
 
 	names := []string{}
 	for index, tc := range testCases { // run all the tests
@@ -420,15 +520,20 @@ func TestResources1(t *testing.T) {
 				},
 			}
 
-			t.Logf("test #%d: running startup()", index)
+			if startup != nil {
-			if err := startup(); err != nil {
+				t.Logf("test #%d: running startup()", index)
-				t.Errorf("test #%d: FAIL", index)
+				if err := startup(); err != nil {
-				t.Errorf("test #%d: could not startup: %+v", index, err)
+					t.Errorf("test #%d: FAIL", index)
+					t.Errorf("test #%d: could not startup: %+v", index, err)
+				}
 			}
 			// run init
 			t.Logf("test #%d: running Init", index)
 			err = res.Init(init)
 			defer func() {
+				if cleanup == nil {
+					return
+				}
 				t.Logf("test #%d: running cleanup()", index)
 				if err := cleanup(); err != nil {
 					t.Errorf("test #%d: FAIL", index)
@@ -580,6 +685,29 @@ func TestResources2(t *testing.T) {
 		cleanup  func() error   // function to run as cleanup
 	}
 
+	type initOptions struct {
+		// graph is the graph that should be passed in with Init
+		graph *pgraph.Graph
+		// TODO: add more options if needed
+
+		// logf specifies the log function for Init to pass through...
+		logf func(format string, v ...interface{})
+	}
+
+	type initOption func(*initOptions)
+
+	addGraph := func(graph *pgraph.Graph) initOption {
+		return func(io *initOptions) {
+			io.graph = graph
+		}
+	}
+
+	addLogf := func(logf func(format string, v ...interface{})) initOption {
+		return func(io *initOptions) {
+			io.logf = logf
+		}
+	}
+
 	// resValidate runs Validate on the res.
 	resValidate := func(res engine.Res) func() error {
 		// run Close
@@ -588,9 +716,18 @@ func TestResources2(t *testing.T) {
 		}
 	}
 	// resInit runs Init on the res.
-	resInit := func(res engine.Res) func() error {
+	resInit := func(res engine.Res, opts ...initOption) func() error {
+
+		io := &initOptions{}              // defaults
+		for _, optionFunc := range opts { // apply the options
+			optionFunc(io)
+		}
+
 		logf := func(format string, v ...interface{}) {
-			// noop for now
+			if io.logf == nil {
+				return
+			}
+			io.logf(fmt.Sprintf("test: ")+format+"\n", v...)
 		}
 		init := &engine.Init{
 			//Debug: debug,
@@ -603,6 +740,22 @@ func TestResources2(t *testing.T) {
 			Recv: func() map[string]*engine.Send {
 				return map[string]*engine.Send{}
 			},
+
+			// Copied from state.go
+			FilteredGraph: func() (*pgraph.Graph, error) {
+				//graph, err := pgraph.NewGraph("filtered")
+				//if err != nil {
+				//	return nil, errwrap.Wrapf(err, "could not create graph")
+				//}
+				// Hack: We just add ourself as allowed since
+				// we're just a one-vertex test suite...
+				//graph.AddVertex(res) // hack!
+				//return graph, nil // we return in a func so it's fresh!
+				if io.graph == nil {
+					return nil, fmt.Errorf("use addGraph to add one here")
+				}
+				return io.graph, nil
+			},
 		}
 		// run Init
 		return func() error {
@@ -621,7 +774,7 @@ func TestResources2(t *testing.T) {
 				return errwrap.Wrapf(e, "error from CheckApply did not match expected")
 			}
 			if checkOK != expCheckOK {
-				return fmt.Errorf("result from CheckApply did not match expected: `%t` != `%t`", checkOK, expCheckOK)
+				return fmt.Errorf("result from CheckApply did not match expected: got: %t exp: %t", checkOK, expCheckOK)
 			}
 			return nil
 		}
@@ -702,10 +855,29 @@ func TestResources2(t *testing.T) {
 			return nil
 		}
 	}
+	fileExists := func(p string, dir bool) func() error {
+		// does the file exist?
+		return func() error {
+			fi, err := os.Stat(p)
+			if err != nil {
+				return fmt.Errorf("file was supposed to be present, got: %+v", err)
+			}
+			if fi.IsDir() != dir {
+				if dir {
+					return fmt.Errorf("not a dir")
+				}
+				return fmt.Errorf("not a regular file")
+			}
+			return nil
+		}
+	}
 	fileAbsent := func(p string) func() error {
 		// does the file exist?
 		return func() error {
 			_, err := os.Stat(p)
+			if err == nil {
+				return fmt.Errorf("file exists, expecting absent")
+			}
 			if !os.IsNotExist(err) {
 				return fmt.Errorf("file was supposed to be absent, got: %+v", err)
 			}
@@ -723,18 +895,27 @@ func TestResources2(t *testing.T) {
 			return nil
 		}
 	}
+	fileMkdir := func(p string, all bool) func() error {
+		// mkdir at the path
+		return func() error {
+			if all {
+				return os.MkdirAll(p, 0777)
+			}
+			return os.Mkdir(p, 0777)
+		}
+	}
 
 	testCases := []test{}
 	{
 		//file "/tmp/somefile" {
-		//	state => "exists",
+		//	state => $const.res.file.state.exists,
 		//	content => "some new text\n",
 		//}
 		r1 := makeRes("file", "r1")
 		res := r1.(*FileRes) // if this panics, the test will panic
 		p := "/tmp/somefile"
 		res.Path = p
-		res.State = "exists"
+		res.State = FileStateExists
 		content := "some new text\n"
 		res.Content = &content
 
@@ -766,7 +947,7 @@ func TestResources2(t *testing.T) {
 		res := r1.(*FileRes) // if this panics, the test will panic
 		p := "/tmp/somefile"
 		res.Path = p
-		//res.State = "exists" // not specified!
+		//res.State = FileStateExists // not specified!
 		content := "some new text\n"
 		res.Content = &content
 
@@ -799,7 +980,7 @@ func TestResources2(t *testing.T) {
 		res := r1.(*FileRes) // if this panics, the test will panic
 		p := "/tmp/somefile"
 		res.Path = p
-		//res.State = "exists" // not specified!
+		//res.State = FileStateExists // not specified!
 		content := "some new text\n"
 		res.Content = &content
 
@@ -823,14 +1004,14 @@ func TestResources2(t *testing.T) {
 	}
 	{
 		//file "/tmp/somefile" {
-		//	state => "absent",
+		//	state => $const.res.file.state.absent,
 		//}
 		// and no existing file exists!
 		r1 := makeRes("file", "r1")
 		res := r1.(*FileRes) // if this panics, the test will panic
 		p := "/tmp/somefile"
 		res.Path = p
-		res.State = "absent"
+		res.State = FileStateAbsent
 
 		timeline := []func() error{
 			fileRemove(p), // nothing here
@@ -852,14 +1033,14 @@ func TestResources2(t *testing.T) {
 	}
 	{
 		//file "/tmp/somefile" {
-		//	state => "absent",
+		//	state => $const.res.file.state.absent,
 		//}
 		// and a file already exists!
 		r1 := makeRes("file", "r1")
 		res := r1.(*FileRes) // if this panics, the test will panic
 		p := "/tmp/somefile"
 		res.Path = p
-		res.State = "absent"
+		res.State = FileStateAbsent
 
 		timeline := []func() error{
 			fileWrite(p, "whatever"),
@@ -882,7 +1063,7 @@ func TestResources2(t *testing.T) {
 	{
 		//file "/tmp/somefile" {
 		//	content => "some new text\n",
-		//	state => "exists",
+		//	state => $const.res.file.state.exists,
 		//
 		//	Meta:reverse => true,
 		//}
@@ -890,7 +1071,7 @@ func TestResources2(t *testing.T) {
 		res := r1.(*FileRes) // if this panics, the test will panic
 		p := "/tmp/somefile"
 		res.Path = p
-		res.State = "exists"
+		res.State = FileStateExists
 		content := "some new text\n"
 		res.Content = &content
 		original := "this is the original state\n" // original state
@@ -951,7 +1132,7 @@ func TestResources2(t *testing.T) {
 		res := r1.(*FileRes) // if this panics, the test will panic
 		p := "/tmp/somefile"
 		res.Path = p
-		//res.State = "exists" // unspecified
+		//res.State = FileStateExists // unspecified
 		content := "some new text\n"
 		res.Content = &content
 		original := "this is the original state\n" // original state
@@ -1016,7 +1197,7 @@ func TestResources2(t *testing.T) {
 		res := r1.(*FileRes) // if this panics, the test will panic
 		p := "/tmp/somefile"
 		res.Path = p
-		//res.State = "exists" // unspecified
+		//res.State = FileStateExists // unspecified
 		content := "some new text\n"
 		res.Content = &content
 		var r2 engine.Res // future reversed resource
@@ -1065,7 +1246,7 @@ func TestResources2(t *testing.T) {
 	}
 	{
 		//file "/tmp/somefile" {
-		//	state => "absent",
+		//	state => $const.res.file.state.absent,
 		//
 		//	Meta:reverse => true,
 		//}
@@ -1073,7 +1254,7 @@ func TestResources2(t *testing.T) {
 		res := r1.(*FileRes) // if this panics, the test will panic
 		p := "/tmp/somefile"
 		res.Path = p
-		res.State = "absent"
+		res.State = FileStateAbsent
 		original := "this is the original state\n" // original state
 		var r2 engine.Res                          // future reversed resource
 
@@ -1121,7 +1302,388 @@ func TestResources2(t *testing.T) {
 			cleanup:  func() error { return nil },
 		})
 	}
+	{
+		//file "/tmp/somefile" {
+		//	state => $const.res.file.state.exists,
+		//	fragments => [
+		//		"/tmp/frag1",
+		//		"/tmp/fragdir1/",
+		//		"/tmp/frag2",
+		//		"/tmp/fragdir2/",
+		//		"/tmp/frag3",
+		//	],
+		//}
+		r1 := makeRes("file", "r1")
+		res := r1.(*FileRes) // if this panics, the test will panic
+		p := "/tmp/somefile"
+		res.Path = p
+		res.State = FileStateExists
+		res.Fragments = []string{
+			"/tmp/frag1",
+			"/tmp/fragdir1/",
+			"/tmp/frag2",
+			"/tmp/fragdir2/",
+			"/tmp/frag3",
+		}
 
+		frag1 := "frag1\n"
+		f1 := "f1\n"
+		f2 := "f2\n"
+		f3 := "f3\n"
+		frag2 := "frag2\n"
+		f1d2 := "f1 from fragdir2\n"
+		f2d2 := "f2 from fragdir2\n"
+		f3d2 := "f3 from fragdir2\n"
+		frag3 := "frag3\n"
+		content := frag1 + f1 + f2 + f3 + frag2 + f1d2 + f2d2 + f3d2 + frag3
+
+		timeline := []func() error{
+			fileWrite("/tmp/frag1", frag1),
+			fileWrite("/tmp/frag2", frag2),
+			fileWrite("/tmp/frag3", frag3),
+			fileMkdir("/tmp/fragdir1/", true),
+			fileWrite("/tmp/fragdir1/f1", f1),
+			fileWrite("/tmp/fragdir1/f2", f2),
+			fileWrite("/tmp/fragdir1/f3", f3),
+			fileMkdir("/tmp/fragdir2/", true),
+			fileWrite("/tmp/fragdir2/f1", f1d2),
+			fileWrite("/tmp/fragdir2/f2", f2d2),
+			fileWrite("/tmp/fragdir2/f3", f3d2),
+			fileWrite(p, "whatever"),
+			resValidate(r1),
+			resInit(r1),
+			resCheckApply(r1, false), // changed
+			fileExpect(p, content),
+			resCheckApply(r1, true), // it's already good
+			resClose(r1),
+			fileExpect(p, content), // ensure it exists
+		}
+
+		testCases = append(testCases, test{
+			name:     "file fragments",
+			timeline: timeline,
+			expect:   func() error { return nil },
+			startup:  func() error { return nil },
+			cleanup:  func() error { return os.Remove(p) },
+		})
+	}
+	{
+		//file "/tmp/somefile" {
+		//	state => $const.res.file.state.exists,
+		//	source => "/tmp/somefiletocopy",
+		//}
+		r1 := makeRes("file", "r1")
+		res := r1.(*FileRes) // if this panics, the test will panic
+		p := "/tmp/somefile"
+		p2 := "/tmp/somefiletocopy"
+		content := "hello this is some file to copy\n"
+		res.Path = p
+		res.State = FileStateExists
+		res.Source = p2
+
+		timeline := []func() error{
+			fileAbsent(p), // ensure it's absent
+			fileWrite(p2, content),
+			resValidate(r1),
+			resInit(r1),
+			resCheckApply(r1, false), // changed
+			fileExpect(p, content),   // should be created like this
+			fileExpect(p2, content),  // should not change
+			resCheckApply(r1, true),  // it's already good
+			fileExpect(p, content),   // should already be like this
+			fileExpect(p2, content),  // should not change either
+			resClose(r1),
+		}
+
+		testCases = append(testCases, test{
+			name:     "copy file with source",
+			timeline: timeline,
+			expect:   func() error { return nil },
+			startup:  func() error { return nil },
+			cleanup:  func() error { return os.Remove(p) },
+		})
+	}
+	{
+		//file "/tmp/somedir/" {
+		//	state => $const.res.file.state.exists,
+		//}
+		r1 := makeRes("file", "r1")
+		res := r1.(*FileRes) // if this panics, the test will panic
+		p := "/tmp/somedir/"
+		res.Path = p
+		res.State = FileStateExists
+
+		timeline := []func() error{
+			fileAbsent(p), // ensure it's absent
+			resValidate(r1),
+			resInit(r1),
+			resCheckApply(r1, false), // changed
+			fileExists(p, true),      // ensure it's a dir
+			resCheckApply(r1, true),  // it's already good
+			fileExists(p, true),      // ensure it's a dir
+			resClose(r1),
+		}
+
+		testCases = append(testCases, test{
+			name:     "make empty directory",
+			timeline: timeline,
+			expect:   func() error { return nil },
+			startup:  func() error { return nil },
+			cleanup:  func() error { return os.RemoveAll(p) },
+		})
+	}
+	{
+		//file "/tmp/somedir/" {
+		//	state => $const.res.file.state.exists,
+		//	source => /tmp/somedirtocopy/,
+		//	recurse => true,
+		//}
+		r1 := makeRes("file", "r1")
+		res := r1.(*FileRes) // if this panics, the test will panic
+		p := "/tmp/somedir/"
+		p2 := "/tmp/somedirtocopy/"
+		res.Path = p
+		res.State = FileStateExists
+		res.Source = p2
+		res.Recurse = true
+
+		f1 := path.Join(p, "f1")
+		f2 := path.Join(p, "f2")
+		d1 := path.Join(p, "d1/")
+		d2 := path.Join(p, "d2/")
+		d1f1 := path.Join(p, "d1/f1")
+		d1f2 := path.Join(p, "d1/f2")
+		d2f1 := path.Join(p, "d2/f1")
+		d2f2 := path.Join(p, "d2/f2")
+		d2f3 := path.Join(p, "d2/f3")
+
+		xf1 := path.Join(p2, "f1")
+		xf2 := path.Join(p2, "f2")
+		xd1 := path.Join(p2, "d1/")
+		xd2 := path.Join(p2, "d2/")
+		xd1f1 := path.Join(p2, "d1/f1")
+		xd1f2 := path.Join(p2, "d1/f2")
+		xd2f1 := path.Join(p2, "d2/f1")
+		xd2f2 := path.Join(p2, "d2/f2")
+		xd2f3 := path.Join(p2, "d2/f3")
+
+		timeline := []func() error{
+			fileMkdir(p2, true),
+			fileWrite(xf1, "f1\n"),
+			fileWrite(xf2, "f2\n"),
+			fileMkdir(xd1, true),
+			fileMkdir(xd2, true),
+			fileWrite(xd1f1, "d1f1\n"),
+			fileWrite(xd1f2, "d1f2\n"),
+			fileWrite(xd2f1, "d2f1\n"),
+			fileWrite(xd2f2, "d2f2\n"),
+			fileWrite(xd2f3, "d2f3\n"),
+			resValidate(r1),
+			resInit(r1),
+			resCheckApply(r1, false), // changed
+			fileExists(p, true),      // ensure it's a dir
+			fileExists(f1, false),    // ensure it's a file
+			fileExists(f2, false),
+			fileExists(d1, true), // ensure it's a dir
+			fileExists(d2, true),
+			fileExists(d1f1, false),
+			fileExists(d1f2, false),
+			fileExists(d2f1, false),
+			fileExists(d2f2, false),
+			fileExists(d2f3, false),
+			resCheckApply(r1, true), // it's already good
+			resClose(r1),
+		}
+
+		testCases = append(testCases, test{
+			name:     "source dir copy",
+			timeline: timeline,
+			expect:   func() error { return nil },
+			startup:  func() error { return nil },
+			cleanup:  func() error { return os.RemoveAll(p) },
+		})
+	}
+	{
+		//file "/tmp/somedir/" {
+		//	state => $const.res.file.state.exists,
+		//	recurse => true,
+		//	purge => true,
+		//}
+		r1 := makeRes("file", "r1")
+		res := r1.(*FileRes) // if this panics, the test will panic
+		p := "/tmp/somedir/"
+		res.Path = p
+		res.State = FileStateExists
+		res.Recurse = true
+		res.Purge = true
+
+		f1 := path.Join(p, "f1")
+		f2 := path.Join(p, "f2")
+		d1 := path.Join(p, "d1/")
+		d2 := path.Join(p, "d2/")
+		d1f1 := path.Join(p, "d1/f1")
+		d1f2 := path.Join(p, "d1/f2")
+		d2f1 := path.Join(p, "d2/f1")
+		d2f2 := path.Join(p, "d2/f2")
+		d2f3 := path.Join(p, "d2/f3")
+
+		graph, err := pgraph.NewGraph("test")
+		if err != nil {
+			panic("can't make graph")
+		}
+		graph.AddVertex(res) // add self
+
+		timeline := []func() error{
+			fileMkdir(p, true),
+			fileWrite(f1, "f1\n"),
+			fileWrite(f2, "f2\n"),
+			fileMkdir(d1, true),
+			fileMkdir(d2, true),
+			fileWrite(d1f1, "d1f1\n"),
+			fileWrite(d1f2, "d1f2\n"),
+			fileWrite(d2f1, "d2f1\n"),
+			fileWrite(d2f2, "d2f2\n"),
+			fileWrite(d2f3, "d2f3\n"),
+			resValidate(r1),
+			resInit(r1, addGraph(graph)),
+			resCheckApply(r1, false), // changed
+			fileExists(p, true),      // ensure it's a dir
+			fileAbsent(f1),           // ensure it's absent
+			fileAbsent(f2),
+			fileAbsent(d1),
+			fileAbsent(d2),
+			fileAbsent(d1f1),
+			fileAbsent(d1f2),
+			fileAbsent(d2f1),
+			fileAbsent(d2f2),
+			fileAbsent(d2f3),
+			resCheckApply(r1, true), // it's already good
+			resClose(r1),
+		}
+
+		testCases = append(testCases, test{
+			name:     "dir purge",
+			timeline: timeline,
+			expect:   func() error { return nil },
+			startup:  func() error { return nil },
+			cleanup:  func() error { return os.RemoveAll(p) },
+		})
+	}
+	{
+		//file "/tmp/somedir/" {
+		//	state => $const.res.file.state.exists,
+		//	recurse => true,
+		//	purge => true,
+		//}
+		// TODO: should State be required for these to not delete them?
+		//file "/tmp/somedir/hello" {
+		//}
+		//file "/tmp/somedir/nested-dir/" {
+		//}
+		//file "/tmp/somedir/nested-dir/nestedfileindir" {
+		//}
+		r1 := makeRes("file", "r1")
+		res := r1.(*FileRes) // if this panics, the test will panic
+		p := "/tmp/somedir/"
+		res.Path = p
+		res.State = FileStateExists
+		res.Recurse = true
+		res.Purge = true
+
+		f1 := path.Join(p, "f1")
+		f2 := path.Join(p, "f2")
+		d1 := path.Join(p, "d1/")
+		d2 := path.Join(p, "d2/")
+		d1f1 := path.Join(p, "d1/f1")
+		d1f2 := path.Join(p, "d1/f2")
+		d2f1 := path.Join(p, "d2/f1")
+		d2f2 := path.Join(p, "d2/f2")
+		d2f3 := path.Join(p, "d2/f3")
+
+		r2 := makeRes("file", "r2")
+		res2 := r2.(*FileRes)
+		p2 := path.Join(p, "hello")
+		res2.Path = p2
+		p2c := "i am a hello file\n"
+		// TODO: should State be required for this to not delete it?
+
+		r3 := makeRes("file", "r3")
+		res3 := r3.(*FileRes)
+		p3 := path.Join(p, "nested-dir/")
+		res3.Path = p3
+		// TODO: should State be required for this to not delete it?
+
+		r4 := makeRes("file", "r4")
+		res4 := r4.(*FileRes)
+		p4 := path.Join(p3, "nestedfileindir")
+		res4.Path = p4
+		p4c := "i am a nested file\n"
+		// TODO: should State be required for this to not delete it?
+
+		graph, err := pgraph.NewGraph("test")
+		if err != nil {
+			panic("can't make graph")
+		}
+		graph.AddVertex(res, res2, res3, res4)
+
+		timeline := []func() error{
+			fileMkdir(p, true),
+			fileWrite(f1, "f1\n"),
+			fileWrite(f2, "f2\n"),
+			fileMkdir(d1, true),
+			fileMkdir(d2, true),
+			fileWrite(d1f1, "d1f1\n"),
+			fileWrite(d1f2, "d1f2\n"),
+			fileWrite(d2f1, "d2f1\n"),
+			fileWrite(d2f2, "d2f2\n"),
+			fileWrite(d2f3, "d2f3\n"),
+			fileWrite(p2, p2c),
+			fileMkdir(p3, true),
+			fileWrite(p4, p4c),
+
+			resValidate(r2),
+			resInit(r2),
+			//resCheckApply(r2, false), // not really needed in test
+			resClose(r2),
+
+			resValidate(r3),
+			resInit(r3),
+			//resCheckApply(r3, false), // not really needed in test
+			resClose(r3),
+
+			resValidate(r4),
+			resInit(r4),
+			//resCheckApply(r4, false), // not really needed in test
+			resClose(r4),
+
+			resValidate(r1),
+			resInit(r1, addGraph(graph), addLogf(nil)), // show the full graph
+			resCheckApply(r1, false),                   // changed
+			fileExists(p, true),                        // ensure it's a dir
+			fileAbsent(f1),                             // ensure it's absent
+			fileAbsent(f2),
+			fileAbsent(d1),
+			fileAbsent(d2),
+			fileAbsent(d1f1),
+			fileAbsent(d1f2),
+			fileAbsent(d2f1),
+			fileAbsent(d2f2),
+			fileAbsent(d2f3),
+			fileExists(p2, false), // ensure it's a file XXX !!!
+			fileExists(p3, true),  // ensure it's a dir
+			fileExists(p4, false),
+			resCheckApply(r1, true), // it's already good
+			resClose(r1),
+		}
+
+		testCases = append(testCases, test{
+			name:     "dir purge with others inside",
+			timeline: timeline,
+			expect:   func() error { return nil },
+			startup:  func() error { return nil },
+			cleanup:  func() error { return os.RemoveAll(p) },
+		})
+	}
 	names := []string{}
 	for index, tc := range testCases { // run all the tests
 		if tc.name == "" {
@@ -1144,12 +1706,17 @@ func TestResources2(t *testing.T) {
 			//	t.Logf(fmt.Sprintf("test #%d: ", index)+format, v...)
 			//}
 
-			t.Logf("test #%d: running startup()", index)
+			if startup != nil {
-			if err := startup(); err != nil {
+				t.Logf("test #%d: running startup()", index)
-				t.Errorf("test #%d: FAIL", index)
+				if err := startup(); err != nil {
-				t.Errorf("test #%d: could not startup: %+v", index, err)
+					t.Errorf("test #%d: FAIL", index)
+					t.Errorf("test #%d: could not startup: %+v", index, err)
+				}
 			}
 			defer func() {
+				if cleanup == nil {
+					return
+				}
 				t.Logf("test #%d: running cleanup()", index)
 				if err := cleanup(); err != nil {
 					t.Errorf("test #%d: FAIL", index)

engine/resources/svc.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -419,7 +419,8 @@ func (obj *SvcResAutoEdges) Next() []engine.ResUID {
 	return []engine.ResUID{value} // we return one, even though api supports N
 }
 
-// Test gets results of the earlier Next() call, & returns if we should continue!
+// Test gets results of the earlier Next() call, & returns if we should
+// continue!
 func (obj *SvcResAutoEdges) Test(input []bool) bool {
 	// if there aren't any more remaining
 	if len(obj.data) <= obj.pointer {
@@ -513,8 +514,8 @@ func (obj *SvcRes) AutoEdges() (engine.AutoEdge, error) {
 	return engineUtil.AutoEdgeCombiner(fileEdge, cronEdge)
 }
 
-// UIDs includes all params to make a unique identification of this object.
+// UIDs includes all params to make a unique identification of this object. Most
-// Most resources only return one, although some resources can return multiple.
+// resources only return one, although some resources can return multiple.
 func (obj *SvcRes) UIDs() []engine.ResUID {
 	x := &SvcUID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
@@ -536,8 +537,8 @@ func (obj *SvcRes) UIDs() []engine.ResUID {
 //	return fmt.Errorf("not possible at the moment")
 //}
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
-// It is primarily useful for setting the defaults.
+// primarily useful for setting the defaults.
 func (obj *SvcRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes SvcRes // indirection to avoid infinite recursion
 

engine/resources/test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -367,8 +367,8 @@ type TestUID struct {
 	name string
 }
 
-// UIDs includes all params to make a unique identification of this object.
+// UIDs includes all params to make a unique identification of this object. Most
-// Most resources only return one, although some resources can return multiple.
+// resources only return one, although some resources can return multiple.
 func (obj *TestRes) UIDs() []engine.ResUID {
 	x := &TestUID{
 		BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
@@ -405,8 +405,8 @@ func (obj *TestRes) Sends() interface{} {
 	}
 }
 
-// UnmarshalYAML is the custom unmarshal handler for this struct.
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
-// It is primarily useful for setting the defaults.
+// primarily useful for setting the defaults.
 func (obj *TestRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawRes TestRes // indirection to avoid infinite recursion